#!/usr/bin/env bash set -euo pipefail SCRIPT_DIR="$(CDPATH= cd -- "$(dirname -- "$0")" && pwd)" # shellcheck disable=SC1091 source "$SCRIPT_DIR/common.sh" LOCK_FILE="${DEPLOY_CLEANUP_LOCK_FILE:-$RUN_DIR/deploy-disk-cleanup.lock}" LOG_DIR="${DEPLOY_CLEANUP_LOG_DIR:-$RUN_DIR/cleanup}" mkdir -p "$LOG_DIR" if command -v flock >/dev/null 2>&1; then exec 9>"$LOCK_FILE" if ! flock -n 9; then echo "deploy disk cleanup is already running" exit 0 fi else echo "flock not found, continuing without process lock" >&2 fi LOG_FILE="$LOG_DIR/deploy-disk-cleanup-$(date +%Y%m%d-%H%M%S).log" exec > >(tee -a "$LOG_FILE") 2>&1 KEEP_DAYS="${DEPLOY_CLEANUP_KEEP_DAYS:-3}" JOB_LOG_KEEP_DAYS="${DEPLOY_CLEANUP_JOB_LOG_KEEP_DAYS:-$KEEP_DAYS}" CLEANUP_LOG_KEEP_DAYS="${DEPLOY_CLEANUP_LOG_KEEP_DAYS:-14}" BUILD_ROOTS="${DEPLOY_CLEANUP_BUILD_ROOTS:-/opt/deploy/builds /opt/deploy/build-cache}" DRY_RUN="${DEPLOY_CLEANUP_DRY_RUN:-0}" require_positive_int() { local name="$1" local value="$2" if [[ ! "$value" =~ ^[1-9][0-9]*$ ]]; then echo "$name must be a positive integer, got: $value" >&2 exit 2 fi } require_positive_int DEPLOY_CLEANUP_KEEP_DAYS "$KEEP_DAYS" require_positive_int DEPLOY_CLEANUP_JOB_LOG_KEEP_DAYS "$JOB_LOG_KEEP_DAYS" require_positive_int DEPLOY_CLEANUP_LOG_KEEP_DAYS "$CLEANUP_LOG_KEEP_DAYS" PRUNE_UNTIL="${DEPLOY_CLEANUP_DOCKER_PRUNE_UNTIL:-$((KEEP_DAYS * 24))h}" KEEP_MINUTES=$((KEEP_DAYS * 24 * 60)) JOB_LOG_KEEP_MINUTES=$((JOB_LOG_KEEP_DAYS * 24 * 60)) CLEANUP_LOG_KEEP_MINUTES=$((CLEANUP_LOG_KEEP_DAYS * 24 * 60)) log_section() { printf '\n--- %s ---\n' "$1" } timestamp() { date '+%Y-%m-%dT%H:%M:%S%z' } run_or_echo() { if [[ "$DRY_RUN" == "1" || "$DRY_RUN" == "true" ]]; then printf '[dry-run]' printf ' %q' "$@" printf '\n' return 0 fi "$@" } show_root_disk() { local output output="$(df -hT / 2>/dev/null || true)" if [ -n "$output" ]; then printf '%s\n' "$output" return 0 fi df -h / } cleanup_build_roots() { log_section "cleanup build directories older than ${KEEP_DAYS}d" local root for root in $BUILD_ROOTS; do if [ ! -d "$root" ]; then echo "skip missing build root: $root" continue fi echo "build root: $root" if [[ "$DRY_RUN" == "1" || "$DRY_RUN" == "true" ]]; then find "$root" -mindepth 2 -maxdepth 2 -type d -mmin +"$KEEP_MINUTES" -print continue fi find "$root" -mindepth 2 -maxdepth 2 -type d -mmin +"$KEEP_MINUTES" -print -exec rm -rf {} + find "$root" -mindepth 1 -maxdepth 1 -type d -empty -print -delete done } cleanup_job_logs() { log_section "cleanup run job logs older than ${JOB_LOG_KEEP_DAYS}d" local dir for dir in "$RUN_DIR/service_update_jobs" "$RUN_DIR/chatapp_cron_deploy_jobs"; do if [ ! -d "$dir" ]; then echo "skip missing job dir: $dir" continue fi echo "job dir: $dir" if [[ "$DRY_RUN" == "1" || "$DRY_RUN" == "true" ]]; then find "$dir" -type f \( -name '*.json' -o -name '*.log' \) -mmin +"$JOB_LOG_KEEP_MINUTES" -print continue fi find "$dir" -type f \( -name '*.json' -o -name '*.log' \) -mmin +"$JOB_LOG_KEEP_MINUTES" -print -delete done } cleanup_own_logs() { log_section "cleanup cleanup logs older than ${CLEANUP_LOG_KEEP_DAYS}d" if [[ "$DRY_RUN" == "1" || "$DRY_RUN" == "true" ]]; then find "$LOG_DIR" -type f -name 'deploy-disk-cleanup-*.log' -mmin +"$CLEANUP_LOG_KEEP_MINUTES" -print return 0 fi find "$LOG_DIR" -type f -name 'deploy-disk-cleanup-*.log' -mmin +"$CLEANUP_LOG_KEEP_MINUTES" -print -delete } cleanup_docker() { log_section "docker prune older than ${PRUNE_UNTIL}" if ! command -v docker >/dev/null 2>&1; then echo "docker not found, skip" return 0 fi docker system df || true run_or_echo docker builder prune -af --filter "until=$PRUNE_UNTIL" run_or_echo docker image prune -af --filter "until=$PRUNE_UNTIL" docker system df || true } configure_harbor_retention_and_gc() { log_section "ensure harbor retention and gc schedule" if [[ "${HARBOR_RETENTION_ENABLED:-1}" =~ ^(0|false|FALSE|no|NO|off|OFF)$ ]]; then echo "harbor retention disabled by HARBOR_RETENTION_ENABLED" return 0 fi if [ -z "${HARBOR_USERNAME:-}" ] || [ -z "${HARBOR_PASSWORD:-}" ]; then echo "HARBOR_USERNAME/HARBOR_PASSWORD missing, skip harbor retention/gc" return 0 fi export HARBOR_API_URL="${HARBOR_API_URL:-http://127.0.0.1:18082}" export HARBOR_RETENTION_KEEP_LATEST="${HARBOR_RETENTION_KEEP_LATEST:-20}" export HARBOR_RETENTION_CRON="${HARBOR_RETENTION_CRON:-0 0 2 * * *}" export HARBOR_GC_CRON="${HARBOR_GC_CRON:-0 0 4 * * *}" export DEPLOY_CLEANUP_DRY_RUN="$DRY_RUN" "$PYTHON_BIN" - <<'PY' from __future__ import annotations import base64 import json import os import re import sys import urllib.error import urllib.parse import urllib.request def truthy(value: str) -> bool: return value.strip().lower() in {"1", "true", "yes", "on"} base_url = os.environ.get("HARBOR_API_URL", "http://127.0.0.1:18082").rstrip("/") + "/api/v2.0" project_name = os.environ.get("HARBOR_PROJECT", "likei").strip() or "likei" username = os.environ.get("HARBOR_USERNAME", "") password = os.environ.get("HARBOR_PASSWORD", "") keep_latest = int(os.environ.get("HARBOR_RETENTION_KEEP_LATEST", "20")) retention_cron = os.environ.get("HARBOR_RETENTION_CRON", "0 0 2 * * *").strip() gc_cron = os.environ.get("HARBOR_GC_CRON", "0 0 4 * * *").strip() dry_run = truthy(os.environ.get("DEPLOY_CLEANUP_DRY_RUN", "0")) if keep_latest < 1: raise SystemExit("HARBOR_RETENTION_KEEP_LATEST must be positive") if not re.fullmatch(r"\S+\s+\S+\s+\S+\s+\S+\s+\S+\s+\S+", retention_cron): raise SystemExit(f"invalid HARBOR_RETENTION_CRON: {retention_cron}") if not re.fullmatch(r"\S+\s+\S+\s+\S+\s+\S+\s+\S+\s+\S+", gc_cron): raise SystemExit(f"invalid HARBOR_GC_CRON: {gc_cron}") auth = base64.b64encode(f"{username}:{password}".encode("utf-8")).decode("ascii") headers = { "Authorization": f"Basic {auth}", "Accept": "application/json", "Content-Type": "application/json", } def request(method: str, path: str, payload: dict | None = None) -> tuple[int, object, dict[str, str]]: data = json.dumps(payload).encode("utf-8") if payload is not None else None req = urllib.request.Request(base_url + path, data=data, method=method, headers=headers) try: with urllib.request.urlopen(req, timeout=20) as resp: text = resp.read().decode("utf-8", errors="replace") body: object = None if text.strip(): try: body = json.loads(text) except json.JSONDecodeError: body = text return resp.status, body, dict(resp.headers.items()) except urllib.error.HTTPError as exc: text = exc.read().decode("utf-8", errors="replace") raise RuntimeError(f"harbor {method} {path} failed: {exc.code} {text[:500]}") from exc def project_path(path: str = "") -> str: encoded = urllib.parse.quote(project_name, safe="") return f"/projects/{encoded}{path}" def retention_policy(project_id: int, policy_id: int | None = None, rule_id: int | None = None) -> dict: rule = { "priority": 1, "disabled": False, "action": "retain", "template": "latestPushedK", "params": {"latestPushedK": keep_latest}, "tag_selectors": [ {"kind": "doublestar", "decoration": "matches", "pattern": "**"}, ], "scope_selectors": { "repository": [ {"kind": "doublestar", "decoration": "repoMatches", "pattern": "**"}, ], }, } if rule_id is not None: rule["id"] = rule_id policy = { "algorithm": "or", "rules": [rule], "trigger": {"kind": "Schedule", "settings": {"cron": retention_cron}}, "scope": {"level": "project", "ref": project_id}, } if policy_id is not None: policy["id"] = policy_id return policy status, project, _ = request("GET", project_path()) if not isinstance(project, dict): raise RuntimeError(f"unexpected harbor project response: {project!r}") project_id = int(project["project_id"]) metadata = dict(project.get("metadata") or {}) retention_id_text = str(metadata.get("retention_id") or "").strip() retention_id = int(retention_id_text) if retention_id_text.isdigit() else None print(f"project={project_name} project_id={project_id} retention_id={retention_id or '-'}") if retention_id is None: payload = retention_policy(project_id) if dry_run: print("[dry-run] create retention", json.dumps(payload, ensure_ascii=False)) else: status, _, response_headers = request("POST", "/retentions", payload) location = response_headers.get("Location") or response_headers.get("location") or "" match = re.search(r"/retentions/(\d+)", location) if match: retention_id = int(match.group(1)) status, refreshed_project, _ = request("GET", project_path()) refreshed_metadata = dict((refreshed_project or {}).get("metadata") or {}) if isinstance(refreshed_project, dict) else {} refreshed_id = str(refreshed_metadata.get("retention_id") or "").strip() if refreshed_id.isdigit(): retention_id = int(refreshed_id) elif retention_id is None: raise RuntimeError(f"retention created but id was not returned: status={status} location={location!r}") print(f"created retention_id={retention_id}") else: status, existing_policy, _ = request("GET", f"/retentions/{retention_id}") existing_rule_id = None if isinstance(existing_policy, dict): existing_rules = existing_policy.get("rules") or [] if existing_rules and isinstance(existing_rules[0], dict) and existing_rules[0].get("id") is not None: existing_rule_id = int(existing_rules[0]["id"]) payload = retention_policy(project_id, policy_id=retention_id, rule_id=existing_rule_id) if dry_run: print("[dry-run] update retention", json.dumps(payload, ensure_ascii=False)) else: request("PUT", f"/retentions/{retention_id}", payload) print(f"updated retention_id={retention_id}") gc_payload = { "schedule": {"type": "Custom", "cron": gc_cron}, "parameters": {}, } status, current_gc, _ = request("GET", "/system/gc/schedule") gc_method = "POST" if not current_gc else "PUT" if dry_run: print(f"[dry-run] {gc_method} gc schedule", json.dumps(gc_payload, ensure_ascii=False)) else: request(gc_method, "/system/gc/schedule", gc_payload) print(f"gc schedule ensured cron={gc_cron}") PY } main() { echo "started_at=$(timestamp)" echo "log_file=$LOG_FILE" echo "dry_run=$DRY_RUN keep_days=$KEEP_DAYS prune_until=$PRUNE_UNTIL" log_section "disk before" show_root_disk cleanup_build_roots cleanup_job_logs cleanup_docker configure_harbor_retention_and_gc cleanup_own_logs log_section "disk after" show_root_disk echo "finished_at=$(timestamp)" } main "$@"