From 96f1c9dacb8a8e864af7bd0fb4631bbb3c796ec5 Mon Sep 17 00:00:00 2001 From: 170-carry Date: Wed, 29 Apr 2026 16:27:00 +0800 Subject: [PATCH] add deploy disk cleanup timer --- scripts/cleanup_deploy_disk.sh | 336 +++++++++++++++++++++++++ scripts/install_systemd.sh | 18 ++ scripts/uninstall_systemd.sh | 3 + systemd/hy-app-monitor-cleanup.service | 11 + systemd/hy-app-monitor-cleanup.timer | 11 + 5 files changed, 379 insertions(+) create mode 100644 scripts/cleanup_deploy_disk.sh create mode 100644 systemd/hy-app-monitor-cleanup.service create mode 100644 systemd/hy-app-monitor-cleanup.timer diff --git a/scripts/cleanup_deploy_disk.sh b/scripts/cleanup_deploy_disk.sh new file mode 100644 index 0000000..9c02eeb --- /dev/null +++ b/scripts/cleanup_deploy_disk.sh @@ -0,0 +1,336 @@ +#!/usr/bin/env bash +set -euo pipefail + +SCRIPT_DIR="$(CDPATH= cd -- "$(dirname -- "$0")" && pwd)" +# shellcheck disable=SC1091 +source "$SCRIPT_DIR/common.sh" + +LOCK_FILE="${DEPLOY_CLEANUP_LOCK_FILE:-$RUN_DIR/deploy-disk-cleanup.lock}" +LOG_DIR="${DEPLOY_CLEANUP_LOG_DIR:-$RUN_DIR/cleanup}" +mkdir -p "$LOG_DIR" + +if command -v flock >/dev/null 2>&1; then + exec 9>"$LOCK_FILE" + if ! flock -n 9; then + echo "deploy disk cleanup is already running" + exit 0 + fi +else + echo "flock not found, continuing without process lock" >&2 +fi + +LOG_FILE="$LOG_DIR/deploy-disk-cleanup-$(date +%Y%m%d-%H%M%S).log" +exec > >(tee -a "$LOG_FILE") 2>&1 + +KEEP_DAYS="${DEPLOY_CLEANUP_KEEP_DAYS:-3}" +JOB_LOG_KEEP_DAYS="${DEPLOY_CLEANUP_JOB_LOG_KEEP_DAYS:-$KEEP_DAYS}" +CLEANUP_LOG_KEEP_DAYS="${DEPLOY_CLEANUP_LOG_KEEP_DAYS:-14}" +BUILD_ROOTS="${DEPLOY_CLEANUP_BUILD_ROOTS:-/opt/deploy/builds /opt/deploy/build-cache}" +DRY_RUN="${DEPLOY_CLEANUP_DRY_RUN:-0}" + +require_positive_int() { + local name="$1" + local value="$2" + if [[ ! "$value" =~ ^[1-9][0-9]*$ ]]; then + echo "$name must be a positive integer, got: $value" >&2 + exit 2 + fi +} + +require_positive_int DEPLOY_CLEANUP_KEEP_DAYS "$KEEP_DAYS" +require_positive_int DEPLOY_CLEANUP_JOB_LOG_KEEP_DAYS "$JOB_LOG_KEEP_DAYS" +require_positive_int DEPLOY_CLEANUP_LOG_KEEP_DAYS "$CLEANUP_LOG_KEEP_DAYS" + +PRUNE_UNTIL="${DEPLOY_CLEANUP_DOCKER_PRUNE_UNTIL:-$((KEEP_DAYS * 24))h}" +KEEP_MINUTES=$((KEEP_DAYS * 24 * 60)) +JOB_LOG_KEEP_MINUTES=$((JOB_LOG_KEEP_DAYS * 24 * 60)) +CLEANUP_LOG_KEEP_MINUTES=$((CLEANUP_LOG_KEEP_DAYS * 24 * 60)) + +log_section() { + printf '\n--- %s ---\n' "$1" +} + +timestamp() { + date '+%Y-%m-%dT%H:%M:%S%z' +} + +run_or_echo() { + if [[ "$DRY_RUN" == "1" || "$DRY_RUN" == "true" ]]; then + printf '[dry-run]' + printf ' %q' "$@" + printf '\n' + return 0 + fi + "$@" +} + +show_root_disk() { + local output + output="$(df -hT / 2>/dev/null || true)" + if [ -n "$output" ]; then + printf '%s\n' "$output" + return 0 + fi + df -h / +} + +cleanup_build_roots() { + log_section "cleanup build directories older than ${KEEP_DAYS}d" + local root + for root in $BUILD_ROOTS; do + if [ ! -d "$root" ]; then + echo "skip missing build root: $root" + continue + fi + + echo "build root: $root" + if [[ "$DRY_RUN" == "1" || "$DRY_RUN" == "true" ]]; then + find "$root" -mindepth 2 -maxdepth 2 -type d -mmin +"$KEEP_MINUTES" -print + continue + fi + + find "$root" -mindepth 2 -maxdepth 2 -type d -mmin +"$KEEP_MINUTES" -print -exec rm -rf {} + + find "$root" -mindepth 1 -maxdepth 1 -type d -empty -print -delete + done +} + +cleanup_job_logs() { + log_section "cleanup run job logs older than ${JOB_LOG_KEEP_DAYS}d" + local dir + for dir in "$RUN_DIR/service_update_jobs" "$RUN_DIR/chatapp_cron_deploy_jobs"; do + if [ ! -d "$dir" ]; then + echo "skip missing job dir: $dir" + continue + fi + + echo "job dir: $dir" + if [[ "$DRY_RUN" == "1" || "$DRY_RUN" == "true" ]]; then + find "$dir" -type f \( -name '*.json' -o -name '*.log' \) -mmin +"$JOB_LOG_KEEP_MINUTES" -print + continue + fi + + find "$dir" -type f \( -name '*.json' -o -name '*.log' \) -mmin +"$JOB_LOG_KEEP_MINUTES" -print -delete + done +} + +cleanup_own_logs() { + log_section "cleanup cleanup logs older than ${CLEANUP_LOG_KEEP_DAYS}d" + if [[ "$DRY_RUN" == "1" || "$DRY_RUN" == "true" ]]; then + find "$LOG_DIR" -type f -name 'deploy-disk-cleanup-*.log' -mmin +"$CLEANUP_LOG_KEEP_MINUTES" -print + return 0 + fi + find "$LOG_DIR" -type f -name 'deploy-disk-cleanup-*.log' -mmin +"$CLEANUP_LOG_KEEP_MINUTES" -print -delete +} + +cleanup_docker() { + log_section "docker prune older than ${PRUNE_UNTIL}" + if ! command -v docker >/dev/null 2>&1; then + echo "docker not found, skip" + return 0 + fi + + docker system df || true + run_or_echo docker builder prune -af --filter "until=$PRUNE_UNTIL" + run_or_echo docker image prune -af --filter "until=$PRUNE_UNTIL" + docker system df || true +} + +configure_harbor_retention_and_gc() { + log_section "ensure harbor retention and gc schedule" + + if [[ "${HARBOR_RETENTION_ENABLED:-1}" =~ ^(0|false|FALSE|no|NO|off|OFF)$ ]]; then + echo "harbor retention disabled by HARBOR_RETENTION_ENABLED" + return 0 + fi + + if [ -z "${HARBOR_USERNAME:-}" ] || [ -z "${HARBOR_PASSWORD:-}" ]; then + echo "HARBOR_USERNAME/HARBOR_PASSWORD missing, skip harbor retention/gc" + return 0 + fi + + export HARBOR_API_URL="${HARBOR_API_URL:-http://127.0.0.1:18082}" + export HARBOR_RETENTION_KEEP_LATEST="${HARBOR_RETENTION_KEEP_LATEST:-20}" + export HARBOR_RETENTION_CRON="${HARBOR_RETENTION_CRON:-0 0 2 * * *}" + export HARBOR_GC_CRON="${HARBOR_GC_CRON:-0 0 4 * * *}" + export DEPLOY_CLEANUP_DRY_RUN="$DRY_RUN" + + "$PYTHON_BIN" - <<'PY' +from __future__ import annotations + +import base64 +import json +import os +import re +import sys +import urllib.error +import urllib.parse +import urllib.request + + +def truthy(value: str) -> bool: + return value.strip().lower() in {"1", "true", "yes", "on"} + + +base_url = os.environ.get("HARBOR_API_URL", "http://127.0.0.1:18082").rstrip("/") + "/api/v2.0" +project_name = os.environ.get("HARBOR_PROJECT", "likei").strip() or "likei" +username = os.environ.get("HARBOR_USERNAME", "") +password = os.environ.get("HARBOR_PASSWORD", "") +keep_latest = int(os.environ.get("HARBOR_RETENTION_KEEP_LATEST", "20")) +retention_cron = os.environ.get("HARBOR_RETENTION_CRON", "0 0 2 * * *").strip() +gc_cron = os.environ.get("HARBOR_GC_CRON", "0 0 4 * * *").strip() +dry_run = truthy(os.environ.get("DEPLOY_CLEANUP_DRY_RUN", "0")) + +if keep_latest < 1: + raise SystemExit("HARBOR_RETENTION_KEEP_LATEST must be positive") + +if not re.fullmatch(r"\S+\s+\S+\s+\S+\s+\S+\s+\S+\s+\S+", retention_cron): + raise SystemExit(f"invalid HARBOR_RETENTION_CRON: {retention_cron}") + +if not re.fullmatch(r"\S+\s+\S+\s+\S+\s+\S+\s+\S+\s+\S+", gc_cron): + raise SystemExit(f"invalid HARBOR_GC_CRON: {gc_cron}") + +auth = base64.b64encode(f"{username}:{password}".encode("utf-8")).decode("ascii") +headers = { + "Authorization": f"Basic {auth}", + "Accept": "application/json", + "Content-Type": "application/json", +} + + +def request(method: str, path: str, payload: dict | None = None) -> tuple[int, object, dict[str, str]]: + data = json.dumps(payload).encode("utf-8") if payload is not None else None + req = urllib.request.Request(base_url + path, data=data, method=method, headers=headers) + try: + with urllib.request.urlopen(req, timeout=20) as resp: + text = resp.read().decode("utf-8", errors="replace") + body: object = None + if text.strip(): + try: + body = json.loads(text) + except json.JSONDecodeError: + body = text + return resp.status, body, dict(resp.headers.items()) + except urllib.error.HTTPError as exc: + text = exc.read().decode("utf-8", errors="replace") + raise RuntimeError(f"harbor {method} {path} failed: {exc.code} {text[:500]}") from exc + + +def project_path(path: str = "") -> str: + encoded = urllib.parse.quote(project_name, safe="") + return f"/projects/{encoded}{path}" + + +def retention_policy(project_id: int, policy_id: int | None = None, rule_id: int | None = None) -> dict: + rule = { + "priority": 1, + "disabled": False, + "action": "retain", + "template": "latestPushedK", + "params": {"latestPushedK": keep_latest}, + "tag_selectors": [ + {"kind": "doublestar", "decoration": "matches", "pattern": "**"}, + ], + "scope_selectors": { + "repository": [ + {"kind": "doublestar", "decoration": "repoMatches", "pattern": "**"}, + ], + }, + } + if rule_id is not None: + rule["id"] = rule_id + + policy = { + "algorithm": "or", + "rules": [rule], + "trigger": {"kind": "Schedule", "settings": {"cron": retention_cron}}, + "scope": {"level": "project", "ref": project_id}, + } + if policy_id is not None: + policy["id"] = policy_id + return policy + + +status, project, _ = request("GET", project_path()) +if not isinstance(project, dict): + raise RuntimeError(f"unexpected harbor project response: {project!r}") + +project_id = int(project["project_id"]) +metadata = dict(project.get("metadata") or {}) +retention_id_text = str(metadata.get("retention_id") or "").strip() +retention_id = int(retention_id_text) if retention_id_text.isdigit() else None +print(f"project={project_name} project_id={project_id} retention_id={retention_id or '-'}") + +if retention_id is None: + payload = retention_policy(project_id) + if dry_run: + print("[dry-run] create retention", json.dumps(payload, ensure_ascii=False)) + else: + status, _, response_headers = request("POST", "/retentions", payload) + location = response_headers.get("Location") or response_headers.get("location") or "" + match = re.search(r"/retentions/(\d+)", location) + if not match: + status, refreshed_project, _ = request("GET", project_path()) + refreshed_metadata = dict((refreshed_project or {}).get("metadata") or {}) if isinstance(refreshed_project, dict) else {} + refreshed_id = str(refreshed_metadata.get("retention_id") or "").strip() + if not refreshed_id.isdigit(): + raise RuntimeError(f"retention created but id was not returned: status={status} location={location!r}") + retention_id = int(refreshed_id) + else: + retention_id = int(match.group(1)) + + try: + request("POST", project_path("/metadatas/"), {"retention_id": str(retention_id)}) + except RuntimeError as exc: + if "409" in str(exc): + request("PUT", project_path("/metadatas/retention_id"), {"retention_id": str(retention_id)}) + else: + raise + print(f"created retention_id={retention_id}") +else: + status, existing_policy, _ = request("GET", f"/retentions/{retention_id}") + existing_rule_id = None + if isinstance(existing_policy, dict): + existing_rules = existing_policy.get("rules") or [] + if existing_rules and isinstance(existing_rules[0], dict) and existing_rules[0].get("id") is not None: + existing_rule_id = int(existing_rules[0]["id"]) + payload = retention_policy(project_id, policy_id=retention_id, rule_id=existing_rule_id) + if dry_run: + print("[dry-run] update retention", json.dumps(payload, ensure_ascii=False)) + else: + request("PUT", f"/retentions/{retention_id}", payload) + print(f"updated retention_id={retention_id}") + +gc_payload = { + "schedule": {"type": "Schedule", "cron": gc_cron}, + "parameters": {}, +} +status, current_gc, _ = request("GET", "/system/gc/schedule") +gc_method = "POST" if not current_gc else "PUT" +if dry_run: + print(f"[dry-run] {gc_method} gc schedule", json.dumps(gc_payload, ensure_ascii=False)) +else: + request(gc_method, "/system/gc/schedule", gc_payload) + print(f"gc schedule ensured cron={gc_cron}") +PY +} + +main() { + echo "started_at=$(timestamp)" + echo "log_file=$LOG_FILE" + echo "dry_run=$DRY_RUN keep_days=$KEEP_DAYS prune_until=$PRUNE_UNTIL" + + log_section "disk before" + show_root_disk + + cleanup_build_roots + cleanup_job_logs + cleanup_docker + configure_harbor_retention_and_gc + cleanup_own_logs + + log_section "disk after" + show_root_disk + echo "finished_at=$(timestamp)" +} + +main "$@" diff --git a/scripts/install_systemd.sh b/scripts/install_systemd.sh index d2b33ae..48a8ab4 100755 --- a/scripts/install_systemd.sh +++ b/scripts/install_systemd.sh @@ -25,10 +25,27 @@ install_unit() { install -m 0644 "$src" "$dst" } +install_extra_unit() { + local unit="$1" + local src + local dst + src="$ROOT_DIR/systemd/$unit" + dst="/etc/systemd/system/$unit" + + if [ ! -f "$src" ]; then + echo "systemd unit missing: $src" >&2 + exit 1 + fi + + install -m 0644 "$src" "$dst" +} + name="" for name in $(suite_service_names); do install_unit "$name" done +install_extra_unit "hy-app-monitor-cleanup.service" +install_extra_unit "hy-app-monitor-cleanup.timer" chmod +x "$ROOT_DIR"/scripts/*.sh stop_all_legacy_processes @@ -37,3 +54,4 @@ systemctl daemon-reload for name in $(suite_service_names); do systemctl enable "$(service_unit_name "$name")" done +systemctl enable --now hy-app-monitor-cleanup.timer diff --git a/scripts/uninstall_systemd.sh b/scripts/uninstall_systemd.sh index 73f58e4..47c7a70 100755 --- a/scripts/uninstall_systemd.sh +++ b/scripts/uninstall_systemd.sh @@ -6,6 +6,7 @@ SCRIPT_DIR="$(CDPATH= cd -- "$(dirname -- "$0")" && pwd)" source "$SCRIPT_DIR/common.sh" if command -v systemctl >/dev/null 2>&1; then + systemctl disable --now hy-app-monitor-cleanup.timer >/dev/null 2>&1 || true name="" for name in $(suite_service_names); do systemctl disable --now "$(service_unit_name "$name")" >/dev/null 2>&1 || true @@ -16,6 +17,8 @@ name="" for name in $(suite_service_names); do rm -f "/etc/systemd/system/$(service_unit_name "$name")" done +rm -f /etc/systemd/system/hy-app-monitor-cleanup.service +rm -f /etc/systemd/system/hy-app-monitor-cleanup.timer if command -v systemctl >/dev/null 2>&1; then systemctl daemon-reload || true diff --git a/systemd/hy-app-monitor-cleanup.service b/systemd/hy-app-monitor-cleanup.service new file mode 100644 index 0000000..626ab57 --- /dev/null +++ b/systemd/hy-app-monitor-cleanup.service @@ -0,0 +1,11 @@ +[Unit] +Description=HY App Monitor Deploy Disk Cleanup +After=network-online.target docker.service +Wants=network-online.target + +[Service] +Type=oneshot +User=root +WorkingDirectory=/opt/hy-app-monitor +EnvironmentFile=-/opt/hy-app-monitor/.env +ExecStart=/opt/hy-app-monitor/scripts/cleanup_deploy_disk.sh diff --git a/systemd/hy-app-monitor-cleanup.timer b/systemd/hy-app-monitor-cleanup.timer new file mode 100644 index 0000000..97db0fc --- /dev/null +++ b/systemd/hy-app-monitor-cleanup.timer @@ -0,0 +1,11 @@ +[Unit] +Description=Run HY App Monitor deploy disk cleanup daily + +[Timer] +OnCalendar=*-*-* 03:30:00 +RandomizedDelaySec=10m +Persistent=true +Unit=hy-app-monitor-cleanup.service + +[Install] +WantedBy=timers.target