add deploy disk cleanup timer
This commit is contained in:
170-carry
parent
b7c388db3e
commit
96f1c9dacb
336
scripts/cleanup_deploy_disk.sh
Normal file
336
scripts/cleanup_deploy_disk.sh
Normal file
@ -0,0 +1,336 @@
|
||||
#!/usr/bin/env bash
|
||||
set -euo pipefail
|
||||
|
||||
SCRIPT_DIR="$(CDPATH= cd -- "$(dirname -- "$0")" && pwd)"
|
||||
# shellcheck disable=SC1091
|
||||
source "$SCRIPT_DIR/common.sh"
|
||||
|
||||
LOCK_FILE="${DEPLOY_CLEANUP_LOCK_FILE:-$RUN_DIR/deploy-disk-cleanup.lock}"
|
||||
LOG_DIR="${DEPLOY_CLEANUP_LOG_DIR:-$RUN_DIR/cleanup}"
|
||||
mkdir -p "$LOG_DIR"
|
||||
|
||||
if command -v flock >/dev/null 2>&1; then
|
||||
exec 9>"$LOCK_FILE"
|
||||
if ! flock -n 9; then
|
||||
echo "deploy disk cleanup is already running"
|
||||
exit 0
|
||||
fi
|
||||
else
|
||||
echo "flock not found, continuing without process lock" >&2
|
||||
fi
|
||||
|
||||
LOG_FILE="$LOG_DIR/deploy-disk-cleanup-$(date +%Y%m%d-%H%M%S).log"
|
||||
exec > >(tee -a "$LOG_FILE") 2>&1
|
||||
|
||||
KEEP_DAYS="${DEPLOY_CLEANUP_KEEP_DAYS:-3}"
|
||||
JOB_LOG_KEEP_DAYS="${DEPLOY_CLEANUP_JOB_LOG_KEEP_DAYS:-$KEEP_DAYS}"
|
||||
CLEANUP_LOG_KEEP_DAYS="${DEPLOY_CLEANUP_LOG_KEEP_DAYS:-14}"
|
||||
BUILD_ROOTS="${DEPLOY_CLEANUP_BUILD_ROOTS:-/opt/deploy/builds /opt/deploy/build-cache}"
|
||||
DRY_RUN="${DEPLOY_CLEANUP_DRY_RUN:-0}"
|
||||
|
||||
require_positive_int() {
|
||||
local name="$1"
|
||||
local value="$2"
|
||||
if [[ ! "$value" =~ ^[1-9][0-9]*$ ]]; then
|
||||
echo "$name must be a positive integer, got: $value" >&2
|
||||
exit 2
|
||||
fi
|
||||
}
|
||||
|
||||
require_positive_int DEPLOY_CLEANUP_KEEP_DAYS "$KEEP_DAYS"
|
||||
require_positive_int DEPLOY_CLEANUP_JOB_LOG_KEEP_DAYS "$JOB_LOG_KEEP_DAYS"
|
||||
require_positive_int DEPLOY_CLEANUP_LOG_KEEP_DAYS "$CLEANUP_LOG_KEEP_DAYS"
|
||||
|
||||
PRUNE_UNTIL="${DEPLOY_CLEANUP_DOCKER_PRUNE_UNTIL:-$((KEEP_DAYS * 24))h}"
|
||||
KEEP_MINUTES=$((KEEP_DAYS * 24 * 60))
|
||||
JOB_LOG_KEEP_MINUTES=$((JOB_LOG_KEEP_DAYS * 24 * 60))
|
||||
CLEANUP_LOG_KEEP_MINUTES=$((CLEANUP_LOG_KEEP_DAYS * 24 * 60))
|
||||
|
||||
log_section() {
|
||||
printf '\n--- %s ---\n' "$1"
|
||||
}
|
||||
|
||||
timestamp() {
|
||||
date '+%Y-%m-%dT%H:%M:%S%z'
|
||||
}
|
||||
|
||||
run_or_echo() {
|
||||
if [[ "$DRY_RUN" == "1" || "$DRY_RUN" == "true" ]]; then
|
||||
printf '[dry-run]'
|
||||
printf ' %q' "$@"
|
||||
printf '\n'
|
||||
return 0
|
||||
fi
|
||||
"$@"
|
||||
}
|
||||
|
||||
show_root_disk() {
|
||||
local output
|
||||
output="$(df -hT / 2>/dev/null || true)"
|
||||
if [ -n "$output" ]; then
|
||||
printf '%s\n' "$output"
|
||||
return 0
|
||||
fi
|
||||
df -h /
|
||||
}
|
||||
|
||||
cleanup_build_roots() {
|
||||
log_section "cleanup build directories older than ${KEEP_DAYS}d"
|
||||
local root
|
||||
for root in $BUILD_ROOTS; do
|
||||
if [ ! -d "$root" ]; then
|
||||
echo "skip missing build root: $root"
|
||||
continue
|
||||
fi
|
||||
|
||||
echo "build root: $root"
|
||||
if [[ "$DRY_RUN" == "1" || "$DRY_RUN" == "true" ]]; then
|
||||
find "$root" -mindepth 2 -maxdepth 2 -type d -mmin +"$KEEP_MINUTES" -print
|
||||
continue
|
||||
fi
|
||||
|
||||
find "$root" -mindepth 2 -maxdepth 2 -type d -mmin +"$KEEP_MINUTES" -print -exec rm -rf {} +
|
||||
find "$root" -mindepth 1 -maxdepth 1 -type d -empty -print -delete
|
||||
done
|
||||
}
|
||||
|
||||
cleanup_job_logs() {
|
||||
log_section "cleanup run job logs older than ${JOB_LOG_KEEP_DAYS}d"
|
||||
local dir
|
||||
for dir in "$RUN_DIR/service_update_jobs" "$RUN_DIR/chatapp_cron_deploy_jobs"; do
|
||||
if [ ! -d "$dir" ]; then
|
||||
echo "skip missing job dir: $dir"
|
||||
continue
|
||||
fi
|
||||
|
||||
echo "job dir: $dir"
|
||||
if [[ "$DRY_RUN" == "1" || "$DRY_RUN" == "true" ]]; then
|
||||
find "$dir" -type f \( -name '*.json' -o -name '*.log' \) -mmin +"$JOB_LOG_KEEP_MINUTES" -print
|
||||
continue
|
||||
fi
|
||||
|
||||
find "$dir" -type f \( -name '*.json' -o -name '*.log' \) -mmin +"$JOB_LOG_KEEP_MINUTES" -print -delete
|
||||
done
|
||||
}
|
||||
|
||||
cleanup_own_logs() {
|
||||
log_section "cleanup cleanup logs older than ${CLEANUP_LOG_KEEP_DAYS}d"
|
||||
if [[ "$DRY_RUN" == "1" || "$DRY_RUN" == "true" ]]; then
|
||||
find "$LOG_DIR" -type f -name 'deploy-disk-cleanup-*.log' -mmin +"$CLEANUP_LOG_KEEP_MINUTES" -print
|
||||
return 0
|
||||
fi
|
||||
find "$LOG_DIR" -type f -name 'deploy-disk-cleanup-*.log' -mmin +"$CLEANUP_LOG_KEEP_MINUTES" -print -delete
|
||||
}
|
||||
|
||||
cleanup_docker() {
|
||||
log_section "docker prune older than ${PRUNE_UNTIL}"
|
||||
if ! command -v docker >/dev/null 2>&1; then
|
||||
echo "docker not found, skip"
|
||||
return 0
|
||||
fi
|
||||
|
||||
docker system df || true
|
||||
run_or_echo docker builder prune -af --filter "until=$PRUNE_UNTIL"
|
||||
run_or_echo docker image prune -af --filter "until=$PRUNE_UNTIL"
|
||||
docker system df || true
|
||||
}
|
||||
|
||||
configure_harbor_retention_and_gc() {
|
||||
log_section "ensure harbor retention and gc schedule"
|
||||
|
||||
if [[ "${HARBOR_RETENTION_ENABLED:-1}" =~ ^(0|false|FALSE|no|NO|off|OFF)$ ]]; then
|
||||
echo "harbor retention disabled by HARBOR_RETENTION_ENABLED"
|
||||
return 0
|
||||
fi
|
||||
|
||||
if [ -z "${HARBOR_USERNAME:-}" ] || [ -z "${HARBOR_PASSWORD:-}" ]; then
|
||||
echo "HARBOR_USERNAME/HARBOR_PASSWORD missing, skip harbor retention/gc"
|
||||
return 0
|
||||
fi
|
||||
|
||||
export HARBOR_API_URL="${HARBOR_API_URL:-http://127.0.0.1:18082}"
|
||||
export HARBOR_RETENTION_KEEP_LATEST="${HARBOR_RETENTION_KEEP_LATEST:-20}"
|
||||
export HARBOR_RETENTION_CRON="${HARBOR_RETENTION_CRON:-0 0 2 * * *}"
|
||||
export HARBOR_GC_CRON="${HARBOR_GC_CRON:-0 0 4 * * *}"
|
||||
export DEPLOY_CLEANUP_DRY_RUN="$DRY_RUN"
|
||||
|
||||
"$PYTHON_BIN" - <<'PY'
|
||||
from __future__ import annotations
|
||||
|
||||
import base64
|
||||
import json
|
||||
import os
|
||||
import re
|
||||
import sys
|
||||
import urllib.error
|
||||
import urllib.parse
|
||||
import urllib.request
|
||||
|
||||
|
||||
def truthy(value: str) -> bool:
|
||||
return value.strip().lower() in {"1", "true", "yes", "on"}
|
||||
|
||||
|
||||
base_url = os.environ.get("HARBOR_API_URL", "http://127.0.0.1:18082").rstrip("/") + "/api/v2.0"
|
||||
project_name = os.environ.get("HARBOR_PROJECT", "likei").strip() or "likei"
|
||||
username = os.environ.get("HARBOR_USERNAME", "")
|
||||
password = os.environ.get("HARBOR_PASSWORD", "")
|
||||
keep_latest = int(os.environ.get("HARBOR_RETENTION_KEEP_LATEST", "20"))
|
||||
retention_cron = os.environ.get("HARBOR_RETENTION_CRON", "0 0 2 * * *").strip()
|
||||
gc_cron = os.environ.get("HARBOR_GC_CRON", "0 0 4 * * *").strip()
|
||||
dry_run = truthy(os.environ.get("DEPLOY_CLEANUP_DRY_RUN", "0"))
|
||||
|
||||
if keep_latest < 1:
|
||||
raise SystemExit("HARBOR_RETENTION_KEEP_LATEST must be positive")
|
||||
|
||||
if not re.fullmatch(r"\S+\s+\S+\s+\S+\s+\S+\s+\S+\s+\S+", retention_cron):
|
||||
raise SystemExit(f"invalid HARBOR_RETENTION_CRON: {retention_cron}")
|
||||
|
||||
if not re.fullmatch(r"\S+\s+\S+\s+\S+\s+\S+\s+\S+\s+\S+", gc_cron):
|
||||
raise SystemExit(f"invalid HARBOR_GC_CRON: {gc_cron}")
|
||||
|
||||
auth = base64.b64encode(f"{username}:{password}".encode("utf-8")).decode("ascii")
|
||||
headers = {
|
||||
"Authorization": f"Basic {auth}",
|
||||
"Accept": "application/json",
|
||||
"Content-Type": "application/json",
|
||||
}
|
||||
|
||||
|
||||
def request(method: str, path: str, payload: dict | None = None) -> tuple[int, object, dict[str, str]]:
|
||||
data = json.dumps(payload).encode("utf-8") if payload is not None else None
|
||||
req = urllib.request.Request(base_url + path, data=data, method=method, headers=headers)
|
||||
try:
|
||||
with urllib.request.urlopen(req, timeout=20) as resp:
|
||||
text = resp.read().decode("utf-8", errors="replace")
|
||||
body: object = None
|
||||
if text.strip():
|
||||
try:
|
||||
body = json.loads(text)
|
||||
except json.JSONDecodeError:
|
||||
body = text
|
||||
return resp.status, body, dict(resp.headers.items())
|
||||
except urllib.error.HTTPError as exc:
|
||||
text = exc.read().decode("utf-8", errors="replace")
|
||||
raise RuntimeError(f"harbor {method} {path} failed: {exc.code} {text[:500]}") from exc
|
||||
|
||||
|
||||
def project_path(path: str = "") -> str:
|
||||
encoded = urllib.parse.quote(project_name, safe="")
|
||||
return f"/projects/{encoded}{path}"
|
||||
|
||||
|
||||
def retention_policy(project_id: int, policy_id: int | None = None, rule_id: int | None = None) -> dict:
|
||||
rule = {
|
||||
"priority": 1,
|
||||
"disabled": False,
|
||||
"action": "retain",
|
||||
"template": "latestPushedK",
|
||||
"params": {"latestPushedK": keep_latest},
|
||||
"tag_selectors": [
|
||||
{"kind": "doublestar", "decoration": "matches", "pattern": "**"},
|
||||
],
|
||||
"scope_selectors": {
|
||||
"repository": [
|
||||
{"kind": "doublestar", "decoration": "repoMatches", "pattern": "**"},
|
||||
],
|
||||
},
|
||||
}
|
||||
if rule_id is not None:
|
||||
rule["id"] = rule_id
|
||||
|
||||
policy = {
|
||||
"algorithm": "or",
|
||||
"rules": [rule],
|
||||
"trigger": {"kind": "Schedule", "settings": {"cron": retention_cron}},
|
||||
"scope": {"level": "project", "ref": project_id},
|
||||
}
|
||||
if policy_id is not None:
|
||||
policy["id"] = policy_id
|
||||
return policy
|
||||
|
||||
|
||||
status, project, _ = request("GET", project_path())
|
||||
if not isinstance(project, dict):
|
||||
raise RuntimeError(f"unexpected harbor project response: {project!r}")
|
||||
|
||||
project_id = int(project["project_id"])
|
||||
metadata = dict(project.get("metadata") or {})
|
||||
retention_id_text = str(metadata.get("retention_id") or "").strip()
|
||||
retention_id = int(retention_id_text) if retention_id_text.isdigit() else None
|
||||
print(f"project={project_name} project_id={project_id} retention_id={retention_id or '-'}")
|
||||
|
||||
if retention_id is None:
|
||||
payload = retention_policy(project_id)
|
||||
if dry_run:
|
||||
print("[dry-run] create retention", json.dumps(payload, ensure_ascii=False))
|
||||
else:
|
||||
status, _, response_headers = request("POST", "/retentions", payload)
|
||||
location = response_headers.get("Location") or response_headers.get("location") or ""
|
||||
match = re.search(r"/retentions/(\d+)", location)
|
||||
if not match:
|
||||
status, refreshed_project, _ = request("GET", project_path())
|
||||
refreshed_metadata = dict((refreshed_project or {}).get("metadata") or {}) if isinstance(refreshed_project, dict) else {}
|
||||
refreshed_id = str(refreshed_metadata.get("retention_id") or "").strip()
|
||||
if not refreshed_id.isdigit():
|
||||
raise RuntimeError(f"retention created but id was not returned: status={status} location={location!r}")
|
||||
retention_id = int(refreshed_id)
|
||||
else:
|
||||
retention_id = int(match.group(1))
|
||||
|
||||
try:
|
||||
request("POST", project_path("/metadatas/"), {"retention_id": str(retention_id)})
|
||||
except RuntimeError as exc:
|
||||
if "409" in str(exc):
|
||||
request("PUT", project_path("/metadatas/retention_id"), {"retention_id": str(retention_id)})
|
||||
else:
|
||||
raise
|
||||
print(f"created retention_id={retention_id}")
|
||||
else:
|
||||
status, existing_policy, _ = request("GET", f"/retentions/{retention_id}")
|
||||
existing_rule_id = None
|
||||
if isinstance(existing_policy, dict):
|
||||
existing_rules = existing_policy.get("rules") or []
|
||||
if existing_rules and isinstance(existing_rules[0], dict) and existing_rules[0].get("id") is not None:
|
||||
existing_rule_id = int(existing_rules[0]["id"])
|
||||
payload = retention_policy(project_id, policy_id=retention_id, rule_id=existing_rule_id)
|
||||
if dry_run:
|
||||
print("[dry-run] update retention", json.dumps(payload, ensure_ascii=False))
|
||||
else:
|
||||
request("PUT", f"/retentions/{retention_id}", payload)
|
||||
print(f"updated retention_id={retention_id}")
|
||||
|
||||
gc_payload = {
|
||||
"schedule": {"type": "Schedule", "cron": gc_cron},
|
||||
"parameters": {},
|
||||
}
|
||||
status, current_gc, _ = request("GET", "/system/gc/schedule")
|
||||
gc_method = "POST" if not current_gc else "PUT"
|
||||
if dry_run:
|
||||
print(f"[dry-run] {gc_method} gc schedule", json.dumps(gc_payload, ensure_ascii=False))
|
||||
else:
|
||||
request(gc_method, "/system/gc/schedule", gc_payload)
|
||||
print(f"gc schedule ensured cron={gc_cron}")
|
||||
PY
|
||||
}
|
||||
|
||||
main() {
|
||||
echo "started_at=$(timestamp)"
|
||||
echo "log_file=$LOG_FILE"
|
||||
echo "dry_run=$DRY_RUN keep_days=$KEEP_DAYS prune_until=$PRUNE_UNTIL"
|
||||
|
||||
log_section "disk before"
|
||||
show_root_disk
|
||||
|
||||
cleanup_build_roots
|
||||
cleanup_job_logs
|
||||
cleanup_docker
|
||||
configure_harbor_retention_and_gc
|
||||
cleanup_own_logs
|
||||
|
||||
log_section "disk after"
|
||||
show_root_disk
|
||||
echo "finished_at=$(timestamp)"
|
||||
}
|
||||
|
||||
main "$@"
|
||||
@ -25,10 +25,27 @@ install_unit() {
|
||||
install -m 0644 "$src" "$dst"
|
||||
}
|
||||
|
||||
install_extra_unit() {
|
||||
local unit="$1"
|
||||
local src
|
||||
local dst
|
||||
src="$ROOT_DIR/systemd/$unit"
|
||||
dst="/etc/systemd/system/$unit"
|
||||
|
||||
if [ ! -f "$src" ]; then
|
||||
echo "systemd unit missing: $src" >&2
|
||||
exit 1
|
||||
fi
|
||||
|
||||
install -m 0644 "$src" "$dst"
|
||||
}
|
||||
|
||||
name=""
|
||||
for name in $(suite_service_names); do
|
||||
install_unit "$name"
|
||||
done
|
||||
install_extra_unit "hy-app-monitor-cleanup.service"
|
||||
install_extra_unit "hy-app-monitor-cleanup.timer"
|
||||
|
||||
chmod +x "$ROOT_DIR"/scripts/*.sh
|
||||
stop_all_legacy_processes
|
||||
@ -37,3 +54,4 @@ systemctl daemon-reload
|
||||
for name in $(suite_service_names); do
|
||||
systemctl enable "$(service_unit_name "$name")"
|
||||
done
|
||||
systemctl enable --now hy-app-monitor-cleanup.timer
|
||||
|
||||
@ -6,6 +6,7 @@ SCRIPT_DIR="$(CDPATH= cd -- "$(dirname -- "$0")" && pwd)"
|
||||
source "$SCRIPT_DIR/common.sh"
|
||||
|
||||
if command -v systemctl >/dev/null 2>&1; then
|
||||
systemctl disable --now hy-app-monitor-cleanup.timer >/dev/null 2>&1 || true
|
||||
name=""
|
||||
for name in $(suite_service_names); do
|
||||
systemctl disable --now "$(service_unit_name "$name")" >/dev/null 2>&1 || true
|
||||
@ -16,6 +17,8 @@ name=""
|
||||
for name in $(suite_service_names); do
|
||||
rm -f "/etc/systemd/system/$(service_unit_name "$name")"
|
||||
done
|
||||
rm -f /etc/systemd/system/hy-app-monitor-cleanup.service
|
||||
rm -f /etc/systemd/system/hy-app-monitor-cleanup.timer
|
||||
|
||||
if command -v systemctl >/dev/null 2>&1; then
|
||||
systemctl daemon-reload || true
|
||||
|
||||
11
systemd/hy-app-monitor-cleanup.service
Normal file
11
systemd/hy-app-monitor-cleanup.service
Normal file
@ -0,0 +1,11 @@
|
||||
[Unit]
|
||||
Description=HY App Monitor Deploy Disk Cleanup
|
||||
After=network-online.target docker.service
|
||||
Wants=network-online.target
|
||||
|
||||
[Service]
|
||||
Type=oneshot
|
||||
User=root
|
||||
WorkingDirectory=/opt/hy-app-monitor
|
||||
EnvironmentFile=-/opt/hy-app-monitor/.env
|
||||
ExecStart=/opt/hy-app-monitor/scripts/cleanup_deploy_disk.sh
|
||||
11
systemd/hy-app-monitor-cleanup.timer
Normal file
11
systemd/hy-app-monitor-cleanup.timer
Normal file
@ -0,0 +1,11 @@
|
||||
[Unit]
|
||||
Description=Run HY App Monitor deploy disk cleanup daily
|
||||
|
||||
[Timer]
|
||||
OnCalendar=*-*-* 03:30:00
|
||||
RandomizedDelaySec=10m
|
||||
Persistent=true
|
||||
Unit=hy-app-monitor-cleanup.service
|
||||
|
||||
[Install]
|
||||
WantedBy=timers.target
|
||||
Loading…
x
Reference in New Issue
Block a user