a2ff4768d3
把部署机上未提交的本地热补丁正式入库：服务更新预拉镜像后，顺手清理目标 宿主机上超过 REMOTE_IMAGE_PRUNE_KEEP_DAYS 天且未被容器引用的历史镜像； cleanup_deploy_disk.sh 同步增加 cleanup_remote_host_images。 发布流程测试补上 prune_remote_host_images 的 mock，避免单测触发真实 SSH。 Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>
439 lines
14 KiB
Bash
Executable File
439 lines
14 KiB
Bash
Executable File
#!/usr/bin/env bash
|
|
set -euo pipefail
|
|
|
|
SCRIPT_DIR="$(CDPATH= cd -- "$(dirname -- "$0")" && pwd)"
|
|
# shellcheck disable=SC1091
|
|
source "$SCRIPT_DIR/common.sh"
|
|
|
|
LOCK_FILE="${DEPLOY_CLEANUP_LOCK_FILE:-$RUN_DIR/deploy-disk-cleanup.lock}"
|
|
LOG_DIR="${DEPLOY_CLEANUP_LOG_DIR:-$RUN_DIR/cleanup}"
|
|
mkdir -p "$LOG_DIR"
|
|
|
|
if command -v flock >/dev/null 2>&1; then
|
|
exec 9>"$LOCK_FILE"
|
|
if ! flock -n 9; then
|
|
echo "deploy disk cleanup is already running"
|
|
exit 0
|
|
fi
|
|
else
|
|
echo "flock not found, continuing without process lock" >&2
|
|
fi
|
|
|
|
LOG_FILE="$LOG_DIR/deploy-disk-cleanup-$(date +%Y%m%d-%H%M%S).log"
|
|
exec > >(tee -a "$LOG_FILE") 2>&1
|
|
|
|
KEEP_DAYS="${DEPLOY_CLEANUP_KEEP_DAYS:-3}"
|
|
JOB_LOG_KEEP_DAYS="${DEPLOY_CLEANUP_JOB_LOG_KEEP_DAYS:-$KEEP_DAYS}"
|
|
CLEANUP_LOG_KEEP_DAYS="${DEPLOY_CLEANUP_LOG_KEEP_DAYS:-14}"
|
|
BUILD_ROOTS="${DEPLOY_CLEANUP_BUILD_ROOTS:-/opt/deploy/builds /opt/deploy/build-cache}"
|
|
DRY_RUN="${DEPLOY_CLEANUP_DRY_RUN:-0}"
|
|
|
|
require_positive_int() {
|
|
local name="$1"
|
|
local value="$2"
|
|
if [[ ! "$value" =~ ^[1-9][0-9]*$ ]]; then
|
|
echo "$name must be a positive integer, got: $value" >&2
|
|
exit 2
|
|
fi
|
|
}
|
|
|
|
require_positive_int DEPLOY_CLEANUP_KEEP_DAYS "$KEEP_DAYS"
|
|
require_positive_int DEPLOY_CLEANUP_JOB_LOG_KEEP_DAYS "$JOB_LOG_KEEP_DAYS"
|
|
require_positive_int DEPLOY_CLEANUP_LOG_KEEP_DAYS "$CLEANUP_LOG_KEEP_DAYS"
|
|
|
|
PRUNE_UNTIL="${DEPLOY_CLEANUP_DOCKER_PRUNE_UNTIL:-$((KEEP_DAYS * 24))h}"
|
|
KEEP_MINUTES=$((KEEP_DAYS * 24 * 60))
|
|
JOB_LOG_KEEP_MINUTES=$((JOB_LOG_KEEP_DAYS * 24 * 60))
|
|
CLEANUP_LOG_KEEP_MINUTES=$((CLEANUP_LOG_KEEP_DAYS * 24 * 60))
|
|
|
|
log_section() {
|
|
printf '\n--- %s ---\n' "$1"
|
|
}
|
|
|
|
timestamp() {
|
|
date '+%Y-%m-%dT%H:%M:%S%z'
|
|
}
|
|
|
|
run_or_echo() {
|
|
if [[ "$DRY_RUN" == "1" || "$DRY_RUN" == "true" ]]; then
|
|
printf '[dry-run]'
|
|
printf ' %q' "$@"
|
|
printf '\n'
|
|
return 0
|
|
fi
|
|
"$@"
|
|
}
|
|
|
|
show_root_disk() {
|
|
local output
|
|
output="$(df -hT / 2>/dev/null || true)"
|
|
if [ -n "$output" ]; then
|
|
printf '%s\n' "$output"
|
|
return 0
|
|
fi
|
|
df -h /
|
|
}
|
|
|
|
cleanup_build_roots() {
|
|
log_section "cleanup build directories older than ${KEEP_DAYS}d"
|
|
local root
|
|
for root in $BUILD_ROOTS; do
|
|
if [ ! -d "$root" ]; then
|
|
echo "skip missing build root: $root"
|
|
continue
|
|
fi
|
|
|
|
echo "build root: $root"
|
|
if [[ "$DRY_RUN" == "1" || "$DRY_RUN" == "true" ]]; then
|
|
find "$root" -mindepth 2 -maxdepth 2 -type d -mmin +"$KEEP_MINUTES" -print
|
|
continue
|
|
fi
|
|
|
|
find "$root" -mindepth 2 -maxdepth 2 -type d -mmin +"$KEEP_MINUTES" -print -exec rm -rf {} +
|
|
find "$root" -mindepth 1 -maxdepth 1 -type d -empty -print -delete
|
|
done
|
|
}
|
|
|
|
cleanup_job_logs() {
|
|
log_section "cleanup run job logs older than ${JOB_LOG_KEEP_DAYS}d"
|
|
local dir
|
|
for dir in "$RUN_DIR/service_update_jobs" "$RUN_DIR/chatapp_cron_deploy_jobs"; do
|
|
if [ ! -d "$dir" ]; then
|
|
echo "skip missing job dir: $dir"
|
|
continue
|
|
fi
|
|
|
|
echo "job dir: $dir"
|
|
if [[ "$DRY_RUN" == "1" || "$DRY_RUN" == "true" ]]; then
|
|
find "$dir" -type f \( -name '*.json' -o -name '*.log' \) -mmin +"$JOB_LOG_KEEP_MINUTES" -print
|
|
continue
|
|
fi
|
|
|
|
find "$dir" -type f \( -name '*.json' -o -name '*.log' \) -mmin +"$JOB_LOG_KEEP_MINUTES" -print -delete
|
|
done
|
|
}
|
|
|
|
cleanup_own_logs() {
|
|
log_section "cleanup cleanup logs older than ${CLEANUP_LOG_KEEP_DAYS}d"
|
|
if [[ "$DRY_RUN" == "1" || "$DRY_RUN" == "true" ]]; then
|
|
find "$LOG_DIR" -type f -name 'deploy-disk-cleanup-*.log' -mmin +"$CLEANUP_LOG_KEEP_MINUTES" -print
|
|
return 0
|
|
fi
|
|
find "$LOG_DIR" -type f -name 'deploy-disk-cleanup-*.log' -mmin +"$CLEANUP_LOG_KEEP_MINUTES" -print -delete
|
|
}
|
|
|
|
cleanup_docker() {
|
|
log_section "docker prune older than ${PRUNE_UNTIL}"
|
|
if ! command -v docker >/dev/null 2>&1; then
|
|
echo "docker not found, skip"
|
|
return 0
|
|
fi
|
|
|
|
docker system df || true
|
|
run_or_echo docker builder prune -af --filter "until=$PRUNE_UNTIL"
|
|
run_or_echo docker image prune -af --filter "until=$PRUNE_UNTIL"
|
|
docker system df || true
|
|
}
|
|
|
|
cleanup_remote_host_images() {
|
|
log_section "remote host image prune"
|
|
"$PYTHON_BIN" - <<'PY'
|
|
from __future__ import annotations
|
|
|
|
import os
|
|
|
|
from core.config import (
|
|
REMOTE_IMAGE_PRUNE_ENABLED,
|
|
REMOTE_IMAGE_PRUNE_KEEP_DAYS,
|
|
REMOTE_IMAGE_PRUNE_TIMEOUT_SECONDS,
|
|
load_hosts,
|
|
)
|
|
from monitors.yumi.tencent_tat import run_shell_command
|
|
|
|
|
|
def truthy(value: str) -> bool:
|
|
# shell 环境变量统一用这个函数识别真假，兼容现有 cleanup dry-run 写法。
|
|
return value.strip().lower() in {"1", "true", "yes", "on"}
|
|
|
|
|
|
dry_run = truthy(os.environ.get("DEPLOY_CLEANUP_DRY_RUN", "0"))
|
|
if not REMOTE_IMAGE_PRUNE_ENABLED:
|
|
print("REMOTE_IMAGE_PRUNE_ENABLED disabled, skip")
|
|
raise SystemExit(0)
|
|
|
|
keep_days = int(REMOTE_IMAGE_PRUNE_KEEP_DAYS)
|
|
if keep_days <= 0:
|
|
raise SystemExit(f"REMOTE_IMAGE_PRUNE_KEEP_DAYS must be positive, got: {keep_days}")
|
|
|
|
prune_until = f"{keep_days * 24}h"
|
|
hosts: list[dict[str, str]] = []
|
|
seen_instances: set[str] = set()
|
|
for host in load_hosts():
|
|
# 只处理真正承载业务服务的宿主机；没有 instanceId 的记录无法通过 TAT 触达，直接跳过。
|
|
if not host.get("services"):
|
|
continue
|
|
instance_id = str(host.get("instanceId") or "").strip()
|
|
if not instance_id or instance_id in seen_instances:
|
|
continue
|
|
seen_instances.add(instance_id)
|
|
hosts.append(
|
|
{
|
|
"host": str(host.get("host") or "").strip(),
|
|
"ip": str(host.get("ip") or "").strip(),
|
|
"instanceId": instance_id,
|
|
}
|
|
)
|
|
|
|
if not hosts:
|
|
print("no runtime hosts with instanceId, skip")
|
|
raise SystemExit(0)
|
|
|
|
print(f"target_hosts={len(hosts)} keep_days={keep_days} prune_until={prune_until}")
|
|
for host in hosts:
|
|
print(f"target={host['host']} {host['ip']} {host['instanceId']}")
|
|
|
|
if dry_run:
|
|
print(f"[dry-run] docker image prune -af --filter until={prune_until}")
|
|
raise SystemExit(0)
|
|
|
|
script = f'''set -eu
|
|
printf 'hostname='; hostname
|
|
printf 'time='; date -Is
|
|
printf 'root_df_before='; df -h / | awk 'NR==2{{print $2","$3","$4","$5}}'
|
|
echo '--- docker system df before ---'
|
|
docker system df || true
|
|
out_file=$(mktemp)
|
|
if docker image prune -af --filter "until={prune_until}" >"$out_file" 2>&1; then
|
|
status=0
|
|
else
|
|
status=$?
|
|
fi
|
|
printf 'prune_exit=%s\\n' "$status"
|
|
printf 'untagged_refs='; grep -c '^Untagged:' "$out_file" || true
|
|
printf 'deleted_layers='; grep -c '^Deleted:' "$out_file" || true
|
|
grep 'Total reclaimed space:' "$out_file" || true
|
|
if [ "$status" -ne 0 ]; then
|
|
echo '--- prune error output ---'
|
|
cat "$out_file"
|
|
exit "$status"
|
|
fi
|
|
rm -f "$out_file"
|
|
echo '--- docker system df after ---'
|
|
docker system df || true
|
|
printf 'root_df_after='; df -h / | awk 'NR==2{{print $2","$3","$4","$5}}'
|
|
'''
|
|
|
|
results = run_shell_command(
|
|
[host["instanceId"] for host in hosts],
|
|
script,
|
|
command_name="hy-remote-image-prune",
|
|
timeout_seconds=max(int(REMOTE_IMAGE_PRUNE_TIMEOUT_SECONDS), 120),
|
|
)
|
|
|
|
for host in hosts:
|
|
result = results.get(host["instanceId"], {})
|
|
status = str(result.get("status") or "UNKNOWN")
|
|
output = str(result.get("output") or "").strip()
|
|
print(f"--- {host['host']} {host['ip']} {host['instanceId']} {status} ---")
|
|
print(output[-3000:] if len(output) > 3000 else output)
|
|
if status != "SUCCESS":
|
|
raise SystemExit(f"remote image prune failed on {host['host']} {host['instanceId']}: {status}")
|
|
PY
|
|
}
|
|
|
|
configure_harbor_retention_and_gc() {
|
|
log_section "ensure harbor retention and gc schedule"
|
|
|
|
if [[ "${HARBOR_RETENTION_ENABLED:-1}" =~ ^(0|false|FALSE|no|NO|off|OFF)$ ]]; then
|
|
echo "harbor retention disabled by HARBOR_RETENTION_ENABLED"
|
|
return 0
|
|
fi
|
|
|
|
if [ -z "${HARBOR_USERNAME:-}" ] || [ -z "${HARBOR_PASSWORD:-}" ]; then
|
|
echo "HARBOR_USERNAME/HARBOR_PASSWORD missing, skip harbor retention/gc"
|
|
return 0
|
|
fi
|
|
|
|
export HARBOR_API_URL="${HARBOR_API_URL:-http://127.0.0.1:18082}"
|
|
export HARBOR_RETENTION_KEEP_LATEST="${HARBOR_RETENTION_KEEP_LATEST:-20}"
|
|
export HARBOR_RETENTION_CRON="${HARBOR_RETENTION_CRON:-0 0 2 * * *}"
|
|
export HARBOR_GC_CRON="${HARBOR_GC_CRON:-0 0 4 * * *}"
|
|
export DEPLOY_CLEANUP_DRY_RUN="$DRY_RUN"
|
|
|
|
"$PYTHON_BIN" - <<'PY'
|
|
from __future__ import annotations
|
|
|
|
import base64
|
|
import json
|
|
import os
|
|
import re
|
|
import sys
|
|
import urllib.error
|
|
import urllib.parse
|
|
import urllib.request
|
|
|
|
|
|
def truthy(value: str) -> bool:
|
|
return value.strip().lower() in {"1", "true", "yes", "on"}
|
|
|
|
|
|
base_url = os.environ.get("HARBOR_API_URL", "http://127.0.0.1:18082").rstrip("/") + "/api/v2.0"
|
|
project_name = os.environ.get("HARBOR_PROJECT", "likei").strip() or "likei"
|
|
username = os.environ.get("HARBOR_USERNAME", "")
|
|
password = os.environ.get("HARBOR_PASSWORD", "")
|
|
keep_latest = int(os.environ.get("HARBOR_RETENTION_KEEP_LATEST", "20"))
|
|
retention_cron = os.environ.get("HARBOR_RETENTION_CRON", "0 0 2 * * *").strip()
|
|
gc_cron = os.environ.get("HARBOR_GC_CRON", "0 0 4 * * *").strip()
|
|
dry_run = truthy(os.environ.get("DEPLOY_CLEANUP_DRY_RUN", "0"))
|
|
|
|
if keep_latest < 1:
|
|
raise SystemExit("HARBOR_RETENTION_KEEP_LATEST must be positive")
|
|
|
|
if not re.fullmatch(r"\S+\s+\S+\s+\S+\s+\S+\s+\S+\s+\S+", retention_cron):
|
|
raise SystemExit(f"invalid HARBOR_RETENTION_CRON: {retention_cron}")
|
|
|
|
if not re.fullmatch(r"\S+\s+\S+\s+\S+\s+\S+\s+\S+\s+\S+", gc_cron):
|
|
raise SystemExit(f"invalid HARBOR_GC_CRON: {gc_cron}")
|
|
|
|
auth = base64.b64encode(f"{username}:{password}".encode("utf-8")).decode("ascii")
|
|
headers = {
|
|
"Authorization": f"Basic {auth}",
|
|
"Accept": "application/json",
|
|
"Content-Type": "application/json",
|
|
}
|
|
|
|
|
|
def request(method: str, path: str, payload: dict | None = None) -> tuple[int, object, dict[str, str]]:
|
|
data = json.dumps(payload).encode("utf-8") if payload is not None else None
|
|
req = urllib.request.Request(base_url + path, data=data, method=method, headers=headers)
|
|
try:
|
|
with urllib.request.urlopen(req, timeout=20) as resp:
|
|
text = resp.read().decode("utf-8", errors="replace")
|
|
body: object = None
|
|
if text.strip():
|
|
try:
|
|
body = json.loads(text)
|
|
except json.JSONDecodeError:
|
|
body = text
|
|
return resp.status, body, dict(resp.headers.items())
|
|
except urllib.error.HTTPError as exc:
|
|
text = exc.read().decode("utf-8", errors="replace")
|
|
raise RuntimeError(f"harbor {method} {path} failed: {exc.code} {text[:500]}") from exc
|
|
|
|
|
|
def project_path(path: str = "") -> str:
|
|
encoded = urllib.parse.quote(project_name, safe="")
|
|
return f"/projects/{encoded}{path}"
|
|
|
|
|
|
def retention_policy(project_id: int, policy_id: int | None = None, rule_id: int | None = None) -> dict:
|
|
rule = {
|
|
"priority": 1,
|
|
"disabled": False,
|
|
"action": "retain",
|
|
"template": "latestPushedK",
|
|
"params": {"latestPushedK": keep_latest},
|
|
"tag_selectors": [
|
|
{"kind": "doublestar", "decoration": "matches", "pattern": "**"},
|
|
],
|
|
"scope_selectors": {
|
|
"repository": [
|
|
{"kind": "doublestar", "decoration": "repoMatches", "pattern": "**"},
|
|
],
|
|
},
|
|
}
|
|
if rule_id is not None:
|
|
rule["id"] = rule_id
|
|
|
|
policy = {
|
|
"algorithm": "or",
|
|
"rules": [rule],
|
|
"trigger": {"kind": "Schedule", "settings": {"cron": retention_cron}},
|
|
"scope": {"level": "project", "ref": project_id},
|
|
}
|
|
if policy_id is not None:
|
|
policy["id"] = policy_id
|
|
return policy
|
|
|
|
|
|
status, project, _ = request("GET", project_path())
|
|
if not isinstance(project, dict):
|
|
raise RuntimeError(f"unexpected harbor project response: {project!r}")
|
|
|
|
project_id = int(project["project_id"])
|
|
metadata = dict(project.get("metadata") or {})
|
|
retention_id_text = str(metadata.get("retention_id") or "").strip()
|
|
retention_id = int(retention_id_text) if retention_id_text.isdigit() else None
|
|
print(f"project={project_name} project_id={project_id} retention_id={retention_id or '-'}")
|
|
|
|
if retention_id is None:
|
|
payload = retention_policy(project_id)
|
|
if dry_run:
|
|
print("[dry-run] create retention", json.dumps(payload, ensure_ascii=False))
|
|
else:
|
|
status, _, response_headers = request("POST", "/retentions", payload)
|
|
location = response_headers.get("Location") or response_headers.get("location") or ""
|
|
match = re.search(r"/retentions/(\d+)", location)
|
|
if match:
|
|
retention_id = int(match.group(1))
|
|
|
|
status, refreshed_project, _ = request("GET", project_path())
|
|
refreshed_metadata = dict((refreshed_project or {}).get("metadata") or {}) if isinstance(refreshed_project, dict) else {}
|
|
refreshed_id = str(refreshed_metadata.get("retention_id") or "").strip()
|
|
if refreshed_id.isdigit():
|
|
retention_id = int(refreshed_id)
|
|
elif retention_id is None:
|
|
raise RuntimeError(f"retention created but id was not returned: status={status} location={location!r}")
|
|
|
|
print(f"created retention_id={retention_id}")
|
|
else:
|
|
status, existing_policy, _ = request("GET", f"/retentions/{retention_id}")
|
|
existing_rule_id = None
|
|
if isinstance(existing_policy, dict):
|
|
existing_rules = existing_policy.get("rules") or []
|
|
if existing_rules and isinstance(existing_rules[0], dict) and existing_rules[0].get("id") is not None:
|
|
existing_rule_id = int(existing_rules[0]["id"])
|
|
payload = retention_policy(project_id, policy_id=retention_id, rule_id=existing_rule_id)
|
|
if dry_run:
|
|
print("[dry-run] update retention", json.dumps(payload, ensure_ascii=False))
|
|
else:
|
|
request("PUT", f"/retentions/{retention_id}", payload)
|
|
print(f"updated retention_id={retention_id}")
|
|
|
|
gc_payload = {
|
|
"schedule": {"type": "Custom", "cron": gc_cron},
|
|
"parameters": {},
|
|
}
|
|
status, current_gc, _ = request("GET", "/system/gc/schedule")
|
|
gc_method = "POST" if not current_gc else "PUT"
|
|
if dry_run:
|
|
print(f"[dry-run] {gc_method} gc schedule", json.dumps(gc_payload, ensure_ascii=False))
|
|
else:
|
|
request(gc_method, "/system/gc/schedule", gc_payload)
|
|
print(f"gc schedule ensured cron={gc_cron}")
|
|
PY
|
|
}
|
|
|
|
main() {
|
|
echo "started_at=$(timestamp)"
|
|
echo "log_file=$LOG_FILE"
|
|
echo "dry_run=$DRY_RUN keep_days=$KEEP_DAYS prune_until=$PRUNE_UNTIL"
|
|
|
|
log_section "disk before"
|
|
show_root_disk
|
|
|
|
cleanup_build_roots
|
|
cleanup_job_logs
|
|
cleanup_docker
|
|
cleanup_remote_host_images
|
|
configure_harbor_retention_and_gc
|
|
cleanup_own_logs
|
|
|
|
log_section "disk after"
|
|
show_root_disk
|
|
echo "finished_at=$(timestamp)"
|
|
}
|
|
|
|
main "$@"
|
|
|