diff --git a/.env.example b/.env.example index 23532b5..93c2b59 100644 --- a/.env.example +++ b/.env.example @@ -134,6 +134,10 @@ GATEWAY_CLB_DRAIN_SECONDS=0 # 恢复健康后再还原权重；不配置则跳过内网 CLB 摘流（历史行为）。 INTERNAL_CLB_IDS= INTERNAL_CLB_DRAIN_SECONDS=10 +# 服务更新时顺手清理目标宿主机上超过 N 天且未被引用的历史镜像。 +REMOTE_IMAGE_PRUNE_ENABLED=1 +REMOTE_IMAGE_PRUNE_KEEP_DAYS=7 +REMOTE_IMAGE_PRUNE_TIMEOUT_SECONDS=300 DEPLOY_SSH_HOST= DEPLOY_SSH_USER=root DEPLOY_SSH_PORT=22 diff --git a/core/config.py b/core/config.py index 4e678d2..b91459a 100644 --- a/core/config.py +++ b/core/config.py @@ -107,6 +107,12 @@ SERVICE_LOG_EXPORT_MAX_LINES = env_int("SERVICE_LOG_EXPORT_MAX_LINES", 10000) SERVICE_BUILD_TIMEOUT_SECONDS = env_int("SERVICE_BUILD_TIMEOUT_SECONDS", 3600) # 服务更新时目标机预拉镜像最长等待秒数。 SERVICE_PULL_TIMEOUT_SECONDS = env_int("SERVICE_PULL_TIMEOUT_SECONDS", 600) +# 服务更新时是否顺手清理目标宿主机历史镜像，默认开启以避免业务机镜像持续堆积。 +REMOTE_IMAGE_PRUNE_ENABLED = env_str("REMOTE_IMAGE_PRUNE_ENABLED", "1").lower() in {"1", "true", "yes", "on"} +# 目标宿主机只清理超过该天数且没有被任何容器引用的镜像。 +REMOTE_IMAGE_PRUNE_KEEP_DAYS = env_int("REMOTE_IMAGE_PRUNE_KEEP_DAYS", 7) +# 目标宿主机镜像清理的最长等待秒数；镜像很多时允许比普通预拉多一点时间。 +REMOTE_IMAGE_PRUNE_TIMEOUT_SECONDS = env_int("REMOTE_IMAGE_PRUNE_TIMEOUT_SECONDS", 300) # 固定构建工作区根目录，留空时按本地 / deploy 机路径自动推导。 SERVICE_BUILD_CACHE_ROOT = env_str("SERVICE_BUILD_CACHE_ROOT", "") # Java Maven 并行线程，默认按 1 个 CPU 组装 reactor。 diff --git a/monitors/yumi/service_release.py b/monitors/yumi/service_release.py index 4c3a2dd..f344089 100644 --- a/monitors/yumi/service_release.py +++ b/monitors/yumi/service_release.py @@ -60,6 +60,9 @@ from core.config import ( ROOT, ROLLING_RESTART_STABILIZE_SECONDS, ROLLING_RESTART_TIMEOUT_SECONDS, + REMOTE_IMAGE_PRUNE_ENABLED, + REMOTE_IMAGE_PRUNE_KEEP_DAYS, + REMOTE_IMAGE_PRUNE_TIMEOUT_SECONDS, RUNTIME_BASE_DIR, SERVICE_BUILD_CACHE_ROOT, SERVICE_BUILD_TIMEOUT_SECONDS, @@ -1359,6 +1362,52 @@ def prepull_host_images(host_entry: dict[str, Any]) -> dict[str, Any]: } +def remote_image_prune_until() -> str: + # Docker 的 until 过滤按小时表达最直观；配置异常时直接失败，避免误删最近镜像。 + keep_days = int(REMOTE_IMAGE_PRUNE_KEEP_DAYS) + if keep_days <= 0: + raise RuntimeError(f"REMOTE_IMAGE_PRUNE_KEEP_DAYS must be positive, got {keep_days}") + return f"{keep_days * 24}h" + + +def prune_remote_host_images(host_entry: dict[str, Any]) -> dict[str, Any]: + # 只清理“未被任何容器引用”的历史镜像，不碰容器、卷和 builder cache，保证运行中服务和回滚窗口不被破坏。 + if not REMOTE_IMAGE_PRUNE_ENABLED: + return { + "status": "SKIPPED", + "output": "REMOTE_IMAGE_PRUNE_ENABLED is disabled", + "keepDays": int(REMOTE_IMAGE_PRUNE_KEEP_DAYS), + "pruneUntil": "", + } + + prune_until = remote_image_prune_until() + script = f"""set -eu +if ! command -v docker >/dev/null 2>&1; then + echo "docker not found, skip image prune" + exit 0 +fi +echo "prune_until={shlex.quote(prune_until)}" +echo "--- docker system df before ---" +docker system df || true +echo "--- docker image prune ---" +docker image prune -af --filter "until={shlex.quote(prune_until)}" +echo "--- docker system df after ---" +docker system df || true +""" + result = remote_host_result( + host_entry, + script, + f"image-prune-{host_entry['host']}", + timeout_seconds=max(REMOTE_IMAGE_PRUNE_TIMEOUT_SECONDS, 120), + ) + return { + "status": result.get("status"), + "output": str(result.get("output") or "").strip()[-2000:], + "keepDays": int(REMOTE_IMAGE_PRUNE_KEEP_DAYS), + "pruneUntil": prune_until, + } + + def dashboard_cdc_env_path(host_name: str) -> str: return f"{RUNTIME_BASE_DIR.rstrip('/')}/{host_name}/dashboard-cdc-worker.env" @@ -1839,6 +1888,17 @@ def deploy_runtime_services_payload( f"image.prepull.{host_entry['host']}", f"预拉完成：{prepull_result['status']} {short_text(prepull_result['output'])}", ) + emit_progress( + progress, + f"image.prune.{host_entry['host']}", + f"清理 {REMOTE_IMAGE_PRUNE_KEEP_DAYS} 天前未使用历史镜像", + ) + image_prune_result = prune_remote_host_images(host_entry) + emit_progress( + progress, + f"image.prune.{host_entry['host']}", + f"清理完成：{image_prune_result['status']} {short_text(image_prune_result['output'])}", + ) prepared_hosts.append( { "host": host_entry["host"], @@ -1855,6 +1915,12 @@ def deploy_runtime_services_payload( "status": prepull_result["status"], "output": prepull_result["output"], }, + "imagePrune": { + "status": image_prune_result["status"], + "output": image_prune_result["output"], + "keepDays": image_prune_result["keepDays"], + "pruneUntil": image_prune_result["pruneUntil"], + }, } ) diff --git a/scripts/cleanup_deploy_disk.sh b/scripts/cleanup_deploy_disk.sh index 9218173..fbdd563 100755 --- a/scripts/cleanup_deploy_disk.sh +++ b/scripts/cleanup_deploy_disk.sh @@ -135,6 +135,112 @@ cleanup_docker() { docker system df || true } +cleanup_remote_host_images() { + log_section "remote host image prune" + "$PYTHON_BIN" - <<'PY' +from __future__ import annotations + +import os + +from core.config import ( + REMOTE_IMAGE_PRUNE_ENABLED, + REMOTE_IMAGE_PRUNE_KEEP_DAYS, + REMOTE_IMAGE_PRUNE_TIMEOUT_SECONDS, + load_hosts, +) +from monitors.yumi.tencent_tat import run_shell_command + + +def truthy(value: str) -> bool: + # shell 环境变量统一用这个函数识别真假，兼容现有 cleanup dry-run 写法。 + return value.strip().lower() in {"1", "true", "yes", "on"} + + +dry_run = truthy(os.environ.get("DEPLOY_CLEANUP_DRY_RUN", "0")) +if not REMOTE_IMAGE_PRUNE_ENABLED: + print("REMOTE_IMAGE_PRUNE_ENABLED disabled, skip") + raise SystemExit(0) + +keep_days = int(REMOTE_IMAGE_PRUNE_KEEP_DAYS) +if keep_days <= 0: + raise SystemExit(f"REMOTE_IMAGE_PRUNE_KEEP_DAYS must be positive, got: {keep_days}") + +prune_until = f"{keep_days * 24}h" +hosts: list[dict[str, str]] = [] +seen_instances: set[str] = set() +for host in load_hosts(): + # 只处理真正承载业务服务的宿主机；没有 instanceId 的记录无法通过 TAT 触达，直接跳过。 + if not host.get("services"): + continue + instance_id = str(host.get("instanceId") or "").strip() + if not instance_id or instance_id in seen_instances: + continue + seen_instances.add(instance_id) + hosts.append( + { + "host": str(host.get("host") or "").strip(), + "ip": str(host.get("ip") or "").strip(), + "instanceId": instance_id, + } + ) + +if not hosts: + print("no runtime hosts with instanceId, skip") + raise SystemExit(0) + +print(f"target_hosts={len(hosts)} keep_days={keep_days} prune_until={prune_until}") +for host in hosts: + print(f"target={host['host']} {host['ip']} {host['instanceId']}") + +if dry_run: + print(f"[dry-run] docker image prune -af --filter until={prune_until}") + raise SystemExit(0) + +script = f'''set -eu +printf 'hostname='; hostname +printf 'time='; date -Is +printf 'root_df_before='; df -h / | awk 'NR==2{{print $2","$3","$4","$5}}' +echo '--- docker system df before ---' +docker system df || true +out_file=$(mktemp) +if docker image prune -af --filter "until={prune_until}" >"$out_file" 2>&1; then + status=0 +else + status=$? +fi +printf 'prune_exit=%s\\n' "$status" +printf 'untagged_refs='; grep -c '^Untagged:' "$out_file" || true +printf 'deleted_layers='; grep -c '^Deleted:' "$out_file" || true +grep 'Total reclaimed space:' "$out_file" || true +if [ "$status" -ne 0 ]; then + echo '--- prune error output ---' + cat "$out_file" + exit "$status" +fi +rm -f "$out_file" +echo '--- docker system df after ---' +docker system df || true +printf 'root_df_after='; df -h / | awk 'NR==2{{print $2","$3","$4","$5}}' +''' + +results = run_shell_command( + [host["instanceId"] for host in hosts], + script, + command_name="hy-remote-image-prune", + timeout_seconds=max(int(REMOTE_IMAGE_PRUNE_TIMEOUT_SECONDS), 120), +) + +for host in hosts: + result = results.get(host["instanceId"], {}) + status = str(result.get("status") or "UNKNOWN") + output = str(result.get("output") or "").strip() + print(f"--- {host['host']} {host['ip']} {host['instanceId']} {status} ---") + print(output[-3000:] if len(output) > 3000 else output) + if status != "SUCCESS": + raise SystemExit(f"remote image prune failed on {host['host']} {host['instanceId']}: {status}") +PY +} + configure_harbor_retention_and_gc() { log_section "ensure harbor retention and gc schedule" @@ -319,6 +425,7 @@ main() { cleanup_build_roots cleanup_job_logs cleanup_docker + cleanup_remote_host_images configure_harbor_retention_and_gc cleanup_own_logs @@ -328,3 +435,4 @@ main() { } main "$@" + diff --git a/tests/test_gateway_release_clb.py b/tests/test_gateway_release_clb.py index a2cc2c9..95c842e 100644 --- a/tests/test_gateway_release_clb.py +++ b/tests/test_gateway_release_clb.py @@ -224,6 +224,10 @@ class GatewayReleaseFlowTests(unittest.TestCase): @patch("monitors.yumi.service_release.host_image_plan") @patch("monitors.yumi.service_release.update_local_service_image") @patch("monitors.yumi.service_release.sync_remote_host_compose_template") + @patch( + "monitors.yumi.service_release.prune_remote_host_images", + new=lambda host_entry: {"status": "SKIPPED", "output": "", "keepDays": 7, "pruneUntil": ""}, + ) @patch("monitors.yumi.service_release.prepull_host_images") @patch("monitors.yumi.service_release.prepare_gateway_release_clb_snapshot") @patch("monitors.yumi.service_release.drain_gateway_release_target") @@ -345,6 +349,10 @@ class GatewayReleaseFlowTests(unittest.TestCase): @patch("monitors.yumi.service_release.host_image_plan") @patch("monitors.yumi.service_release.update_local_service_image") @patch("monitors.yumi.service_release.sync_remote_host_compose_template") + @patch( + "monitors.yumi.service_release.prune_remote_host_images", + new=lambda host_entry: {"status": "SKIPPED", "output": "", "keepDays": 7, "pruneUntil": ""}, + ) @patch("monitors.yumi.service_release.prepull_host_images") @patch("monitors.yumi.service_release.prepare_gateway_release_clb_snapshot") @patch("monitors.yumi.service_release.drain_gateway_release_target") @@ -422,6 +430,10 @@ class GatewayReleaseFlowTests(unittest.TestCase): @patch("monitors.yumi.service_release.host_image_plan") @patch("monitors.yumi.service_release.update_local_service_image") @patch("monitors.yumi.service_release.sync_remote_host_compose_template") + @patch( + "monitors.yumi.service_release.prune_remote_host_images", + new=lambda host_entry: {"status": "SKIPPED", "output": "", "keepDays": 7, "pruneUntil": ""}, + ) @patch("monitors.yumi.service_release.prepull_host_images") @patch("monitors.yumi.service_release.prepare_gateway_release_clb_snapshot") @patch("monitors.yumi.service_release.restart_service_instance") @@ -498,6 +510,10 @@ class GatewayReleaseFlowTests(unittest.TestCase): @patch("monitors.yumi.service_release.host_image_plan") @patch("monitors.yumi.service_release.update_local_service_image") @patch("monitors.yumi.service_release.sync_remote_host_compose_template") + @patch( + "monitors.yumi.service_release.prune_remote_host_images", + new=lambda host_entry: {"status": "SKIPPED", "output": "", "keepDays": 7, "pruneUntil": ""}, + ) @patch("monitors.yumi.service_release.prepull_host_images") @patch("monitors.yumi.service_release.drain_java_registration") @patch("monitors.yumi.service_release.restart_service_instance")