feat: adopt remote image prune hot-patch from deploy machine

把部署机上未提交的本地热补丁正式入库：服务更新预拉镜像后，顺手清理目标宿主机上超过 REMOTE_IMAGE_PRUNE_KEEP_DAYS 天且未被容器引用的历史镜像； cleanup_deploy_disk.sh 同步增加 cleanup_remote_host_images。发布流程测试补上 prune_remote_host_images 的 mock，避免单测触发真实 SSH。 Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>
2026-07-10 13:05:51 +08:00 · 2026-07-10 13:05:51 +08:00 · a2ff4768d3
commit a2ff4768d3
parent f89ebe797e
5 changed files with 200 additions and 0 deletions
--- a/.env.example
+++ b/.env.example
@ -134,6 +134,10 @@ GATEWAY_CLB_DRAIN_SECONDS=0
 # 恢复健康后再还原权重；不配置则跳过内网 CLB 摘流（历史行为）。
 INTERNAL_CLB_IDS=
 INTERNAL_CLB_DRAIN_SECONDS=10
+# 服务更新时顺手清理目标宿主机上超过 N 天且未被引用的历史镜像。
+REMOTE_IMAGE_PRUNE_ENABLED=1
+REMOTE_IMAGE_PRUNE_KEEP_DAYS=7
+REMOTE_IMAGE_PRUNE_TIMEOUT_SECONDS=300
 DEPLOY_SSH_HOST=
 DEPLOY_SSH_USER=root
 DEPLOY_SSH_PORT=22
--- a/core/config.py
+++ b/core/config.py
@ -107,6 +107,12 @@ SERVICE_LOG_EXPORT_MAX_LINES = env_int("SERVICE_LOG_EXPORT_MAX_LINES", 10000)
 SERVICE_BUILD_TIMEOUT_SECONDS = env_int("SERVICE_BUILD_TIMEOUT_SECONDS", 3600)
 # 服务更新时目标机预拉镜像最长等待秒数。
 SERVICE_PULL_TIMEOUT_SECONDS = env_int("SERVICE_PULL_TIMEOUT_SECONDS", 600)
+# 服务更新时是否顺手清理目标宿主机历史镜像，默认开启以避免业务机镜像持续堆积。
+REMOTE_IMAGE_PRUNE_ENABLED = env_str("REMOTE_IMAGE_PRUNE_ENABLED", "1").lower() in {"1", "true", "yes", "on"}
+# 目标宿主机只清理超过该天数且没有被任何容器引用的镜像。
+REMOTE_IMAGE_PRUNE_KEEP_DAYS = env_int("REMOTE_IMAGE_PRUNE_KEEP_DAYS", 7)
+# 目标宿主机镜像清理的最长等待秒数；镜像很多时允许比普通预拉多一点时间。
+REMOTE_IMAGE_PRUNE_TIMEOUT_SECONDS = env_int("REMOTE_IMAGE_PRUNE_TIMEOUT_SECONDS", 300)
 # 固定构建工作区根目录，留空时按本地 / deploy 机路径自动推导。
 SERVICE_BUILD_CACHE_ROOT = env_str("SERVICE_BUILD_CACHE_ROOT", "")
 # Java Maven 并行线程，默认按 1 个 CPU 组装 reactor。
--- a/monitors/yumi/service_release.py
+++ b/monitors/yumi/service_release.py
@ -60,6 +60,9 @@ from core.config import (
    ROOT,
    ROLLING_RESTART_STABILIZE_SECONDS,
    ROLLING_RESTART_TIMEOUT_SECONDS,
+    REMOTE_IMAGE_PRUNE_ENABLED,
+    REMOTE_IMAGE_PRUNE_KEEP_DAYS,
+    REMOTE_IMAGE_PRUNE_TIMEOUT_SECONDS,
    RUNTIME_BASE_DIR,
    SERVICE_BUILD_CACHE_ROOT,
    SERVICE_BUILD_TIMEOUT_SECONDS,
@ -1359,6 +1362,52 @@ def prepull_host_images(host_entry: dict[str, Any]) -> dict[str, Any]:
    }


+def remote_image_prune_until() -> str:
+    # Docker 的 until 过滤按小时表达最直观；配置异常时直接失败，避免误删最近镜像。
+    keep_days = int(REMOTE_IMAGE_PRUNE_KEEP_DAYS)
+    if keep_days <= 0:
+        raise RuntimeError(f"REMOTE_IMAGE_PRUNE_KEEP_DAYS must be positive, got {keep_days}")
+    return f"{keep_days * 24}h"
+
+
+def prune_remote_host_images(host_entry: dict[str, Any]) -> dict[str, Any]:
+    # 只清理“未被任何容器引用”的历史镜像，不碰容器、卷和 builder cache，保证运行中服务和回滚窗口不被破坏。
+    if not REMOTE_IMAGE_PRUNE_ENABLED:
+        return {
+            "status": "SKIPPED",
+            "output": "REMOTE_IMAGE_PRUNE_ENABLED is disabled",
+            "keepDays": int(REMOTE_IMAGE_PRUNE_KEEP_DAYS),
+            "pruneUntil": "",
+        }
+
+    prune_until = remote_image_prune_until()
+    script = f"""set -eu
+if ! command -v docker >/dev/null 2>&1; then
+  echo "docker not found, skip image prune"
+  exit 0
+fi
+echo "prune_until={shlex.quote(prune_until)}"
+echo "--- docker system df before ---"
+docker system df || true
+echo "--- docker image prune ---"
+docker image prune -af --filter "until={shlex.quote(prune_until)}"
+echo "--- docker system df after ---"
+docker system df || true
+"""
+    result = remote_host_result(
+        host_entry,
+        script,
+        f"image-prune-{host_entry['host']}",
+        timeout_seconds=max(REMOTE_IMAGE_PRUNE_TIMEOUT_SECONDS, 120),
+    )
+    return {
+        "status": result.get("status"),
+        "output": str(result.get("output") or "").strip()[-2000:],
+        "keepDays": int(REMOTE_IMAGE_PRUNE_KEEP_DAYS),
+        "pruneUntil": prune_until,
+    }
+
+
 def dashboard_cdc_env_path(host_name: str) -> str:
    return f"{RUNTIME_BASE_DIR.rstrip('/')}/{host_name}/dashboard-cdc-worker.env"

@ -1839,6 +1888,17 @@ def deploy_runtime_services_payload(
                f"image.prepull.{host_entry['host']}",
                f"预拉完成：{prepull_result['status']} {short_text(prepull_result['output'])}",
            )
+            emit_progress(
+                progress,
+                f"image.prune.{host_entry['host']}",
+                f"清理 {REMOTE_IMAGE_PRUNE_KEEP_DAYS} 天前未使用历史镜像",
+            )
+            image_prune_result = prune_remote_host_images(host_entry)
+            emit_progress(
+                progress,
+                f"image.prune.{host_entry['host']}",
+                f"清理完成：{image_prune_result['status']} {short_text(image_prune_result['output'])}",
+            )
            prepared_hosts.append(
                {
                    "host": host_entry["host"],
@ -1855,6 +1915,12 @@ def deploy_runtime_services_payload(
                        "status": prepull_result["status"],
                        "output": prepull_result["output"],
                    },
+                    "imagePrune": {
+                        "status": image_prune_result["status"],
+                        "output": image_prune_result["output"],
+                        "keepDays": image_prune_result["keepDays"],
+                        "pruneUntil": image_prune_result["pruneUntil"],
+                    },
                }
            )

--- a/scripts/cleanup_deploy_disk.sh
+++ b/scripts/cleanup_deploy_disk.sh
@ -135,6 +135,112 @@ cleanup_docker() {
  docker system df || true
 }

+cleanup_remote_host_images() {
+  log_section "remote host image prune"
+  "$PYTHON_BIN" - <<'PY'
+from __future__ import annotations
+
+import os
+
+from core.config import (
+    REMOTE_IMAGE_PRUNE_ENABLED,
+    REMOTE_IMAGE_PRUNE_KEEP_DAYS,
+    REMOTE_IMAGE_PRUNE_TIMEOUT_SECONDS,
+    load_hosts,
+)
+from monitors.yumi.tencent_tat import run_shell_command
+
+
+def truthy(value: str) -> bool:
+    # shell 环境变量统一用这个函数识别真假，兼容现有 cleanup dry-run 写法。
+    return value.strip().lower() in {"1", "true", "yes", "on"}
+
+
+dry_run = truthy(os.environ.get("DEPLOY_CLEANUP_DRY_RUN", "0"))
+if not REMOTE_IMAGE_PRUNE_ENABLED:
+    print("REMOTE_IMAGE_PRUNE_ENABLED disabled, skip")
+    raise SystemExit(0)
+
+keep_days = int(REMOTE_IMAGE_PRUNE_KEEP_DAYS)
+if keep_days <= 0:
+    raise SystemExit(f"REMOTE_IMAGE_PRUNE_KEEP_DAYS must be positive, got: {keep_days}")
+
+prune_until = f"{keep_days * 24}h"
+hosts: list[dict[str, str]] = []
+seen_instances: set[str] = set()
+for host in load_hosts():
+    # 只处理真正承载业务服务的宿主机；没有 instanceId 的记录无法通过 TAT 触达，直接跳过。
+    if not host.get("services"):
+        continue
+    instance_id = str(host.get("instanceId") or "").strip()
+    if not instance_id or instance_id in seen_instances:
+        continue
+    seen_instances.add(instance_id)
+    hosts.append(
+        {
+            "host": str(host.get("host") or "").strip(),
+            "ip": str(host.get("ip") or "").strip(),
+            "instanceId": instance_id,
+        }
+    )
+
+if not hosts:
+    print("no runtime hosts with instanceId, skip")
+    raise SystemExit(0)
+
+print(f"target_hosts={len(hosts)} keep_days={keep_days} prune_until={prune_until}")
+for host in hosts:
+    print(f"target={host['host']} {host['ip']} {host['instanceId']}")
+
+if dry_run:
+    print(f"[dry-run] docker image prune -af --filter until={prune_until}")
+    raise SystemExit(0)
+
+script = f'''set -eu
+printf 'hostname='; hostname
+printf 'time='; date -Is
+printf 'root_df_before='; df -h / | awk 'NR==2{{print $2","$3","$4","$5}}'
+echo '--- docker system df before ---'
+docker system df || true
+out_file=$(mktemp)
+if docker image prune -af --filter "until={prune_until}" >"$out_file" 2>&1; then
+  status=0
+else
+  status=$?
+fi
+printf 'prune_exit=%s\\n' "$status"
+printf 'untagged_refs='; grep -c '^Untagged:' "$out_file" || true
+printf 'deleted_layers='; grep -c '^Deleted:' "$out_file" || true
+grep 'Total reclaimed space:' "$out_file" || true
+if [ "$status" -ne 0 ]; then
+  echo '--- prune error output ---'
+  cat "$out_file"
+  exit "$status"
+fi
+rm -f "$out_file"
+echo '--- docker system df after ---'
+docker system df || true
+printf 'root_df_after='; df -h / | awk 'NR==2{{print $2","$3","$4","$5}}'
+'''
+
+results = run_shell_command(
+    [host["instanceId"] for host in hosts],
+    script,
+    command_name="hy-remote-image-prune",
+    timeout_seconds=max(int(REMOTE_IMAGE_PRUNE_TIMEOUT_SECONDS), 120),
+)
+
+for host in hosts:
+    result = results.get(host["instanceId"], {})
+    status = str(result.get("status") or "UNKNOWN")
+    output = str(result.get("output") or "").strip()
+    print(f"--- {host['host']} {host['ip']} {host['instanceId']} {status} ---")
+    print(output[-3000:] if len(output) > 3000 else output)
+    if status != "SUCCESS":
+        raise SystemExit(f"remote image prune failed on {host['host']} {host['instanceId']}: {status}")
+PY
+}
+
 configure_harbor_retention_and_gc() {
  log_section "ensure harbor retention and gc schedule"

@ -319,6 +425,7 @@ main() {
  cleanup_build_roots
  cleanup_job_logs
  cleanup_docker
+  cleanup_remote_host_images
  configure_harbor_retention_and_gc
  cleanup_own_logs

@ -328,3 +435,4 @@ main() {
 }

 main "$@"
+
--- a/tests/test_gateway_release_clb.py
+++ b/tests/test_gateway_release_clb.py
@ -224,6 +224,10 @@ class GatewayReleaseFlowTests(unittest.TestCase):
    @patch("monitors.yumi.service_release.host_image_plan")
    @patch("monitors.yumi.service_release.update_local_service_image")
    @patch("monitors.yumi.service_release.sync_remote_host_compose_template")
+    @patch(
+        "monitors.yumi.service_release.prune_remote_host_images",
+        new=lambda host_entry: {"status": "SKIPPED", "output": "", "keepDays": 7, "pruneUntil": ""},
+    )
    @patch("monitors.yumi.service_release.prepull_host_images")
    @patch("monitors.yumi.service_release.prepare_gateway_release_clb_snapshot")
    @patch("monitors.yumi.service_release.drain_gateway_release_target")
@ -345,6 +349,10 @@ class GatewayReleaseFlowTests(unittest.TestCase):
    @patch("monitors.yumi.service_release.host_image_plan")
    @patch("monitors.yumi.service_release.update_local_service_image")
    @patch("monitors.yumi.service_release.sync_remote_host_compose_template")
+    @patch(
+        "monitors.yumi.service_release.prune_remote_host_images",
+        new=lambda host_entry: {"status": "SKIPPED", "output": "", "keepDays": 7, "pruneUntil": ""},
+    )
    @patch("monitors.yumi.service_release.prepull_host_images")
    @patch("monitors.yumi.service_release.prepare_gateway_release_clb_snapshot")
    @patch("monitors.yumi.service_release.drain_gateway_release_target")
@ -422,6 +430,10 @@ class GatewayReleaseFlowTests(unittest.TestCase):
    @patch("monitors.yumi.service_release.host_image_plan")
    @patch("monitors.yumi.service_release.update_local_service_image")
    @patch("monitors.yumi.service_release.sync_remote_host_compose_template")
+    @patch(
+        "monitors.yumi.service_release.prune_remote_host_images",
+        new=lambda host_entry: {"status": "SKIPPED", "output": "", "keepDays": 7, "pruneUntil": ""},
+    )
    @patch("monitors.yumi.service_release.prepull_host_images")
    @patch("monitors.yumi.service_release.prepare_gateway_release_clb_snapshot")
    @patch("monitors.yumi.service_release.restart_service_instance")
@ -498,6 +510,10 @@ class GatewayReleaseFlowTests(unittest.TestCase):
    @patch("monitors.yumi.service_release.host_image_plan")
    @patch("monitors.yumi.service_release.update_local_service_image")
    @patch("monitors.yumi.service_release.sync_remote_host_compose_template")
+    @patch(
+        "monitors.yumi.service_release.prune_remote_host_images",
+        new=lambda host_entry: {"status": "SKIPPED", "output": "", "keepDays": 7, "pruneUntil": ""},
+    )
    @patch("monitors.yumi.service_release.prepull_host_images")
    @patch("monitors.yumi.service_release.drain_java_registration")
    @patch("monitors.yumi.service_release.restart_service_instance")