from __future__ import annotations from concurrent.futures import ThreadPoolExecutor, as_completed # 镜像 tag 需要拼时间戳。 from datetime import datetime # 本地构建和 git worktree 都依赖标准库 subprocess / tempfile。 import json import os from pathlib import Path import re import shlex import shutil import subprocess import tempfile import threading import time from typing import Any, Callable from .compose_templates import ( load_local_host_compose_model, sync_remote_host_compose_template, update_local_service_image, ) from core.config import ( ADMIN_FRONTEND_BUILD_SCRIPT, ADMIN_FRONTEND_COMPOSE_FILE, ADMIN_FRONTEND_COMPOSE_SERVICE, ADMIN_FRONTEND_DIST_DIR, ADMIN_FRONTEND_ENABLED, ADMIN_FRONTEND_HOST, ADMIN_FRONTEND_INSTANCE_ID, ADMIN_FRONTEND_IP, ADMIN_FRONTEND_PUBLIC_URL, ADMIN_FRONTEND_RELEASE_META_PATH, ADMIN_FRONTEND_REMOTE_DIR, ADMIN_FRONTEND_REPO_ROOT, ADMIN_FRONTEND_REPO_URL, ADMIN_FRONTEND_SERVICE_NAME, ADMIN_FRONTEND_SITE_DIR, ADMIN_FRONTEND_SSH_HOST, ADMIN_FRONTEND_SSH_PORT, ADMIN_FRONTEND_SSH_USE_SUDO, ADMIN_FRONTEND_SSH_USER, BUILD_DOCKER_PLATFORM, GOLANG_REPO_ROOT, HARBOR_PASSWORD, HARBOR_PROJECT, HARBOR_REGISTRY, HARBOR_USERNAME, JAVA_IMAGE_BUILD_MAX_WORKERS, JAVA_MAVEN_THREADS, JAVA_REPO_ROOT, ROOT, ROLLING_RESTART_STABILIZE_SECONDS, ROLLING_RESTART_TIMEOUT_SECONDS, SERVICE_BUILD_CACHE_ROOT, SERVICE_BUILD_TIMEOUT_SECONDS, SERVICE_PULL_TIMEOUT_SECONDS, UPDATE_DEFAULT_ADMIN_GIT_REF, UPDATE_DEFAULT_GO_GIT_REF, UPDATE_DEFAULT_JAVA_GIT_REF, utc_now, ) from .service_ops import ( clear_operation_caches, restart_service_instance, service_records_by_name, wait_http_ready, wait_java_registration_ready, ) from .ssh_remote import copy_local_path_to_host, read_remote_text, run_host_script_checked, target_display, write_remote_text # Java 仓里当前支持 Harbor 更新的服务集合。 JAVA_BUILDABLE_SERVICES = ("auth", "gateway", "external", "console", "other", "live", "wallet", "order") # Go 仓目前只有主业务服务。 GOLANG_BUILDABLE_SERVICES = ("golang",) # Admin 前端当前只接入一个发布目标。 FRONTEND_BUILDABLE_SERVICES = (ADMIN_FRONTEND_SERVICE_NAME,) # Java 批量 Maven 构建时，需要映射回对应的启动模块。 JAVA_SERVICE_MODULES = { "auth": "rc-auth", "gateway": "rc-gateway", "external": "rc-service/rc-service-external/external-start", "console": "rc-service/rc-service-console/console-start", "other": "rc-service/rc-service-other/other-start", "live": "rc-service/rc-service-live/live-start", "wallet": "rc-service/rc-service-wallet/wallet-start", "order": "rc-service/rc-service-order/order-start", } # 当前线上镜像仓路径并不完全统一，这里只给缺失模板信息时兜底。 SERVICE_IMAGE_REPOSITORY_SUFFIXES = { "auth": "java/chatapp3-auth", "gateway": "java/chatapp3-gateway", "external": "java/chatapp3-external", "console": "console", "other": "java/chatapp3-other", "live": "java/chatapp3-live", "wallet": "java/chatapp3-wallet", "order": "order", "golang": "golang", } # 后台任务的日志回调统一走这个签名。 ProgressLogger = Callable[[str, str, str], None] # 私有 Maven 安装缓存统一放到 run 目录，避免混进 Git 工作区。 PRIVATE_MAVEN_CACHE_DIR = ROOT / "run" / "maven_private_cache" # Admin 前端站点备份最多保留最近若干份。 FRONTEND_SITE_BACKUP_KEEP = 3 def emit_progress(progress: ProgressLogger | None, stage: str, message: str, *, level: str = "info") -> None: # 所有过程日志都经这里发出，调用方传空时自动静默。 if progress is None: return progress(str(stage or "").strip(), str(level or "info").strip() or "info", str(message or "").rstrip()) def short_text(text: str, limit: int = 600) -> str: # 远端输出和异常信息只保留一小段，避免页面日志爆掉。 normalized = " ".join(str(text or "").split()) if len(normalized) <= limit: return normalized return normalized[:limit].rstrip() + " ..." def stream_log_file_lines( log_path: Path, stop_event: threading.Event, *, stage: str, progress: ProgressLogger | None, ) -> None: # 构建日志落本地文件后，这里持续追尾并把新增行推给前端。 if progress is None: return position = 0 remainder = "" while not stop_event.is_set(): if log_path.exists(): with log_path.open("r", encoding="utf-8", errors="replace") as handle: handle.seek(position) chunk = handle.read() position = handle.tell() if chunk: remainder += chunk lines = remainder.splitlines(keepends=True) remainder = "" if lines and not lines[-1].endswith(("\n", "\r")): remainder = lines.pop() for line in lines: text = line.rstrip() if text: emit_progress(progress, stage, text) stop_event.wait(0.4) if remainder.strip(): emit_progress(progress, stage, remainder.rstrip()) def buildable_service_names() -> list[str]: # 页面和接口统一按这里判定哪些服务允许“构建并更新”。 names = [*JAVA_BUILDABLE_SERVICES, *GOLANG_BUILDABLE_SERVICES] if frontend_release_enabled(): names.extend(service_name for service_name in FRONTEND_BUILDABLE_SERVICES if str(service_name or "").strip()) return names def normalize_service_names(service_names: list[str]) -> list[str]: # 请求里可能重复勾选同一个服务，这里先去重并保序。 normalized = [str(name or "").strip() for name in service_names if str(name or "").strip()] return list(dict.fromkeys(normalized)) def service_kind(service_name: str) -> str: # 更新链路目前分 Java / Golang / Frontend 三类仓库。 if service_name in JAVA_BUILDABLE_SERVICES: return "java" if service_name in GOLANG_BUILDABLE_SERVICES: return "golang" if frontend_release_enabled() and service_name in FRONTEND_BUILDABLE_SERVICES: return "frontend" raise RuntimeError(f"unsupported update service: {service_name}") def ensure_repo_root(repo_root: Path, label: str) -> Path: # 构建前先确认业务仓真的存在，避免跑到一半才发现路径错了。 resolved = Path(repo_root).expanduser().resolve() if not resolved.exists(): raise RuntimeError(f"{label} repo root missing: {resolved}") if not (resolved / ".git").exists(): raise RuntimeError(f"{label} repo root is not a git repo: {resolved}") return resolved def ensure_frontend_repo_root(*, progress: ProgressLogger | None = None) -> Path: # Admin 前端允许第一次使用时按配置的 repo URL 自动 clone 到 deploy 机。 repo_root = Path(ADMIN_FRONTEND_REPO_ROOT).expanduser().resolve() if (repo_root / ".git").exists(): return repo_root repo_url = str(ADMIN_FRONTEND_REPO_URL or "").strip() if not repo_url: raise RuntimeError("missing ADMIN_FRONTEND_REPO_URL in .env") repo_root.parent.mkdir(parents=True, exist_ok=True) emit_progress(progress, "build.frontend.repo", f"初始化前端仓库：{repo_url} -> {repo_root}") local_command_output( ["git", "clone", repo_url, str(repo_root)], cwd=repo_root.parent, label="frontend clone repo", timeout_seconds=900, ) return ensure_repo_root(repo_root, "frontend") def local_command_output( command: list[str], *, cwd: Path, label: str, env: dict[str, str] | None = None, input_text: str = "", timeout_seconds: int = 120, ) -> str: # 小输出命令直接走 capture_output，适合 git rev-parse / docker inspect 这类场景。 merged_env = os.environ.copy() if env: merged_env.update(env) result = subprocess.run( command, cwd=str(cwd), text=True, input=input_text if input_text else None, capture_output=True, env=merged_env, timeout=timeout_seconds, check=False, ) if result.returncode != 0: detail = (result.stderr or result.stdout or "").strip() raise RuntimeError(f"{label} failed: {detail or f'exit {result.returncode}'}") return (result.stdout or "").strip() def tail_text(path: Path, max_bytes: int = 8192) -> str: # 大构建日志只回最后一小段，避免把整份 Maven 输出塞进响应体。 if not path.exists(): return "" with path.open("rb") as handle: handle.seek(0, os.SEEK_END) size = handle.tell() handle.seek(max(0, size - max_bytes)) return handle.read().decode("utf-8", errors="replace").strip() def frontend_release_enabled() -> bool: # Admin 前端需要仓库、主机和发布目录都齐备后才开放到页面。 return bool( ADMIN_FRONTEND_ENABLED and str(ADMIN_FRONTEND_SERVICE_NAME or "").strip() and str(ADMIN_FRONTEND_HOST or "").strip() and str(ADMIN_FRONTEND_SITE_DIR or "").strip() and (str(ADMIN_FRONTEND_SSH_HOST or "").strip() or str(ADMIN_FRONTEND_IP or "").strip()) and (str(ADMIN_FRONTEND_REPO_URL or "").strip() or str(ADMIN_FRONTEND_REPO_ROOT or "").strip()) ) def frontend_host_record(service_name: str) -> dict[str, Any]: # 当前只有 admin 前端走这条链路，后续再扩展时继续在这里分流。 if not frontend_release_enabled() or service_name != ADMIN_FRONTEND_SERVICE_NAME: raise RuntimeError(f"unsupported frontend service: {service_name}") return { "host": ADMIN_FRONTEND_HOST, "ip": ADMIN_FRONTEND_IP, "instanceId": ADMIN_FRONTEND_INSTANCE_ID, "sshHost": ADMIN_FRONTEND_SSH_HOST or ADMIN_FRONTEND_IP, "sshUser": ADMIN_FRONTEND_SSH_USER, "sshUseSudo": ADMIN_FRONTEND_SSH_USE_SUDO, "sshPort": ADMIN_FRONTEND_SSH_PORT, } def run_logged_command( command: list[str], *, cwd: Path, label: str, env: dict[str, str] | None = None, timeout_seconds: int = SERVICE_BUILD_TIMEOUT_SECONDS, progress_stage: str = "", progress: ProgressLogger | None = None, ) -> str: # Maven / docker build 输出很大，单独落临时日志再取尾部更稳。 merged_env = os.environ.copy() if env: merged_env.update(env) log_file = tempfile.NamedTemporaryFile(prefix="hy-app-monitor-build-", suffix=".log", delete=False) log_path = Path(log_file.name) log_file.close() try: stop_event = threading.Event() follower: threading.Thread | None = None with log_path.open("w", encoding="utf-8") as handle: process = subprocess.Popen( command, cwd=str(cwd), stdout=handle, stderr=subprocess.STDOUT, text=True, env=merged_env, ) if progress is not None: follower = threading.Thread( target=stream_log_file_lines, args=(log_path, stop_event), kwargs={"stage": progress_stage or label, "progress": progress}, daemon=True, ) follower.start() try: return_code = process.wait(timeout=timeout_seconds) except subprocess.TimeoutExpired as exc: process.kill() process.wait() stop_event.set() if follower is not None: follower.join(timeout=1) raise RuntimeError(f"{label} timed out: {tail_text(log_path)}") from exc finally: stop_event.set() if follower is not None: follower.join(timeout=1) output_tail = tail_text(log_path) if return_code != 0: raise RuntimeError(f"{label} failed: {output_tail or f'exit {return_code}'}") return output_tail finally: log_path.unlink(missing_ok=True) def fetch_repo_refs(repo_root: Path, label: str) -> None: # 统一先刷新远端分支和 tag，后面的 ref 解析才可信。 local_command_output( ["git", "-C", str(repo_root), "fetch", "--all", "--tags", "--prune"], cwd=repo_root, label=f"{label} fetch refs", timeout_seconds=300, ) def local_maven_repo_root() -> Path: # 兼容外部显式指定 MAVEN_REPO_LOCAL，默认仍沿用 ~/.m2/repository。 override = str(os.environ.get("MAVEN_REPO_LOCAL") or "").strip() if override: return Path(override).expanduser() return Path.home() / ".m2" / "repository" def private_maven_tree_id(repo_root: Path, commit: str) -> str: # 用 git tree id 表示 maven_private 内容，只有私有依赖目录变化时才会变化。 try: return local_command_output( ["git", "-C", str(repo_root), "rev-parse", "--verify", f"{commit}:maven_private"], cwd=repo_root, label="resolve private maven tree", ) except Exception: # noqa: BLE001 return "" def private_maven_cache_stamp_path(repo_root: Path) -> Path: # 每个仓保留一份最近已安装的私有依赖 tree id。 return PRIVATE_MAVEN_CACHE_DIR / f"{repo_root.name}.tree" def private_maven_artifacts_present() -> bool: # 本地 Maven 仓至少得有一份私有制品，缓存命中才有意义。 artifacts_root = local_maven_repo_root() / "com" / "red" / "circle" if not artifacts_root.exists(): return False for path in artifacts_root.rglob("*.pom"): if path.is_file(): return True return False def ensure_private_maven_installed( repo_root: Path, worktree_root: Path, commit: str, *, progress: ProgressLogger | None = None, ) -> None: # 私有依赖没变化时直接复用本机 ~/.m2 缓存，避免每次更新都重复 install-file。 tree_id = private_maven_tree_id(repo_root, commit) stamp_path = private_maven_cache_stamp_path(repo_root) cached_tree_id = stamp_path.read_text(encoding="utf-8").strip() if stamp_path.exists() else "" if tree_id and cached_tree_id == tree_id and private_maven_artifacts_present(): emit_progress(progress, "build.java.maven", f"私有 Maven 依赖未变化，跳过安装：{tree_id[:12]}") return emit_progress(progress, "build.java.maven", "开始安装私有 Maven 依赖") run_logged_command( [str(worktree_root / "ci" / "install-private-maven.sh")], cwd=worktree_root, label="java install private maven", timeout_seconds=min(SERVICE_BUILD_TIMEOUT_SECONDS, 1200), progress_stage="build.java.maven.install", progress=progress, ) if tree_id: PRIVATE_MAVEN_CACHE_DIR.mkdir(parents=True, exist_ok=True) stamp_path.write_text(f"{tree_id}\n", encoding="utf-8") emit_progress(progress, "build.java.maven", f"私有 Maven 依赖安装完成：{tree_id[:12]}") return emit_progress(progress, "build.java.maven", "私有 Maven 依赖安装完成") def resolve_repo_commit(repo_root: Path, git_ref: str, label: str) -> str: # 优先把分支解析到 origin/，避免本地旧分支 HEAD 干扰构建。 candidates = [ f"refs/remotes/origin/{git_ref}^{{commit}}", f"refs/tags/{git_ref}^{{commit}}", f"{git_ref}^{{commit}}", ] for candidate in candidates: try: return local_command_output( ["git", "-C", str(repo_root), "rev-parse", "--verify", candidate], cwd=repo_root, label=f"{label} resolve ref", ) except Exception: # noqa: BLE001 continue raise RuntimeError(f"{label} git ref not found: {git_ref}") def build_cache_root(repo_root: Path) -> Path: # 显式配置优先；未配置时按本地 / deploy 机目录结构推导默认缓存根目录。 override = str(SERVICE_BUILD_CACHE_ROOT or "").strip() if override: return Path(override).expanduser() if str(repo_root).startswith("/opt/deploy/"): return Path("/opt/deploy/build-cache") return ROOT / "run" / "build_cache" def repo_origin_url(repo_root: Path, label: str) -> str: # 固定构建工作区优先直接连 origin，避免本地源码仓 owner 不一致触发 safe.directory。 try: return local_command_output( ["git", "-C", str(repo_root), "remote", "get-url", "origin"], cwd=repo_root, label=f"{label} get origin url", timeout_seconds=120, ) except Exception: # noqa: BLE001 return str(repo_root) def persistent_build_workspace_path(repo_root: Path, git_ref: str) -> Path: # 同一仓库同一 ref 固定复用一份构建目录，最大化保留 target 缓存。 return build_cache_root(repo_root) / repo_root.name / sanitized_ref_fragment(git_ref) def prepare_persistent_build_workspace( repo_root: Path, git_ref: str, commit: str, label: str, *, progress: ProgressLogger | None = None, ) -> Path: # 固定构建工作区是专用缓存目录，允许 reset --hard 到目标提交。 workspace_root = persistent_build_workspace_path(repo_root, git_ref) workspace_root.parent.mkdir(parents=True, exist_ok=True) origin_url = repo_origin_url(repo_root, label) if not (workspace_root / ".git").exists(): if workspace_root.exists(): shutil.rmtree(workspace_root, ignore_errors=True) local_command_output( ["git", "clone", "--no-checkout", origin_url, str(workspace_root)], cwd=workspace_root.parent, label=f"{label} clone build workspace", timeout_seconds=600, ) local_command_output( ["git", "-C", str(workspace_root), "remote", "set-url", "origin", origin_url], cwd=workspace_root, label=f"{label} set build workspace origin", timeout_seconds=120, ) local_command_output( ["git", "-C", str(workspace_root), "fetch", "origin", "--tags", "--prune"], cwd=workspace_root, label=f"{label} fetch build workspace", timeout_seconds=300, ) local_command_output( ["git", "-C", str(workspace_root), "fetch", "origin", commit], cwd=workspace_root, label=f"{label} fetch build workspace commit", timeout_seconds=300, ) local_command_output( ["git", "-C", str(workspace_root), "reset", "--hard", commit], cwd=workspace_root, label=f"{label} reset build workspace", timeout_seconds=300, ) emit_progress(progress, f"build.{label}", f"已准备固定构建工作区：{workspace_root} -> {commit[:12]}") return workspace_root def create_temp_worktree(repo_root: Path, commit: str, label: str) -> Path: # 用临时 worktree 构建，避免直接改业务仓当前工作区。 worktree_root = Path(tempfile.mkdtemp(prefix=f"hy-app-monitor-{repo_root.name}-")) try: local_command_output( ["git", "-C", str(repo_root), "worktree", "add", "--detach", str(worktree_root), commit], cwd=repo_root, label=f"{label} create worktree", timeout_seconds=300, ) return worktree_root except Exception: shutil.rmtree(worktree_root, ignore_errors=True) raise def remove_temp_worktree(repo_root: Path, worktree_root: Path) -> None: # 构建结束后清理 worktree，避免 deploy 机累积一堆临时目录。 try: local_command_output( ["git", "-C", str(repo_root), "worktree", "remove", "--force", str(worktree_root)], cwd=repo_root, label="remove worktree", timeout_seconds=120, ) except Exception: # noqa: BLE001 shutil.rmtree(worktree_root, ignore_errors=True) def require_harbor_credentials() -> None: # 构建推送和目标机预拉都依赖 Harbor 账号。 if not HARBOR_REGISTRY: raise RuntimeError("missing HARBOR_REGISTRY in .env") if not HARBOR_USERNAME or not HARBOR_PASSWORD: raise RuntimeError("missing HARBOR_USERNAME / HARBOR_PASSWORD in .env") def docker_login_local(repo_root: Path, *, progress: ProgressLogger | None = None) -> None: # 本机构建前先登录 Harbor，避免 push 时才失败。 require_harbor_credentials() emit_progress(progress, "prepare", f"登录 Harbor：{HARBOR_REGISTRY}") local_command_output( ["docker", "login", HARBOR_REGISTRY, "-u", HARBOR_USERNAME, "--password-stdin"], cwd=repo_root, label="docker login harbor", input_text=HARBOR_PASSWORD, timeout_seconds=120, ) emit_progress(progress, "prepare", "Harbor 登录成功") def image_repository_ref(image_ref: str) -> str: # 把 tag / digest 去掉，保留完整仓库地址。 text = str(image_ref or "").strip() if "@" in text: return text.split("@", 1)[0] last_slash = text.rfind("/") last_colon = text.rfind(":") if last_colon > last_slash: return text[:last_colon] return text def sanitized_ref_fragment(git_ref: str) -> str: # Harbor tag 只保留相对安全的字符。 cleaned = re.sub(r"[^0-9A-Za-z._-]+", "-", str(git_ref or "").strip()) or "manual" return cleaned.strip("-") or "manual" def derive_image_tag(git_ref: str, commit: str, image_tag: str) -> str: # 用户没指定 tag 时，默认用 ref + commit12 + 时间戳。 explicit = str(image_tag or "").strip() if explicit: return explicit timestamp = datetime.utcnow().strftime("%Y%m%d-%H%M%S") return f"{sanitized_ref_fragment(git_ref)}-{commit[:12]}-{timestamp}" def frontend_version_label(git_ref: str, commit: str) -> str: # 前端发布结果统一显示成 ref@commit12，便于和镜像 tag 区分。 resolved_ref = str(git_ref or "").strip() or UPDATE_DEFAULT_ADMIN_GIT_REF return f"{resolved_ref}@{commit[:12]}" def frontend_release_metadata(service_name: str) -> dict[str, Any]: # 当前前端站点的已部署版本由远端元数据文件提供；缺失时允许返回空结构。 host = frontend_host_record(service_name) try: raw = read_remote_text(host, ADMIN_FRONTEND_RELEASE_META_PATH, timeout_seconds=30).strip() except Exception as exc: # noqa: BLE001 return {"error": str(exc)} if not raw: return {} try: payload = json.loads(raw) except json.JSONDecodeError: return {"error": f"invalid frontend release metadata on {target_display(host)}"} if not isinstance(payload, dict): return {"error": f"invalid frontend release metadata on {target_display(host)}"} return payload def image_release_display(current_images: list[str]) -> dict[str, str]: # 镜像型服务在表格里固定展示 tag / digest / 可复制的主镜像。 primary_image = str((current_images or [""])[0] or "").strip() if not primary_image: return { "currentLabel": "-", "currentTitle": "-", "detailLabel": "-", "detailTitle": "-", "copyValue": "", } repository = image_repository_ref(primary_image) meta = { "repository": repository, "tag": "", "digest": "", } without_digest, _, digest_text = primary_image.partition("@") meta["digest"] = digest_text or "" last_slash = without_digest.rfind("/") last_colon = without_digest.rfind(":") if last_colon > last_slash: meta["tag"] = without_digest[last_colon + 1 :] return { "currentLabel": meta["tag"] or "-", "currentTitle": "\n".join(current_images) if len(current_images) > 1 else primary_image, "detailLabel": (meta["digest"][:12] + "...") if len(meta["digest"]) > 15 else (meta["digest"] or "-"), "detailTitle": primary_image, "copyValue": primary_image, } def frontend_release_display(service_name: str) -> dict[str, str]: # 前端目标优先显示最近一次由监控面板写入的 gitRef / commit。 metadata = frontend_release_metadata(service_name) version = str(metadata.get("version") or "").strip() commit = str(metadata.get("commit") or "").strip() git_ref = str(metadata.get("gitRef") or "").strip() deployed_at = str(metadata.get("deployedAt") or "").strip() error = str(metadata.get("error") or "").strip() if version or commit or git_ref: label = version or git_ref or commit[:12] title_parts = [part for part in [version, git_ref, commit, deployed_at, ADMIN_FRONTEND_PUBLIC_URL] if part] return { "currentLabel": label, "currentTitle": " | ".join(title_parts) if title_parts else label, "detailLabel": commit[:12] if commit else "-", "detailTitle": commit or deployed_at or label, "copyValue": commit or git_ref, } if error: return { "currentLabel": "未记录", "currentTitle": error, "detailLabel": "SSH", "detailTitle": error, "copyValue": "", } return { "currentLabel": "未记录", "currentTitle": "当前站点还没有由 hy-app-monitor 写入过版本元数据", "detailLabel": "-", "detailTitle": "-", "copyValue": "", } def docker_digest(image_ref: str, *, cwd: Path) -> str: # push 完要拿 digest，后续 compose 一律按 digest 部署。 return local_command_output( ["docker", "image", "inspect", image_ref, "--format", "{{index .RepoDigests 0}}"], cwd=cwd, label=f"inspect digest {image_ref}", timeout_seconds=120, ) def current_template_image(service_name: str) -> str: # 优先从本地完整模板里拿当前运行时镜像仓路径。 records = service_records_by_name().get(service_name) or [] for record in records: model = load_local_host_compose_model(record["host"]) image_ref = str(((model.get("services") or {}).get(service_name) or {}).get("image") or "").strip() if image_ref.startswith(f"{HARBOR_REGISTRY}/{HARBOR_PROJECT}/"): return image_ref return "" def service_repository_ref(service_name: str) -> str: # 线上已有镜像时沿用现有仓路径；没有时再走保守兜底。 current_image = current_template_image(service_name) if current_image: return image_repository_ref(current_image) suffix = SERVICE_IMAGE_REPOSITORY_SUFFIXES.get(service_name, "").strip() if not suffix: raise RuntimeError(f"missing Harbor repository mapping for service: {service_name}") return f"{HARBOR_REGISTRY}/{HARBOR_PROJECT}/{suffix}" def maven_module_list(service_names: list[str]) -> str: # Java 多服务更新时，先一次性 Maven 打包对应模块，避免重复 clean package。 modules: list[str] = [] for service_name in service_names: module = JAVA_SERVICE_MODULES.get(service_name) if not module: raise RuntimeError(f"missing java maven module mapping: {service_name}") if module not in modules: modules.append(module) return ",".join(modules) def java_maven_package_command(service_names: list[str], *, clean: bool = False) -> list[str]: # Java 默认走增量 package，并允许通过 .env 配 Maven 并行线程。 command = ["mvn"] maven_threads = str(JAVA_MAVEN_THREADS or "").strip() if maven_threads: command.extend(["-T", maven_threads]) command.extend( [ "-pl", maven_module_list(service_names), "-am", "-Dmaven.test.skip=true", ] ) if clean: command.append("clean") command.append("package") return command def should_retry_java_maven_with_clean(error_text: str) -> bool: # 固定工作区第一次启用时，历史 target 产物偶尔会有损坏；命中后退回一次 clean package 自愈。 normalized = str(error_text or "") markers = ( "java.util.zip.ZipException", "java.io.EOFException", "bad class file", "invalid loc", "invalid block type", "Unexpected end of ZLIB input stream", ) return any(marker in normalized for marker in markers) def build_single_java_image( service_name: str, *, worktree_root: Path, short_commit: str, resolved_image_tag: str, progress: ProgressLogger | None = None, ) -> dict[str, Any]: # Maven 产物已经在固定工作区准备好，这里只负责单服务镜像构建和推送。 repository_ref = service_repository_ref(service_name) target_image_ref = f"{repository_ref}:{resolved_image_tag}" emit_progress(progress, f"build.java.{service_name}", f"开始构建镜像：{target_image_ref}") build_tail = run_logged_command( [str(worktree_root / "ci" / "build-service-image.sh"), service_name, target_image_ref, short_commit], cwd=worktree_root, label=f"java build {service_name}", env={ "DOCKER_PLATFORM": BUILD_DOCKER_PLATFORM, # 外层已经统一决定过是否 Maven 打包；镜像阶段一律复用现成产物。 "SKIP_MAVEN_BUILD": "1", }, timeout_seconds=SERVICE_BUILD_TIMEOUT_SECONDS, progress_stage=f"build.java.{service_name}.build", progress=progress, ) emit_progress(progress, f"build.java.{service_name}", f"开始推送 Harbor：{target_image_ref}") push_tail = run_logged_command( ["docker", "push", target_image_ref], cwd=worktree_root, label=f"java push {service_name}", timeout_seconds=SERVICE_BUILD_TIMEOUT_SECONDS, progress_stage=f"build.java.{service_name}.push", progress=progress, ) digest_ref = docker_digest(target_image_ref, cwd=worktree_root) emit_progress(progress, f"build.java.{service_name}", f"镜像已推送完成：{digest_ref}") return { "service": service_name, "repository": repository_ref, "tag": target_image_ref, "image": digest_ref, "buildTail": build_tail, "pushTail": push_tail, } def build_java_images( service_names: list[str], git_ref: str, image_tag: str, *, skip_maven_build: bool, progress: ProgressLogger | None = None, ) -> dict[str, Any]: # Java 服务共享一个仓库和 Maven 本地缓存，这里按仓统一构建。 repo_root = ensure_repo_root(JAVA_REPO_ROOT, "java") resolved_ref = str(git_ref or "").strip() or UPDATE_DEFAULT_JAVA_GIT_REF emit_progress(progress, "build.java", f"刷新 Java 仓库 refs：{resolved_ref}") fetch_repo_refs(repo_root, "java") commit = resolve_repo_commit(repo_root, resolved_ref, "java") emit_progress(progress, "build.java", f"Java ref 已解析：{resolved_ref} -> {commit[:12]}") worktree_root = prepare_persistent_build_workspace( repo_root, resolved_ref, commit, "java", progress=progress, ) short_commit = commit[:12] resolved_image_tag = derive_image_tag(resolved_ref, commit, image_tag) outputs: list[dict[str, Any]] = [] service_images: dict[str, str] = {} try: if not skip_maven_build: ensure_private_maven_installed( repo_root, worktree_root, commit, progress=progress, ) emit_progress(progress, "build.java.maven", f"开始 Maven 打包：{maven_module_list(service_names)}") try: run_logged_command( java_maven_package_command(service_names), cwd=worktree_root, label="java maven package", timeout_seconds=SERVICE_BUILD_TIMEOUT_SECONDS, progress_stage="build.java.maven.package", progress=progress, ) except Exception as exc: # noqa: BLE001 if not should_retry_java_maven_with_clean(str(exc)): raise emit_progress( progress, "build.java.maven", "检测到缓存产物损坏，退回一次 clean package 自愈", level="warning", ) run_logged_command( java_maven_package_command(service_names, clean=True), cwd=worktree_root, label="java maven clean package", timeout_seconds=SERVICE_BUILD_TIMEOUT_SECONDS, progress_stage="build.java.maven.clean-package", progress=progress, ) else: emit_progress(progress, "build.java.maven", "跳过 Maven，直接复用已有 Jar") max_workers = max(1, min(JAVA_IMAGE_BUILD_MAX_WORKERS, len(service_names))) emit_progress(progress, "build.java", f"Java 镜像并行度：{max_workers}") if max_workers == 1: for service_name in service_names: item = build_single_java_image( service_name, worktree_root=worktree_root, short_commit=short_commit, resolved_image_tag=resolved_image_tag, progress=progress, ) service_images[service_name] = item["image"] outputs.append(item) else: items_by_service: dict[str, dict[str, Any]] = {} errors: list[str] = [] with ThreadPoolExecutor(max_workers=max_workers, thread_name_prefix="java-image-build") as executor: future_map = { executor.submit( build_single_java_image, service_name, worktree_root=worktree_root, short_commit=short_commit, resolved_image_tag=resolved_image_tag, progress=progress, ): service_name for service_name in service_names } for future in as_completed(future_map): service_name = future_map[future] try: item = future.result() except Exception as exc: # noqa: BLE001 errors.append(f"{service_name}: {exc}") emit_progress(progress, f"build.java.{service_name}", str(exc), level="error") continue items_by_service[service_name] = item service_images[service_name] = item["image"] if errors: raise RuntimeError("java image build failed: " + "; ".join(errors)) outputs = [items_by_service[service_name] for service_name in service_names] finally: emit_progress(progress, "build.java", f"固定构建工作区保留复用：{worktree_root}") return { "kind": "java", "repoRoot": str(repo_root), "gitRef": resolved_ref, "commit": commit, "imageTag": resolved_image_tag, "services": list(service_names), "items": outputs, "serviceImages": service_images, } def build_golang_images( service_names: list[str], git_ref: str, image_tag: str, *, progress: ProgressLogger | None = None, ) -> dict[str, Any]: # Go 服务当前是单仓构建，逻辑更简单。 repo_root = ensure_repo_root(GOLANG_REPO_ROOT, "golang") resolved_ref = str(git_ref or "").strip() or UPDATE_DEFAULT_GO_GIT_REF emit_progress(progress, "build.golang", f"刷新 Go 仓库 refs：{resolved_ref}") fetch_repo_refs(repo_root, "golang") commit = resolve_repo_commit(repo_root, resolved_ref, "golang") emit_progress(progress, "build.golang", f"Go ref 已解析：{resolved_ref} -> {commit[:12]}") worktree_root = create_temp_worktree(repo_root, commit, "golang") emit_progress(progress, "build.golang", f"已创建 Go 临时 worktree：{worktree_root}") short_commit = commit[:12] resolved_image_tag = derive_image_tag(resolved_ref, commit, image_tag) outputs: list[dict[str, Any]] = [] service_images: dict[str, str] = {} try: for service_name in service_names: repository_ref = service_repository_ref(service_name) target_image_ref = f"{repository_ref}:{resolved_image_tag}" emit_progress(progress, f"build.golang.{service_name}", f"开始构建镜像：{target_image_ref}") build_tail = run_logged_command( [str(worktree_root / "ci" / "build-image.sh"), target_image_ref, short_commit], cwd=worktree_root, label=f"golang build {service_name}", env={"DOCKER_PLATFORM": BUILD_DOCKER_PLATFORM}, timeout_seconds=SERVICE_BUILD_TIMEOUT_SECONDS, progress_stage=f"build.golang.{service_name}.build", progress=progress, ) emit_progress(progress, f"build.golang.{service_name}", f"开始推送 Harbor：{target_image_ref}") push_tail = run_logged_command( ["docker", "push", target_image_ref], cwd=worktree_root, label=f"golang push {service_name}", timeout_seconds=SERVICE_BUILD_TIMEOUT_SECONDS, progress_stage=f"build.golang.{service_name}.push", progress=progress, ) digest_ref = docker_digest(target_image_ref, cwd=worktree_root) emit_progress(progress, f"build.golang.{service_name}", f"镜像已推送完成：{digest_ref}") service_images[service_name] = digest_ref outputs.append( { "service": service_name, "repository": repository_ref, "tag": target_image_ref, "image": digest_ref, "buildTail": build_tail, "pushTail": push_tail, } ) finally: remove_temp_worktree(repo_root, worktree_root) emit_progress(progress, "build.golang", f"已清理 Go worktree：{worktree_root}") return { "kind": "golang", "repoRoot": str(repo_root), "gitRef": resolved_ref, "commit": commit, "imageTag": resolved_image_tag, "services": list(service_names), "items": outputs, "serviceImages": service_images, } def build_frontend_bundle( service_name: str, git_ref: str, *, progress: ProgressLogger | None = None, ) -> dict[str, Any]: # Admin 前端在 deploy 机做 pnpm install + build，最后产出静态 dist 目录。 repo_root = ensure_frontend_repo_root(progress=progress) resolved_ref = str(git_ref or "").strip() or UPDATE_DEFAULT_ADMIN_GIT_REF emit_progress(progress, "build.frontend", f"刷新前端仓库 refs：{resolved_ref}") fetch_repo_refs(repo_root, "frontend") commit = resolve_repo_commit(repo_root, resolved_ref, "frontend") emit_progress(progress, "build.frontend", f"Frontend ref 已解析：{resolved_ref} -> {commit[:12]}") worktree_root = prepare_persistent_build_workspace( repo_root, resolved_ref, commit, "frontend", progress=progress, ) shim_dir = worktree_root / ".hy-app-monitor-bin" shim_dir.mkdir(parents=True, exist_ok=True) pnpm_shim = shim_dir / "pnpm" pnpm_shim.write_text("#!/bin/sh\nexec corepack pnpm \"$@\"\n", encoding="utf-8") pnpm_shim.chmod(0o755) frontend_env = { "CI": "true", "NODE_OPTIONS": "--max-old-space-size=8192", "PATH": f"{shim_dir}:{os.environ.get('PATH', '')}", } emit_progress(progress, "build.frontend", f"已注入 pnpm shim：{pnpm_shim}") emit_progress(progress, "build.frontend.install", "开始安装前端依赖") run_logged_command( ["pnpm", "install", "--frozen-lockfile"], cwd=worktree_root, label="frontend pnpm install", env=frontend_env, timeout_seconds=SERVICE_BUILD_TIMEOUT_SECONDS, progress_stage="build.frontend.install", progress=progress, ) emit_progress(progress, "build.frontend.build", f"开始构建静态站点：{ADMIN_FRONTEND_BUILD_SCRIPT}") run_logged_command( ["pnpm", "run", ADMIN_FRONTEND_BUILD_SCRIPT], cwd=worktree_root, label="frontend pnpm build", env=frontend_env, timeout_seconds=SERVICE_BUILD_TIMEOUT_SECONDS, progress_stage="build.frontend.build", progress=progress, ) artifact_root = (worktree_root / ADMIN_FRONTEND_DIST_DIR).resolve() if not artifact_root.exists(): raise RuntimeError(f"frontend dist missing after build: {artifact_root}") if not artifact_root.is_dir(): raise RuntimeError(f"frontend dist is not a directory: {artifact_root}") version = frontend_version_label(resolved_ref, commit) emit_progress(progress, "build.frontend", f"前端静态产物已生成：{artifact_root}") return { "kind": "frontend", "repoRoot": str(repo_root), "repoUrl": str(ADMIN_FRONTEND_REPO_URL or "").strip(), "gitRef": resolved_ref, "commit": commit, "imageTag": version, "services": [service_name], "artifactPath": str(artifact_root), "serviceArtifacts": {service_name: str(artifact_root)}, } def build_service_images( service_names: list[str], *, java_git_ref: str, go_git_ref: str, image_tag: str, skip_maven_build: bool, progress: ProgressLogger | None = None, ) -> dict[str, Any]: # 一个请求里可能同时选了 Java 和 Go，需要分仓构建后再合并 digest。 normalized = normalize_service_names(service_names) java_services = [service_name for service_name in normalized if service_kind(service_name) == "java"] golang_services = [service_name for service_name in normalized if service_kind(service_name) == "golang"] if not normalized: raise RuntimeError("services are required") docker_login_local( ensure_repo_root(JAVA_REPO_ROOT if java_services else GOLANG_REPO_ROOT, "build"), progress=progress, ) builds: list[dict[str, Any]] = [] service_images: dict[str, str] = {} if java_services: java_build = build_java_images( java_services, java_git_ref, image_tag, skip_maven_build=skip_maven_build, progress=progress, ) builds.append(java_build) service_images.update(java_build["serviceImages"]) if golang_services: golang_build = build_golang_images(golang_services, go_git_ref, image_tag, progress=progress) builds.append(golang_build) service_images.update(golang_build["serviceImages"]) return { "builds": builds, "serviceImages": service_images, } def host_image_plan(service_names: list[str], service_images: dict[str, str]) -> list[dict[str, Any]]: # 先按宿主机聚合 image，后面可以先预拉再做滚动重建。 records_map = service_records_by_name() items: list[dict[str, Any]] = [] host_index: dict[str, dict[str, Any]] = {} for service_name in service_names: image_ref = str(service_images.get(service_name) or "").strip() if not image_ref: raise RuntimeError(f"missing built image for service: {service_name}") for record in records_map.get(service_name) or []: host_entry = host_index.get(record["host"]) if host_entry is None: host_entry = { "host": record["host"], "ip": record["ip"], "instanceId": record["instanceId"], "services": [], "images": [], } host_index[record["host"]] = host_entry items.append(host_entry) if service_name not in host_entry["services"]: host_entry["services"].append(service_name) if image_ref not in host_entry["images"]: host_entry["images"].append(image_ref) return items def render_remote_login_lines() -> list[str]: # 目标机拉 Harbor 镜像前同样需要显式登录。 require_harbor_credentials() return [ f"printf '%s' {shlex.quote(HARBOR_PASSWORD)} | docker login {shlex.quote(HARBOR_REGISTRY)} -u {shlex.quote(HARBOR_USERNAME)} --password-stdin", ] def remote_host_result(host_entry: dict[str, Any], script: str, command_name: str, timeout_seconds: int) -> dict[str, Any]: # 统一执行单宿主机命令，便于预拉镜像和其他操作复用。 return run_host_script_checked( host_entry, script, command_name=command_name, timeout_seconds=timeout_seconds, ) def prepull_host_images(host_entry: dict[str, Any]) -> dict[str, Any]: # 先把该宿主机上要更新的镜像拉好，减少单实例重建时的等待。 lines = ["set -eu", *render_remote_login_lines()] for image_ref in host_entry.get("images") or []: lines.append(f"docker pull {shlex.quote(str(image_ref))}") script = "\n".join(lines) + "\n" result = remote_host_result( host_entry, script, f"prepull-{host_entry['host']}", timeout_seconds=max(SERVICE_PULL_TIMEOUT_SECONDS, 120), ) return { "status": result.get("status"), "output": str(result.get("output") or "").strip()[-1200:], } def prepare_frontend_remote_tmp(host: dict[str, Any]) -> str: # 先在远端站点目录旁边建一份临时上传目录，避免直接覆盖线上内容。 site_dir = str(ADMIN_FRONTEND_SITE_DIR or "").rstrip("/") parent_dir = str(Path(site_dir).parent) script = f"""set -eu PARENT_DIR={shlex.quote(parent_dir)} mkdir -p "$PARENT_DIR" mktemp -d "$PARENT_DIR/.hy-app-monitor-site.XXXXXX" """ result = run_host_script_checked( host, script, command_name=f"frontend-tmp-{host['host']}", timeout_seconds=30, ) tmp_dir = str(result.get("output") or "").strip().splitlines()[-1].strip() if not tmp_dir: raise RuntimeError(f"failed to create frontend temp dir on {target_display(host)}") return tmp_dir def activate_frontend_site( host: dict[str, Any], *, tmp_dir: str, backup_stamp: str, ) -> dict[str, str]: # 站点目录保留原 inode，只在目录内替换文件，避免 bind mount 指向失效。 site_dir = str(ADMIN_FRONTEND_SITE_DIR or "").rstrip("/") remote_dir = str(ADMIN_FRONTEND_REMOTE_DIR or "").rstrip("/") compose_file = str(ADMIN_FRONTEND_COMPOSE_FILE or "").rstrip("/") compose_service = str(ADMIN_FRONTEND_COMPOSE_SERVICE or "").strip() or "caddy" backup_root = str(Path(site_dir).parent / ".hy-app-monitor-site-backups") keep_count = max(1, int(FRONTEND_SITE_BACKUP_KEEP)) script = f"""set -eu SITE_DIR={shlex.quote(site_dir)} TMP_DIR={shlex.quote(tmp_dir)} BACKUP_ROOT={shlex.quote(backup_root)} BACKUP_DIR="$BACKUP_ROOT/{shlex.quote(backup_stamp)}" REMOTE_DIR={shlex.quote(remote_dir)} COMPOSE_FILE={shlex.quote(compose_file)} COMPOSE_SERVICE={shlex.quote(compose_service)} KEEP_COUNT={keep_count} mkdir -p "$SITE_DIR" "$BACKUP_ROOT" if [ -d "$SITE_DIR" ]; then mkdir -p "$BACKUP_DIR" cp -a "$SITE_DIR"/. "$BACKUP_DIR"/ 2>/dev/null || true fi find "$SITE_DIR" -mindepth 1 -maxdepth 1 -exec rm -rf {{}} + cp -a "$TMP_DIR"/. "$SITE_DIR"/ rm -rf "$TMP_DIR" if [ -d "$BACKUP_ROOT" ]; then ls -1dt "$BACKUP_ROOT"/* 2>/dev/null | awk "NR>$KEEP_COUNT" | xargs -r rm -rf fi cd "$REMOTE_DIR" docker compose -f "$COMPOSE_FILE" up -d "$COMPOSE_SERVICE" >/dev/null docker compose -f "$COMPOSE_FILE" ps "$COMPOSE_SERVICE" || true """ result = run_host_script_checked( host, script, command_name=f"frontend-activate-{host['host']}", timeout_seconds=max(SERVICE_BUILD_TIMEOUT_SECONDS, 180), ) return { "backupPath": f"{backup_root}/{backup_stamp}", "output": str(result.get("output") or "").strip()[-1200:], } def deploy_frontend_site( service_name: str, *, git_ref: str, progress: ProgressLogger | None = None, ) -> dict[str, Any]: # 前端发布链路：构建 dist -> SCP 到目标机 -> 原地切换站点内容 -> 写版本元数据。 host = frontend_host_record(service_name) build_result = build_frontend_bundle(service_name, git_ref, progress=progress) artifact_path = Path(str(build_result.get("artifactPath") or "")).expanduser().resolve() version = str(build_result.get("imageTag") or "").strip() backup_stamp = datetime.utcnow().strftime("%Y%m%d-%H%M%S") step_stage = f"frontend.{host['host']}.{service_name}" started_at = time.time() step: dict[str, Any] = { "service": service_name, "host": host["host"], "ip": host["ip"], "kind": "frontend", "image": version, "status": "running", } tmp_dir = "" try: tmp_dir = prepare_frontend_remote_tmp(host) emit_progress(progress, step_stage, f"上传静态产物：{artifact_path} -> {target_display(host)}:{tmp_dir}") copy_local_path_to_host( host, f"{artifact_path}/.", f"{tmp_dir}/", recursive=True, timeout_seconds=max(SERVICE_BUILD_TIMEOUT_SECONDS, 180), ) emit_progress(progress, step_stage, "上传完成，开始切换站点目录") activate_result = activate_frontend_site(host, tmp_dir=tmp_dir, backup_stamp=backup_stamp) except Exception as exc: # noqa: BLE001 # 上传或切换失败时尽量收尾远端临时目录，避免后续发布被历史垃圾影响。 cleanup_script = f"""set -eu TMP_DIR={shlex.quote(tmp_dir)} if [ -d "$TMP_DIR" ]; then rm -rf "$TMP_DIR" fi """ if tmp_dir: try: run_host_script_checked( host, cleanup_script, command_name=f"frontend-cleanup-{host['host']}", timeout_seconds=30, ) except Exception: # noqa: BLE001 pass step["status"] = "failed" step["error"] = str(exc) step["durationSeconds"] = round(time.time() - started_at, 2) emit_progress(progress, step_stage, f"前端发布失败：{step['error']}", level="error") return { "ok": False, "updatedAt": utc_now(), "services": [service_name], "build": [build_result], "templates": [], "preparedHosts": [], "steps": [step], "error": step["error"], } deployed_at = utc_now() metadata = { "service": service_name, "kind": "frontend", "repoUrl": str(ADMIN_FRONTEND_REPO_URL or "").strip(), "repoRoot": str(ADMIN_FRONTEND_REPO_ROOT), "gitRef": str(build_result.get("gitRef") or "").strip(), "commit": str(build_result.get("commit") or "").strip(), "version": version, "publicUrl": ADMIN_FRONTEND_PUBLIC_URL, "siteDir": ADMIN_FRONTEND_SITE_DIR, "composeFile": ADMIN_FRONTEND_COMPOSE_FILE, "deployedAt": deployed_at, "backupPath": activate_result["backupPath"], "deployedBy": "hy-app-monitor", } write_remote_text( host, ADMIN_FRONTEND_RELEASE_META_PATH, json.dumps(metadata, ensure_ascii=False, indent=2) + "\n", timeout_seconds=30, ) emit_progress(progress, step_stage, f"版本元数据已写入：{ADMIN_FRONTEND_RELEASE_META_PATH}") step["status"] = "success" step["remoteStatus"] = "SUCCESS" step["remoteOutput"] = activate_result["output"] step["durationSeconds"] = round(time.time() - started_at, 2) return { "ok": True, "updatedAt": deployed_at, "services": [service_name], "build": [build_result], "templates": [], "preparedHosts": [], "steps": [step], "message": "service update complete", } def build_update_targets_payload() -> dict[str, Any]: # 页面需要展示当前可更新服务、宿主机，以及本地构建所依赖的仓路径。 records_map = service_records_by_name() items: list[dict[str, Any]] = [] for service_name in buildable_service_names(): records = records_map.get(service_name) or [] if not records: continue current_images: list[str] = [] for record in records: model = load_local_host_compose_model(record["host"]) image_ref = str(((model.get("services") or {}).get(service_name) or {}).get("image") or "").strip() if image_ref and image_ref not in current_images: current_images.append(image_ref) display = image_release_display(current_images) items.append( { "service": service_name, "kind": service_kind(service_name), "repository": service_repository_ref(service_name), "currentLabel": display["currentLabel"], "currentTitle": display["currentTitle"], "detailLabel": display["detailLabel"], "detailTitle": display["detailTitle"], "copyValue": display["copyValue"], "currentImages": current_images, "hosts": [ { "host": record["host"], "ip": record["ip"], "instanceId": record["instanceId"], "port": record["port"], "path": record["path"], } for record in records ], } ) if frontend_release_enabled(): frontend_service = ADMIN_FRONTEND_SERVICE_NAME display = frontend_release_display(frontend_service) items.append( { "service": frontend_service, "kind": "frontend", "repository": str(ADMIN_FRONTEND_REPO_URL or ADMIN_FRONTEND_REPO_ROOT), "currentLabel": display["currentLabel"], "currentTitle": display["currentTitle"], "detailLabel": display["detailLabel"], "detailTitle": display["detailTitle"], "copyValue": display["copyValue"], "currentImages": [], "hosts": [ { "host": ADMIN_FRONTEND_HOST, "ip": ADMIN_FRONTEND_IP, "instanceId": ADMIN_FRONTEND_INSTANCE_ID, "port": 0, "path": ADMIN_FRONTEND_SITE_DIR, } ], } ) return { "ok": True, "updatedAt": utc_now(), "defaults": { "javaGitRef": UPDATE_DEFAULT_JAVA_GIT_REF, "goGitRef": UPDATE_DEFAULT_GO_GIT_REF, "adminGitRef": UPDATE_DEFAULT_ADMIN_GIT_REF, "dockerPlatform": BUILD_DOCKER_PLATFORM, }, "builder": { "harborRegistry": HARBOR_REGISTRY, "harborProject": HARBOR_PROJECT, "javaRepoRoot": str(Path(JAVA_REPO_ROOT).expanduser()), "golangRepoRoot": str(Path(GOLANG_REPO_ROOT).expanduser()), "frontendRepoRoot": str(Path(ADMIN_FRONTEND_REPO_ROOT).expanduser()), "frontendRepoUrl": str(ADMIN_FRONTEND_REPO_URL or "").strip(), "frontendPublicUrl": ADMIN_FRONTEND_PUBLIC_URL, }, "items": items, } def deploy_runtime_services_payload( service_names: list[str], *, java_git_ref: str, go_git_ref: str, image_tag: str, skip_maven_build: bool, progress: ProgressLogger | None = None, ) -> dict[str, Any]: # Java / Go 服务沿用镜像构建、完整 compose、预拉、滚动重建这条链路。 normalized = normalize_service_names(service_names) if not normalized: raise RuntimeError("services are required") records_map = service_records_by_name() unsupported = [service_name for service_name in normalized if service_name not in buildable_service_names()] if unsupported: raise RuntimeError(f"unsupported update services: {', '.join(unsupported)}") for service_name in normalized: if service_name not in records_map: raise RuntimeError(f"service not found in runtime topology: {service_name}") emit_progress(progress, "prepare", f"准备更新服务：{', '.join(normalized)}") build_result = build_service_images( normalized, java_git_ref=java_git_ref, go_git_ref=go_git_ref, image_tag=image_tag, skip_maven_build=skip_maven_build, progress=progress, ) service_images = dict(build_result.get("serviceImages") or {}) emit_progress(progress, "prepare", "镜像构建完成，开始更新本地完整 compose 模板") template_updates: list[dict[str, Any]] = [] for service_name in normalized: for record in records_map[service_name]: image_ref = service_images[service_name] emit_progress(progress, "compose.template", f"更新模板：{record['host']} / {service_name} -> {image_ref}") template_updates.append(update_local_service_image(record["host"], service_name, image_ref)) host_plan = host_image_plan(normalized, service_images) prepared_hosts: list[dict[str, Any]] = [] steps: list[dict[str, Any]] = [] try: for host_entry in host_plan: emit_progress(progress, f"compose.sync.{host_entry['host']}", f"同步完整 compose 到远端：{host_entry['host']}") compose_result = sync_remote_host_compose_template(host_entry["host"], refresh_from_remote=False) emit_progress( progress, f"compose.sync.{host_entry['host']}", f"compose 同步完成：changed={bool(compose_result.get('changed'))} path={compose_result.get('path')}", ) emit_progress(progress, f"image.prepull.{host_entry['host']}", f"预拉镜像：{', '.join(host_entry.get('images') or [])}") prepull_result = prepull_host_images(host_entry) emit_progress( progress, f"image.prepull.{host_entry['host']}", f"预拉完成：{prepull_result['status']} {short_text(prepull_result['output'])}", ) prepared_hosts.append( { "host": host_entry["host"], "ip": host_entry["ip"], "services": list(host_entry["services"]), "images": list(host_entry["images"]), "compose": { "path": compose_result["path"], "changed": bool(compose_result.get("changed")), "source": str(compose_result.get("source") or ""), "backupPath": str(compose_result.get("backupPath") or ""), }, "prepull": { "status": prepull_result["status"], "output": prepull_result["output"], }, } ) for service_name in normalized: for service in records_map[service_name]: started_at = time.time() step_stage = f"restart.{service['host']}.{service_name}" step: dict[str, Any] = { "service": service_name, "host": service["host"], "ip": service["ip"], "kind": service["kind"], "image": service_images[service_name], "status": "running", } try: emit_progress(progress, step_stage, f"开始滚动更新：{service['host']} / {service_name}") restart_result = restart_service_instance(service) step["remoteStatus"] = restart_result.get("status") step["remoteOutput"] = str(restart_result.get("output") or "").strip()[:500] emit_progress( progress, step_stage, f"远端重建完成：{restart_result.get('status')} {short_text(step['remoteOutput'])}", ) http_result = wait_http_ready(service, timeout_seconds=ROLLING_RESTART_TIMEOUT_SECONDS) step["http"] = { "ok": bool(http_result.get("ok")), "statusCode": int(http_result.get("statusCode") or 0), "latencyMs": int(http_result.get("latencyMs") or 0), "detail": str(http_result.get("detail") or ""), } emit_progress( progress, step_stage, f"HTTP 恢复：{step['http']['statusCode']} / {step['http']['latencyMs']}ms", ) if service["kind"] == "java": step["nacos"] = wait_java_registration_ready( service, timeout_seconds=ROLLING_RESTART_TIMEOUT_SECONDS, ) emit_progress( progress, step_stage, f"Nacos 注册恢复：{step['nacos'].get('source') or 'ok'}", ) if ROLLING_RESTART_STABILIZE_SECONDS > 0: emit_progress( progress, step_stage, f"稳定观察 {ROLLING_RESTART_STABILIZE_SECONDS}s", ) time.sleep(ROLLING_RESTART_STABILIZE_SECONDS) step["status"] = "success" step["durationSeconds"] = round(time.time() - started_at, 2) steps.append(step) emit_progress( progress, step_stage, f"实例更新成功，耗时 {step['durationSeconds']}s", ) except Exception as exc: # noqa: BLE001 step["status"] = "failed" step["error"] = str(exc) step["durationSeconds"] = round(time.time() - started_at, 2) steps.append(step) emit_progress(progress, step_stage, f"实例更新失败：{step['error']}", level="error") clear_operation_caches() return { "ok": False, "updatedAt": utc_now(), "services": normalized, "build": build_result.get("builds") or [], "templates": template_updates, "preparedHosts": prepared_hosts, "steps": steps, "error": step["error"], } finally: clear_operation_caches() emit_progress(progress, "finish", "已清理监控缓存") return { "ok": True, "updatedAt": utc_now(), "services": normalized, "build": build_result.get("builds") or [], "templates": template_updates, "preparedHosts": prepared_hosts, "steps": steps, "message": "service update complete", } def merge_update_results(service_names: list[str], results: list[dict[str, Any]]) -> dict[str, Any]: # 一个任务里允许同时带镜像服务和前端服务，这里统一把结果拼回页面结构。 merged = { "ok": True, "updatedAt": utc_now(), "services": list(service_names), "build": [], "templates": [], "preparedHosts": [], "steps": [], "message": "service update complete", } for result in results: if not result: continue merged["ok"] = merged["ok"] and bool(result.get("ok")) merged["build"].extend(list(result.get("build") or result.get("builds") or [])) merged["templates"].extend(list(result.get("templates") or [])) merged["preparedHosts"].extend(list(result.get("preparedHosts") or [])) merged["steps"].extend(list(result.get("steps") or [])) if not result.get("ok"): merged["error"] = str(result.get("error") or "service update failed").strip() merged["message"] = str(result.get("message") or merged["error"] or "service update failed").strip() return merged if str(result.get("message") or "").strip(): merged["message"] = str(result.get("message") or "").strip() return merged def deploy_updated_services_payload( service_names: list[str], *, java_git_ref: str, go_git_ref: str, admin_git_ref: str, image_tag: str, skip_maven_build: bool, progress: ProgressLogger | None = None, ) -> dict[str, Any]: # 顶层按服务类型拆分：容器服务走 Harbor/compose，Admin 前端走 pnpm build + SSH 发布。 normalized = normalize_service_names(service_names) if not normalized: raise RuntimeError("services are required") unsupported = [service_name for service_name in normalized if service_name not in buildable_service_names()] if unsupported: raise RuntimeError(f"unsupported update services: {', '.join(unsupported)}") runtime_services = [service_name for service_name in normalized if service_kind(service_name) in {"java", "golang"}] frontend_services = [service_name for service_name in normalized if service_kind(service_name) == "frontend"] records_map = service_records_by_name() for service_name in runtime_services: if service_name not in records_map: raise RuntimeError(f"service not found in runtime topology: {service_name}") emit_progress(progress, "prepare", f"准备更新服务：{', '.join(normalized)}") results: list[dict[str, Any]] = [] if runtime_services: runtime_result = deploy_runtime_services_payload( runtime_services, java_git_ref=java_git_ref, go_git_ref=go_git_ref, image_tag=image_tag, skip_maven_build=skip_maven_build, progress=progress, ) results.append(runtime_result) if not runtime_result.get("ok"): return merge_update_results(normalized, results) for service_name in frontend_services: frontend_result = deploy_frontend_site(service_name, git_ref=admin_git_ref, progress=progress) results.append(frontend_result) if not frontend_result.get("ok"): return merge_update_results(normalized, results) emit_progress(progress, "finish", "全部服务更新完成") return merge_update_results(normalized, results)