2101 lines
84 KiB
Python
2101 lines
84 KiB
Python
from __future__ import annotations
|
||
|
||
from concurrent.futures import ThreadPoolExecutor, as_completed
|
||
# 镜像 tag 需要拼时间戳。
|
||
from datetime import datetime
|
||
# 本地构建和 git worktree 都依赖标准库 subprocess / tempfile。
|
||
import json
|
||
import os
|
||
from pathlib import Path
|
||
import re
|
||
import shlex
|
||
import shutil
|
||
import subprocess
|
||
import tempfile
|
||
import threading
|
||
import time
|
||
from typing import Any, Callable
|
||
from urllib.parse import quote, urlparse
|
||
|
||
from .compose_templates import (
|
||
load_local_host_compose_model,
|
||
save_local_host_compose_model,
|
||
sync_remote_host_compose_template,
|
||
update_local_service_image,
|
||
)
|
||
from core.config import (
|
||
ADMIN_FRONTEND_BUILD_SCRIPT,
|
||
ADMIN_FRONTEND_COMPOSE_FILE,
|
||
ADMIN_FRONTEND_COMPOSE_SERVICE,
|
||
ADMIN_FRONTEND_DIST_DIR,
|
||
ADMIN_FRONTEND_ENABLED,
|
||
ADMIN_FRONTEND_HOST,
|
||
ADMIN_FRONTEND_INSTANCE_ID,
|
||
ADMIN_FRONTEND_IP,
|
||
ADMIN_FRONTEND_PUBLIC_URL,
|
||
ADMIN_FRONTEND_RELEASE_META_PATH,
|
||
ADMIN_FRONTEND_REMOTE_DIR,
|
||
ADMIN_FRONTEND_REPO_ROOT,
|
||
ADMIN_FRONTEND_REPO_URL,
|
||
ADMIN_FRONTEND_SERVICE_NAME,
|
||
ADMIN_FRONTEND_SITE_DIR,
|
||
ADMIN_FRONTEND_SSH_HOST,
|
||
ADMIN_FRONTEND_SSH_PORT,
|
||
ADMIN_FRONTEND_SSH_USE_SUDO,
|
||
ADMIN_FRONTEND_SSH_USER,
|
||
BUILD_DOCKER_PLATFORM,
|
||
DASHBOARD_CDC_ENABLED,
|
||
DASHBOARD_CDC_REPO_ROOT,
|
||
DASHBOARD_CDC_REPO_URL,
|
||
DASHBOARD_CDC_SERVICE_NAME,
|
||
GATEWAY_CLB_DRAIN_SECONDS,
|
||
GOLANG_REPO_ROOT,
|
||
HARBOR_PASSWORD,
|
||
HARBOR_PROJECT,
|
||
HARBOR_REGISTRY,
|
||
HARBOR_USERNAME,
|
||
JAVA_IMAGE_BUILD_MAX_WORKERS,
|
||
JAVA_MAVEN_THREADS,
|
||
JAVA_REPO_ROOT,
|
||
ROOT,
|
||
ROLLING_RESTART_STABILIZE_SECONDS,
|
||
ROLLING_RESTART_TIMEOUT_SECONDS,
|
||
RUNTIME_BASE_DIR,
|
||
SERVICE_BUILD_CACHE_ROOT,
|
||
SERVICE_BUILD_TIMEOUT_SECONDS,
|
||
SERVICE_PULL_TIMEOUT_SECONDS,
|
||
UPDATE_DEFAULT_ADMIN_GIT_REF,
|
||
UPDATE_DEFAULT_GO_GIT_REF,
|
||
UPDATE_DEFAULT_JAVA_GIT_REF,
|
||
utc_now,
|
||
)
|
||
from .service_ops import (
|
||
clear_operation_caches,
|
||
drain_java_registration,
|
||
restart_service_instance,
|
||
restore_java_registration,
|
||
service_records_by_name,
|
||
update_java_registration_enabled,
|
||
wait_http_ready,
|
||
wait_java_registration_ready,
|
||
)
|
||
from .ssh_remote import copy_local_path_to_host, read_remote_text, run_host_script_checked, target_display, write_remote_text
|
||
from .tencent_clb import (
|
||
drain_gateway_release_target,
|
||
gateway_clb_configuration_error,
|
||
prepare_gateway_release_clb_snapshot,
|
||
restore_gateway_release_target,
|
||
wait_gateway_release_target_healthy,
|
||
)
|
||
|
||
|
||
# Java 仓里当前支持 Harbor 更新的服务集合。
|
||
JAVA_BUILDABLE_SERVICES = ("auth", "gateway", "external", "console", "other", "live", "wallet", "order")
|
||
# Go 仓目前只有主业务服务。
|
||
GOLANG_BUILDABLE_SERVICES = ("golang",)
|
||
# Dashboard CDC Worker 是独立 Go 仓库，但部署形态仍是普通镜像服务。
|
||
DASHBOARD_CDC_BUILDABLE_SERVICES = (DASHBOARD_CDC_SERVICE_NAME,) if DASHBOARD_CDC_ENABLED else ()
|
||
# Admin 前端当前只接入一个发布目标。
|
||
FRONTEND_BUILDABLE_SERVICES = (ADMIN_FRONTEND_SERVICE_NAME,)
|
||
# Java 批量 Maven 构建时，需要映射回对应的启动模块。
|
||
JAVA_SERVICE_MODULES = {
|
||
"auth": "rc-auth",
|
||
"gateway": "rc-gateway",
|
||
"external": "rc-service/rc-service-external/external-start",
|
||
"console": "rc-service/rc-service-console/console-start",
|
||
"other": "rc-service/rc-service-other/other-start",
|
||
"live": "rc-service/rc-service-live/live-start",
|
||
"wallet": "rc-service/rc-service-wallet/wallet-start",
|
||
"order": "rc-service/rc-service-order/order-start",
|
||
}
|
||
# 当前线上镜像仓路径并不完全统一，这里只给缺失模板信息时兜底。
|
||
SERVICE_IMAGE_REPOSITORY_SUFFIXES = {
|
||
"auth": "java/chatapp3-auth",
|
||
"gateway": "java/chatapp3-gateway",
|
||
"external": "java/chatapp3-external",
|
||
"console": "console",
|
||
"other": "java/chatapp3-other",
|
||
"live": "java/chatapp3-live",
|
||
"wallet": "java/chatapp3-wallet",
|
||
"order": "order",
|
||
"golang": "golang",
|
||
DASHBOARD_CDC_SERVICE_NAME: "dashboard-cdc-worker",
|
||
}
|
||
|
||
# 后台任务的日志回调统一走这个签名。
|
||
ProgressLogger = Callable[[str, str, str], None]
|
||
# 私有 Maven 安装缓存统一放到 run 目录，避免混进 Git 工作区。
|
||
PRIVATE_MAVEN_CACHE_DIR = ROOT / "run" / "maven_private_cache"
|
||
# Admin 前端站点备份最多保留最近若干份。
|
||
FRONTEND_SITE_BACKUP_KEEP = 3
|
||
|
||
|
||
def emit_progress(progress: ProgressLogger | None, stage: str, message: str, *, level: str = "info") -> None:
|
||
# 所有过程日志都经这里发出，调用方传空时自动静默。
|
||
if progress is None:
|
||
return
|
||
progress(str(stage or "").strip(), str(level or "info").strip() or "info", str(message or "").rstrip())
|
||
|
||
|
||
def short_text(text: str, limit: int = 600) -> str:
|
||
# 远端输出和异常信息只保留一小段，避免页面日志爆掉。
|
||
normalized = " ".join(str(text or "").split())
|
||
if len(normalized) <= limit:
|
||
return normalized
|
||
return normalized[:limit].rstrip() + " ..."
|
||
|
||
|
||
def stream_log_file_lines(
|
||
log_path: Path,
|
||
stop_event: threading.Event,
|
||
*,
|
||
stage: str,
|
||
progress: ProgressLogger | None,
|
||
) -> None:
|
||
# 构建日志落本地文件后，这里持续追尾并把新增行推给前端。
|
||
if progress is None:
|
||
return
|
||
|
||
position = 0
|
||
remainder = ""
|
||
|
||
while not stop_event.is_set():
|
||
if log_path.exists():
|
||
with log_path.open("r", encoding="utf-8", errors="replace") as handle:
|
||
handle.seek(position)
|
||
chunk = handle.read()
|
||
position = handle.tell()
|
||
if chunk:
|
||
remainder += chunk
|
||
lines = remainder.splitlines(keepends=True)
|
||
remainder = ""
|
||
if lines and not lines[-1].endswith(("\n", "\r")):
|
||
remainder = lines.pop()
|
||
for line in lines:
|
||
text = line.rstrip()
|
||
if text:
|
||
emit_progress(progress, stage, text)
|
||
stop_event.wait(0.4)
|
||
|
||
if remainder.strip():
|
||
emit_progress(progress, stage, remainder.rstrip())
|
||
|
||
|
||
def buildable_service_names() -> list[str]:
|
||
# 页面和接口统一按这里判定哪些服务允许“构建并更新”。
|
||
names = [*JAVA_BUILDABLE_SERVICES, *GOLANG_BUILDABLE_SERVICES, *DASHBOARD_CDC_BUILDABLE_SERVICES]
|
||
if frontend_release_enabled():
|
||
names.extend(service_name for service_name in FRONTEND_BUILDABLE_SERVICES if str(service_name or "").strip())
|
||
return names
|
||
|
||
|
||
def normalize_service_names(service_names: list[str]) -> list[str]:
|
||
# 请求里可能重复勾选同一个服务，这里先去重并保序。
|
||
normalized = [str(name or "").strip() for name in service_names if str(name or "").strip()]
|
||
return list(dict.fromkeys(normalized))
|
||
|
||
|
||
def service_kind(service_name: str) -> str:
|
||
# 更新链路目前分 Java / Golang / Frontend 三类仓库。
|
||
if service_name in JAVA_BUILDABLE_SERVICES:
|
||
return "java"
|
||
if service_name in GOLANG_BUILDABLE_SERVICES:
|
||
return "golang"
|
||
if service_name in DASHBOARD_CDC_BUILDABLE_SERVICES:
|
||
return "golang"
|
||
if frontend_release_enabled() and service_name in FRONTEND_BUILDABLE_SERVICES:
|
||
return "frontend"
|
||
raise RuntimeError(f"unsupported update service: {service_name}")
|
||
|
||
|
||
def ensure_repo_root(repo_root: Path, label: str) -> Path:
|
||
# 构建前先确认业务仓真的存在，避免跑到一半才发现路径错了。
|
||
resolved = Path(repo_root).expanduser().resolve()
|
||
if not resolved.exists():
|
||
raise RuntimeError(f"{label} repo root missing: {resolved}")
|
||
if not (resolved / ".git").exists():
|
||
raise RuntimeError(f"{label} repo root is not a git repo: {resolved}")
|
||
return resolved
|
||
|
||
|
||
def ensure_frontend_repo_root(*, progress: ProgressLogger | None = None) -> Path:
|
||
# Admin 前端允许第一次使用时按配置的 repo URL 自动 clone 到 deploy 机。
|
||
repo_root = Path(ADMIN_FRONTEND_REPO_ROOT).expanduser().resolve()
|
||
if (repo_root / ".git").exists():
|
||
return repo_root
|
||
|
||
repo_url = str(ADMIN_FRONTEND_REPO_URL or "").strip()
|
||
if not repo_url:
|
||
raise RuntimeError("missing ADMIN_FRONTEND_REPO_URL in .env")
|
||
|
||
repo_root.parent.mkdir(parents=True, exist_ok=True)
|
||
emit_progress(progress, "build.frontend.repo", f"初始化前端仓库：{repo_url} -> {repo_root}")
|
||
local_command_output(
|
||
["git", "clone", repo_url, str(repo_root)],
|
||
cwd=repo_root.parent,
|
||
label="frontend clone repo",
|
||
timeout_seconds=900,
|
||
)
|
||
return ensure_repo_root(repo_root, "frontend")
|
||
|
||
|
||
def ensure_dashboard_cdc_repo_root(*, progress: ProgressLogger | None = None) -> Path:
|
||
# Dashboard CDC Worker 是新独立仓库，deploy 机第一次使用时自动 clone。
|
||
repo_root = Path(DASHBOARD_CDC_REPO_ROOT).expanduser().resolve()
|
||
if (repo_root / ".git").exists():
|
||
return repo_root
|
||
|
||
repo_url = str(DASHBOARD_CDC_REPO_URL or "").strip()
|
||
if not repo_url:
|
||
raise RuntimeError("missing DASHBOARD_CDC_REPO_URL in .env")
|
||
|
||
repo_root.parent.mkdir(parents=True, exist_ok=True)
|
||
emit_progress(progress, "build.dashboard-cdc.repo", f"初始化 Dashboard CDC 仓库：{repo_url} -> {repo_root}")
|
||
local_command_output(
|
||
["git", "clone", repo_url, str(repo_root)],
|
||
cwd=repo_root.parent,
|
||
label="dashboard cdc clone repo",
|
||
timeout_seconds=900,
|
||
)
|
||
return ensure_repo_root(repo_root, "dashboard-cdc")
|
||
|
||
|
||
def local_command_output(
|
||
command: list[str],
|
||
*,
|
||
cwd: Path,
|
||
label: str,
|
||
env: dict[str, str] | None = None,
|
||
input_text: str = "",
|
||
timeout_seconds: int = 120,
|
||
) -> str:
|
||
# 小输出命令直接走 capture_output，适合 git rev-parse / docker inspect 这类场景。
|
||
merged_env = os.environ.copy()
|
||
if env:
|
||
merged_env.update(env)
|
||
|
||
result = subprocess.run(
|
||
command,
|
||
cwd=str(cwd),
|
||
text=True,
|
||
input=input_text if input_text else None,
|
||
capture_output=True,
|
||
env=merged_env,
|
||
timeout=timeout_seconds,
|
||
check=False,
|
||
)
|
||
if result.returncode != 0:
|
||
detail = (result.stderr or result.stdout or "").strip()
|
||
raise RuntimeError(f"{label} failed: {detail or f'exit {result.returncode}'}")
|
||
return (result.stdout or "").strip()
|
||
|
||
|
||
def tail_text(path: Path, max_bytes: int = 8192) -> str:
|
||
# 大构建日志只回最后一小段，避免把整份 Maven 输出塞进响应体。
|
||
if not path.exists():
|
||
return ""
|
||
|
||
with path.open("rb") as handle:
|
||
handle.seek(0, os.SEEK_END)
|
||
size = handle.tell()
|
||
handle.seek(max(0, size - max_bytes))
|
||
return handle.read().decode("utf-8", errors="replace").strip()
|
||
|
||
|
||
def frontend_release_enabled() -> bool:
|
||
# Admin 前端需要仓库、主机和发布目录都齐备后才开放到页面。
|
||
return bool(
|
||
ADMIN_FRONTEND_ENABLED
|
||
and str(ADMIN_FRONTEND_SERVICE_NAME or "").strip()
|
||
and str(ADMIN_FRONTEND_HOST or "").strip()
|
||
and str(ADMIN_FRONTEND_SITE_DIR or "").strip()
|
||
and (str(ADMIN_FRONTEND_SSH_HOST or "").strip() or str(ADMIN_FRONTEND_IP or "").strip())
|
||
and (str(ADMIN_FRONTEND_REPO_URL or "").strip() or str(ADMIN_FRONTEND_REPO_ROOT or "").strip())
|
||
)
|
||
|
||
|
||
def frontend_host_record(service_name: str) -> dict[str, Any]:
|
||
# 当前只有 admin 前端走这条链路，后续再扩展时继续在这里分流。
|
||
if not frontend_release_enabled() or service_name != ADMIN_FRONTEND_SERVICE_NAME:
|
||
raise RuntimeError(f"unsupported frontend service: {service_name}")
|
||
|
||
return {
|
||
"host": ADMIN_FRONTEND_HOST,
|
||
"ip": ADMIN_FRONTEND_IP,
|
||
"instanceId": ADMIN_FRONTEND_INSTANCE_ID,
|
||
"sshHost": ADMIN_FRONTEND_SSH_HOST or ADMIN_FRONTEND_IP,
|
||
"sshUser": ADMIN_FRONTEND_SSH_USER,
|
||
"sshUseSudo": ADMIN_FRONTEND_SSH_USE_SUDO,
|
||
"sshPort": ADMIN_FRONTEND_SSH_PORT,
|
||
}
|
||
|
||
|
||
def run_logged_command(
|
||
command: list[str],
|
||
*,
|
||
cwd: Path,
|
||
label: str,
|
||
env: dict[str, str] | None = None,
|
||
timeout_seconds: int = SERVICE_BUILD_TIMEOUT_SECONDS,
|
||
progress_stage: str = "",
|
||
progress: ProgressLogger | None = None,
|
||
) -> str:
|
||
# Maven / docker build 输出很大，单独落临时日志再取尾部更稳。
|
||
merged_env = os.environ.copy()
|
||
if env:
|
||
merged_env.update(env)
|
||
|
||
log_file = tempfile.NamedTemporaryFile(prefix="hy-app-monitor-build-", suffix=".log", delete=False)
|
||
log_path = Path(log_file.name)
|
||
log_file.close()
|
||
|
||
try:
|
||
stop_event = threading.Event()
|
||
follower: threading.Thread | None = None
|
||
|
||
with log_path.open("w", encoding="utf-8") as handle:
|
||
process = subprocess.Popen(
|
||
command,
|
||
cwd=str(cwd),
|
||
stdout=handle,
|
||
stderr=subprocess.STDOUT,
|
||
text=True,
|
||
env=merged_env,
|
||
)
|
||
if progress is not None:
|
||
follower = threading.Thread(
|
||
target=stream_log_file_lines,
|
||
args=(log_path, stop_event),
|
||
kwargs={"stage": progress_stage or label, "progress": progress},
|
||
daemon=True,
|
||
)
|
||
follower.start()
|
||
try:
|
||
return_code = process.wait(timeout=timeout_seconds)
|
||
except subprocess.TimeoutExpired as exc:
|
||
process.kill()
|
||
process.wait()
|
||
stop_event.set()
|
||
if follower is not None:
|
||
follower.join(timeout=1)
|
||
raise RuntimeError(f"{label} timed out: {tail_text(log_path)}") from exc
|
||
finally:
|
||
stop_event.set()
|
||
if follower is not None:
|
||
follower.join(timeout=1)
|
||
|
||
output_tail = tail_text(log_path)
|
||
if return_code != 0:
|
||
raise RuntimeError(f"{label} failed: {output_tail or f'exit {return_code}'}")
|
||
return output_tail
|
||
finally:
|
||
log_path.unlink(missing_ok=True)
|
||
|
||
|
||
def fetch_repo_refs(repo_root: Path, label: str) -> None:
|
||
# 统一先刷新远端分支和 tag，后面的 ref 解析才可信。
|
||
local_command_output(
|
||
["git", "-C", str(repo_root), "fetch", "--all", "--tags", "--prune"],
|
||
cwd=repo_root,
|
||
label=f"{label} fetch refs",
|
||
timeout_seconds=300,
|
||
)
|
||
|
||
|
||
def local_maven_repo_root() -> Path:
|
||
# 兼容外部显式指定 MAVEN_REPO_LOCAL，默认仍沿用 ~/.m2/repository。
|
||
override = str(os.environ.get("MAVEN_REPO_LOCAL") or "").strip()
|
||
if override:
|
||
return Path(override).expanduser()
|
||
return Path.home() / ".m2" / "repository"
|
||
|
||
|
||
def private_maven_tree_id(repo_root: Path, commit: str) -> str:
|
||
# 用 git tree id 表示 maven_private 内容，只有私有依赖目录变化时才会变化。
|
||
try:
|
||
return local_command_output(
|
||
["git", "-C", str(repo_root), "rev-parse", "--verify", f"{commit}:maven_private"],
|
||
cwd=repo_root,
|
||
label="resolve private maven tree",
|
||
)
|
||
except Exception: # noqa: BLE001
|
||
return ""
|
||
|
||
|
||
def private_maven_cache_stamp_path(repo_root: Path) -> Path:
|
||
# 每个仓保留一份最近已安装的私有依赖 tree id。
|
||
return PRIVATE_MAVEN_CACHE_DIR / f"{repo_root.name}.tree"
|
||
|
||
|
||
def private_maven_artifacts_present() -> bool:
|
||
# 本地 Maven 仓至少得有一份私有制品，缓存命中才有意义。
|
||
artifacts_root = local_maven_repo_root() / "com" / "red" / "circle"
|
||
if not artifacts_root.exists():
|
||
return False
|
||
for path in artifacts_root.rglob("*.pom"):
|
||
if path.is_file():
|
||
return True
|
||
return False
|
||
|
||
|
||
def ensure_private_maven_installed(
|
||
repo_root: Path,
|
||
worktree_root: Path,
|
||
commit: str,
|
||
*,
|
||
progress: ProgressLogger | None = None,
|
||
) -> None:
|
||
# 私有依赖没变化时直接复用本机 ~/.m2 缓存，避免每次更新都重复 install-file。
|
||
tree_id = private_maven_tree_id(repo_root, commit)
|
||
stamp_path = private_maven_cache_stamp_path(repo_root)
|
||
cached_tree_id = stamp_path.read_text(encoding="utf-8").strip() if stamp_path.exists() else ""
|
||
|
||
if tree_id and cached_tree_id == tree_id and private_maven_artifacts_present():
|
||
emit_progress(progress, "build.java.maven", f"私有 Maven 依赖未变化，跳过安装：{tree_id[:12]}")
|
||
return
|
||
|
||
emit_progress(progress, "build.java.maven", "开始安装私有 Maven 依赖")
|
||
run_logged_command(
|
||
[str(worktree_root / "ci" / "install-private-maven.sh")],
|
||
cwd=worktree_root,
|
||
label="java install private maven",
|
||
timeout_seconds=min(SERVICE_BUILD_TIMEOUT_SECONDS, 1200),
|
||
progress_stage="build.java.maven.install",
|
||
progress=progress,
|
||
)
|
||
|
||
if tree_id:
|
||
PRIVATE_MAVEN_CACHE_DIR.mkdir(parents=True, exist_ok=True)
|
||
stamp_path.write_text(f"{tree_id}\n", encoding="utf-8")
|
||
emit_progress(progress, "build.java.maven", f"私有 Maven 依赖安装完成：{tree_id[:12]}")
|
||
return
|
||
|
||
emit_progress(progress, "build.java.maven", "私有 Maven 依赖安装完成")
|
||
|
||
|
||
def resolve_repo_commit(repo_root: Path, git_ref: str, label: str) -> str:
|
||
# 优先把分支解析到 origin/<branch>，避免本地旧分支 HEAD 干扰构建。
|
||
candidates = [
|
||
f"refs/remotes/origin/{git_ref}^{{commit}}",
|
||
f"refs/tags/{git_ref}^{{commit}}",
|
||
f"{git_ref}^{{commit}}",
|
||
]
|
||
for candidate in candidates:
|
||
try:
|
||
return local_command_output(
|
||
["git", "-C", str(repo_root), "rev-parse", "--verify", candidate],
|
||
cwd=repo_root,
|
||
label=f"{label} resolve ref",
|
||
)
|
||
except Exception: # noqa: BLE001
|
||
continue
|
||
raise RuntimeError(f"{label} git ref not found: {git_ref}")
|
||
|
||
|
||
def build_cache_root(repo_root: Path) -> Path:
|
||
# 显式配置优先；未配置时按本地 / deploy 机目录结构推导默认缓存根目录。
|
||
override = str(SERVICE_BUILD_CACHE_ROOT or "").strip()
|
||
if override:
|
||
return Path(override).expanduser()
|
||
if str(repo_root).startswith("/opt/deploy/"):
|
||
return Path("/opt/deploy/build-cache")
|
||
return ROOT / "run" / "build_cache"
|
||
|
||
|
||
def repo_origin_url(repo_root: Path, label: str) -> str:
|
||
# 固定构建工作区优先直接连 origin，避免本地源码仓 owner 不一致触发 safe.directory。
|
||
try:
|
||
return local_command_output(
|
||
["git", "-C", str(repo_root), "remote", "get-url", "origin"],
|
||
cwd=repo_root,
|
||
label=f"{label} get origin url",
|
||
timeout_seconds=120,
|
||
)
|
||
except Exception: # noqa: BLE001
|
||
return str(repo_root)
|
||
|
||
|
||
def persistent_build_workspace_path(repo_root: Path, git_ref: str) -> Path:
|
||
# 同一仓库同一 ref 固定复用一份构建目录，最大化保留 target 缓存。
|
||
return build_cache_root(repo_root) / repo_root.name / sanitized_ref_fragment(git_ref)
|
||
|
||
|
||
def prepare_persistent_build_workspace(
|
||
repo_root: Path,
|
||
git_ref: str,
|
||
commit: str,
|
||
label: str,
|
||
*,
|
||
progress: ProgressLogger | None = None,
|
||
) -> Path:
|
||
# 固定构建工作区是专用缓存目录，允许 reset --hard 到目标提交。
|
||
workspace_root = persistent_build_workspace_path(repo_root, git_ref)
|
||
workspace_root.parent.mkdir(parents=True, exist_ok=True)
|
||
origin_url = repo_origin_url(repo_root, label)
|
||
|
||
if not (workspace_root / ".git").exists():
|
||
if workspace_root.exists():
|
||
shutil.rmtree(workspace_root, ignore_errors=True)
|
||
local_command_output(
|
||
["git", "clone", "--no-checkout", origin_url, str(workspace_root)],
|
||
cwd=workspace_root.parent,
|
||
label=f"{label} clone build workspace",
|
||
timeout_seconds=600,
|
||
)
|
||
|
||
local_command_output(
|
||
["git", "-C", str(workspace_root), "remote", "set-url", "origin", origin_url],
|
||
cwd=workspace_root,
|
||
label=f"{label} set build workspace origin",
|
||
timeout_seconds=120,
|
||
)
|
||
local_command_output(
|
||
["git", "-C", str(workspace_root), "fetch", "origin", "--tags", "--prune"],
|
||
cwd=workspace_root,
|
||
label=f"{label} fetch build workspace",
|
||
timeout_seconds=300,
|
||
)
|
||
local_command_output(
|
||
["git", "-C", str(workspace_root), "fetch", "origin", commit],
|
||
cwd=workspace_root,
|
||
label=f"{label} fetch build workspace commit",
|
||
timeout_seconds=300,
|
||
)
|
||
local_command_output(
|
||
["git", "-C", str(workspace_root), "reset", "--hard", commit],
|
||
cwd=workspace_root,
|
||
label=f"{label} reset build workspace",
|
||
timeout_seconds=300,
|
||
)
|
||
emit_progress(progress, f"build.{label}", f"已准备固定构建工作区：{workspace_root} -> {commit[:12]}")
|
||
return workspace_root
|
||
|
||
|
||
def create_temp_worktree(repo_root: Path, commit: str, label: str) -> Path:
|
||
# 用临时 worktree 构建，避免直接改业务仓当前工作区。
|
||
worktree_root = Path(tempfile.mkdtemp(prefix=f"hy-app-monitor-{repo_root.name}-"))
|
||
try:
|
||
local_command_output(
|
||
["git", "-C", str(repo_root), "worktree", "add", "--detach", str(worktree_root), commit],
|
||
cwd=repo_root,
|
||
label=f"{label} create worktree",
|
||
timeout_seconds=300,
|
||
)
|
||
return worktree_root
|
||
except Exception:
|
||
shutil.rmtree(worktree_root, ignore_errors=True)
|
||
raise
|
||
|
||
|
||
def remove_temp_worktree(repo_root: Path, worktree_root: Path) -> None:
|
||
# 构建结束后清理 worktree，避免 deploy 机累积一堆临时目录。
|
||
try:
|
||
local_command_output(
|
||
["git", "-C", str(repo_root), "worktree", "remove", "--force", str(worktree_root)],
|
||
cwd=repo_root,
|
||
label="remove worktree",
|
||
timeout_seconds=120,
|
||
)
|
||
except Exception: # noqa: BLE001
|
||
shutil.rmtree(worktree_root, ignore_errors=True)
|
||
|
||
|
||
def require_harbor_credentials() -> None:
|
||
# 构建推送和目标机预拉都依赖 Harbor 账号。
|
||
if not HARBOR_REGISTRY:
|
||
raise RuntimeError("missing HARBOR_REGISTRY in .env")
|
||
if not HARBOR_USERNAME or not HARBOR_PASSWORD:
|
||
raise RuntimeError("missing HARBOR_USERNAME / HARBOR_PASSWORD in .env")
|
||
|
||
|
||
def docker_login_local(repo_root: Path, *, progress: ProgressLogger | None = None) -> None:
|
||
# 本机构建前先登录 Harbor，避免 push 时才失败。
|
||
require_harbor_credentials()
|
||
emit_progress(progress, "prepare", f"登录 Harbor：{HARBOR_REGISTRY}")
|
||
local_command_output(
|
||
["docker", "login", HARBOR_REGISTRY, "-u", HARBOR_USERNAME, "--password-stdin"],
|
||
cwd=repo_root,
|
||
label="docker login harbor",
|
||
input_text=HARBOR_PASSWORD,
|
||
timeout_seconds=120,
|
||
)
|
||
emit_progress(progress, "prepare", "Harbor 登录成功")
|
||
|
||
|
||
def image_repository_ref(image_ref: str) -> str:
|
||
# 把 tag / digest 去掉，保留完整仓库地址。
|
||
text = str(image_ref or "").strip()
|
||
if "@" in text:
|
||
return text.split("@", 1)[0]
|
||
|
||
last_slash = text.rfind("/")
|
||
last_colon = text.rfind(":")
|
||
if last_colon > last_slash:
|
||
return text[:last_colon]
|
||
return text
|
||
|
||
|
||
def sanitized_ref_fragment(git_ref: str) -> str:
|
||
# Harbor tag 只保留相对安全的字符。
|
||
cleaned = re.sub(r"[^0-9A-Za-z._-]+", "-", str(git_ref or "").strip()) or "manual"
|
||
return cleaned.strip("-") or "manual"
|
||
|
||
|
||
def derive_image_tag(git_ref: str, commit: str, image_tag: str) -> str:
|
||
# 用户没指定 tag 时，默认用 ref + commit12 + 时间戳。
|
||
explicit = str(image_tag or "").strip()
|
||
if explicit:
|
||
return explicit
|
||
timestamp = datetime.utcnow().strftime("%Y%m%d-%H%M%S")
|
||
return f"{sanitized_ref_fragment(git_ref)}-{commit[:12]}-{timestamp}"
|
||
|
||
|
||
def frontend_version_label(git_ref: str, commit: str) -> str:
|
||
# 前端发布结果统一显示成 ref@commit12，便于和镜像 tag 区分。
|
||
resolved_ref = str(git_ref or "").strip() or UPDATE_DEFAULT_ADMIN_GIT_REF
|
||
return f"{resolved_ref}@{commit[:12]}"
|
||
|
||
|
||
def frontend_release_metadata(service_name: str) -> dict[str, Any]:
|
||
# 当前前端站点的已部署版本由远端元数据文件提供；缺失时允许返回空结构。
|
||
host = frontend_host_record(service_name)
|
||
try:
|
||
raw = read_remote_text(host, ADMIN_FRONTEND_RELEASE_META_PATH, timeout_seconds=30).strip()
|
||
except Exception as exc: # noqa: BLE001
|
||
return {"error": str(exc)}
|
||
|
||
if not raw:
|
||
return {}
|
||
|
||
try:
|
||
payload = json.loads(raw)
|
||
except json.JSONDecodeError:
|
||
return {"error": f"invalid frontend release metadata on {target_display(host)}"}
|
||
|
||
if not isinstance(payload, dict):
|
||
return {"error": f"invalid frontend release metadata on {target_display(host)}"}
|
||
return payload
|
||
|
||
|
||
def image_release_display(current_images: list[str]) -> dict[str, str]:
|
||
# 镜像型服务在表格里固定展示 tag / digest / 可复制的主镜像。
|
||
primary_image = str((current_images or [""])[0] or "").strip()
|
||
if not primary_image:
|
||
return {
|
||
"currentLabel": "-",
|
||
"currentTitle": "-",
|
||
"detailLabel": "-",
|
||
"detailTitle": "-",
|
||
"copyValue": "",
|
||
}
|
||
|
||
repository = image_repository_ref(primary_image)
|
||
meta = {
|
||
"repository": repository,
|
||
"tag": "",
|
||
"digest": "",
|
||
}
|
||
without_digest, _, digest_text = primary_image.partition("@")
|
||
meta["digest"] = digest_text or ""
|
||
last_slash = without_digest.rfind("/")
|
||
last_colon = without_digest.rfind(":")
|
||
if last_colon > last_slash:
|
||
meta["tag"] = without_digest[last_colon + 1 :]
|
||
|
||
return {
|
||
"currentLabel": meta["tag"] or "-",
|
||
"currentTitle": "\n".join(current_images) if len(current_images) > 1 else primary_image,
|
||
"detailLabel": (meta["digest"][:12] + "...") if len(meta["digest"]) > 15 else (meta["digest"] or "-"),
|
||
"detailTitle": primary_image,
|
||
"copyValue": primary_image,
|
||
}
|
||
|
||
|
||
def frontend_release_display(service_name: str) -> dict[str, str]:
|
||
# 前端目标优先显示最近一次由监控面板写入的 gitRef / commit。
|
||
metadata = frontend_release_metadata(service_name)
|
||
version = str(metadata.get("version") or "").strip()
|
||
commit = str(metadata.get("commit") or "").strip()
|
||
git_ref = str(metadata.get("gitRef") or "").strip()
|
||
deployed_at = str(metadata.get("deployedAt") or "").strip()
|
||
error = str(metadata.get("error") or "").strip()
|
||
|
||
if version or commit or git_ref:
|
||
label = version or git_ref or commit[:12]
|
||
title_parts = [part for part in [version, git_ref, commit, deployed_at, ADMIN_FRONTEND_PUBLIC_URL] if part]
|
||
return {
|
||
"currentLabel": label,
|
||
"currentTitle": " | ".join(title_parts) if title_parts else label,
|
||
"detailLabel": commit[:12] if commit else "-",
|
||
"detailTitle": commit or deployed_at or label,
|
||
"copyValue": commit or git_ref,
|
||
}
|
||
|
||
if error:
|
||
return {
|
||
"currentLabel": "未记录",
|
||
"currentTitle": error,
|
||
"detailLabel": "SSH",
|
||
"detailTitle": error,
|
||
"copyValue": "",
|
||
}
|
||
|
||
return {
|
||
"currentLabel": "未记录",
|
||
"currentTitle": "当前站点还没有由 hy-app-monitor 写入过版本元数据",
|
||
"detailLabel": "-",
|
||
"detailTitle": "-",
|
||
"copyValue": "",
|
||
}
|
||
|
||
|
||
def docker_digest(image_ref: str, *, cwd: Path) -> str:
|
||
# push 完要拿 digest，后续 compose 一律按 digest 部署。
|
||
return local_command_output(
|
||
["docker", "image", "inspect", image_ref, "--format", "{{index .RepoDigests 0}}"],
|
||
cwd=cwd,
|
||
label=f"inspect digest {image_ref}",
|
||
timeout_seconds=120,
|
||
)
|
||
|
||
|
||
def current_template_image(service_name: str) -> str:
|
||
# 优先从本地完整模板里拿当前运行时镜像仓路径。
|
||
records = service_records_by_name().get(service_name) or []
|
||
for record in records:
|
||
try:
|
||
model = load_local_host_compose_model(record["host"])
|
||
except Exception: # noqa: BLE001
|
||
# 发布页只需要尽量推导仓路径；模板暂时不可读时继续尝试其他主机并允许后续兜底。
|
||
continue
|
||
image_ref = str(((model.get("services") or {}).get(service_name) or {}).get("image") or "").strip()
|
||
if image_ref.startswith(f"{HARBOR_REGISTRY}/{HARBOR_PROJECT}/"):
|
||
return image_ref
|
||
return ""
|
||
|
||
|
||
def service_repository_ref(service_name: str) -> str:
|
||
# 线上已有镜像时沿用现有仓路径；没有时再走保守兜底。
|
||
current_image = current_template_image(service_name)
|
||
if current_image:
|
||
return image_repository_ref(current_image)
|
||
|
||
suffix = SERVICE_IMAGE_REPOSITORY_SUFFIXES.get(service_name, "").strip()
|
||
if not suffix:
|
||
raise RuntimeError(f"missing Harbor repository mapping for service: {service_name}")
|
||
return f"{HARBOR_REGISTRY}/{HARBOR_PROJECT}/{suffix}"
|
||
|
||
|
||
def maven_module_list(service_names: list[str]) -> str:
|
||
# Java 多服务更新时，先一次性 Maven 打包对应模块，避免重复 clean package。
|
||
modules: list[str] = []
|
||
for service_name in service_names:
|
||
module = JAVA_SERVICE_MODULES.get(service_name)
|
||
if not module:
|
||
raise RuntimeError(f"missing java maven module mapping: {service_name}")
|
||
if module not in modules:
|
||
modules.append(module)
|
||
return ",".join(modules)
|
||
|
||
|
||
def java_maven_package_command(service_names: list[str], *, clean: bool = False) -> list[str]:
|
||
# Java 默认走增量 package，并允许通过 .env 配 Maven 并行线程。
|
||
command = ["mvn"]
|
||
maven_threads = str(JAVA_MAVEN_THREADS or "").strip()
|
||
if maven_threads:
|
||
command.extend(["-T", maven_threads])
|
||
command.extend(
|
||
[
|
||
"-pl",
|
||
maven_module_list(service_names),
|
||
"-am",
|
||
"-Dmaven.test.skip=true",
|
||
]
|
||
)
|
||
if clean:
|
||
command.append("clean")
|
||
command.append("package")
|
||
return command
|
||
|
||
|
||
def should_retry_java_maven_with_clean(error_text: str) -> bool:
|
||
# 固定工作区第一次启用时，历史 target 产物偶尔会有损坏；命中后退回一次 clean package 自愈。
|
||
normalized = str(error_text or "")
|
||
markers = (
|
||
"java.util.zip.ZipException",
|
||
"java.io.EOFException",
|
||
"bad class file",
|
||
"invalid loc",
|
||
"invalid block type",
|
||
"Unexpected end of ZLIB input stream",
|
||
)
|
||
return any(marker in normalized for marker in markers)
|
||
|
||
|
||
def build_single_java_image(
|
||
service_name: str,
|
||
*,
|
||
worktree_root: Path,
|
||
short_commit: str,
|
||
resolved_image_tag: str,
|
||
progress: ProgressLogger | None = None,
|
||
) -> dict[str, Any]:
|
||
# Maven 产物已经在固定工作区准备好，这里只负责单服务镜像构建和推送。
|
||
repository_ref = service_repository_ref(service_name)
|
||
target_image_ref = f"{repository_ref}:{resolved_image_tag}"
|
||
emit_progress(progress, f"build.java.{service_name}", f"开始构建镜像：{target_image_ref}")
|
||
build_tail = run_logged_command(
|
||
[str(worktree_root / "ci" / "build-service-image.sh"), service_name, target_image_ref, short_commit],
|
||
cwd=worktree_root,
|
||
label=f"java build {service_name}",
|
||
env={
|
||
"DOCKER_PLATFORM": BUILD_DOCKER_PLATFORM,
|
||
# 外层已经统一决定过是否 Maven 打包；镜像阶段一律复用现成产物。
|
||
"SKIP_MAVEN_BUILD": "1",
|
||
},
|
||
timeout_seconds=SERVICE_BUILD_TIMEOUT_SECONDS,
|
||
progress_stage=f"build.java.{service_name}.build",
|
||
progress=progress,
|
||
)
|
||
emit_progress(progress, f"build.java.{service_name}", f"开始推送 Harbor：{target_image_ref}")
|
||
push_tail = run_logged_command(
|
||
["docker", "push", target_image_ref],
|
||
cwd=worktree_root,
|
||
label=f"java push {service_name}",
|
||
timeout_seconds=SERVICE_BUILD_TIMEOUT_SECONDS,
|
||
progress_stage=f"build.java.{service_name}.push",
|
||
progress=progress,
|
||
)
|
||
digest_ref = docker_digest(target_image_ref, cwd=worktree_root)
|
||
emit_progress(progress, f"build.java.{service_name}", f"镜像已推送完成：{digest_ref}")
|
||
return {
|
||
"service": service_name,
|
||
"repository": repository_ref,
|
||
"tag": target_image_ref,
|
||
"image": digest_ref,
|
||
"buildTail": build_tail,
|
||
"pushTail": push_tail,
|
||
}
|
||
|
||
|
||
def build_java_images(
|
||
service_names: list[str],
|
||
git_ref: str,
|
||
image_tag: str,
|
||
*,
|
||
skip_maven_build: bool,
|
||
progress: ProgressLogger | None = None,
|
||
) -> dict[str, Any]:
|
||
# Java 服务共享一个仓库和 Maven 本地缓存，这里按仓统一构建。
|
||
repo_root = ensure_repo_root(JAVA_REPO_ROOT, "java")
|
||
resolved_ref = str(git_ref or "").strip() or UPDATE_DEFAULT_JAVA_GIT_REF
|
||
emit_progress(progress, "build.java", f"刷新 Java 仓库 refs：{resolved_ref}")
|
||
fetch_repo_refs(repo_root, "java")
|
||
commit = resolve_repo_commit(repo_root, resolved_ref, "java")
|
||
emit_progress(progress, "build.java", f"Java ref 已解析：{resolved_ref} -> {commit[:12]}")
|
||
worktree_root = prepare_persistent_build_workspace(
|
||
repo_root,
|
||
resolved_ref,
|
||
commit,
|
||
"java",
|
||
progress=progress,
|
||
)
|
||
short_commit = commit[:12]
|
||
resolved_image_tag = derive_image_tag(resolved_ref, commit, image_tag)
|
||
|
||
outputs: list[dict[str, Any]] = []
|
||
service_images: dict[str, str] = {}
|
||
|
||
try:
|
||
if not skip_maven_build:
|
||
ensure_private_maven_installed(
|
||
repo_root,
|
||
worktree_root,
|
||
commit,
|
||
progress=progress,
|
||
)
|
||
emit_progress(progress, "build.java.maven", f"开始 Maven 打包：{maven_module_list(service_names)}")
|
||
try:
|
||
run_logged_command(
|
||
java_maven_package_command(service_names),
|
||
cwd=worktree_root,
|
||
label="java maven package",
|
||
timeout_seconds=SERVICE_BUILD_TIMEOUT_SECONDS,
|
||
progress_stage="build.java.maven.package",
|
||
progress=progress,
|
||
)
|
||
except Exception as exc: # noqa: BLE001
|
||
if not should_retry_java_maven_with_clean(str(exc)):
|
||
raise
|
||
emit_progress(
|
||
progress,
|
||
"build.java.maven",
|
||
"检测到缓存产物损坏，退回一次 clean package 自愈",
|
||
level="warning",
|
||
)
|
||
run_logged_command(
|
||
java_maven_package_command(service_names, clean=True),
|
||
cwd=worktree_root,
|
||
label="java maven clean package",
|
||
timeout_seconds=SERVICE_BUILD_TIMEOUT_SECONDS,
|
||
progress_stage="build.java.maven.clean-package",
|
||
progress=progress,
|
||
)
|
||
else:
|
||
emit_progress(progress, "build.java.maven", "跳过 Maven，直接复用已有 Jar")
|
||
|
||
max_workers = max(1, min(JAVA_IMAGE_BUILD_MAX_WORKERS, len(service_names)))
|
||
emit_progress(progress, "build.java", f"Java 镜像并行度：{max_workers}")
|
||
if max_workers == 1:
|
||
for service_name in service_names:
|
||
item = build_single_java_image(
|
||
service_name,
|
||
worktree_root=worktree_root,
|
||
short_commit=short_commit,
|
||
resolved_image_tag=resolved_image_tag,
|
||
progress=progress,
|
||
)
|
||
service_images[service_name] = item["image"]
|
||
outputs.append(item)
|
||
else:
|
||
items_by_service: dict[str, dict[str, Any]] = {}
|
||
errors: list[str] = []
|
||
with ThreadPoolExecutor(max_workers=max_workers, thread_name_prefix="java-image-build") as executor:
|
||
future_map = {
|
||
executor.submit(
|
||
build_single_java_image,
|
||
service_name,
|
||
worktree_root=worktree_root,
|
||
short_commit=short_commit,
|
||
resolved_image_tag=resolved_image_tag,
|
||
progress=progress,
|
||
): service_name
|
||
for service_name in service_names
|
||
}
|
||
for future in as_completed(future_map):
|
||
service_name = future_map[future]
|
||
try:
|
||
item = future.result()
|
||
except Exception as exc: # noqa: BLE001
|
||
errors.append(f"{service_name}: {exc}")
|
||
emit_progress(progress, f"build.java.{service_name}", str(exc), level="error")
|
||
continue
|
||
items_by_service[service_name] = item
|
||
service_images[service_name] = item["image"]
|
||
|
||
if errors:
|
||
raise RuntimeError("java image build failed: " + "; ".join(errors))
|
||
|
||
outputs = [items_by_service[service_name] for service_name in service_names]
|
||
finally:
|
||
emit_progress(progress, "build.java", f"固定构建工作区保留复用：{worktree_root}")
|
||
|
||
return {
|
||
"kind": "java",
|
||
"repoRoot": str(repo_root),
|
||
"gitRef": resolved_ref,
|
||
"commit": commit,
|
||
"imageTag": resolved_image_tag,
|
||
"services": list(service_names),
|
||
"items": outputs,
|
||
"serviceImages": service_images,
|
||
}
|
||
|
||
|
||
def build_golang_images(
|
||
service_names: list[str],
|
||
git_ref: str,
|
||
image_tag: str,
|
||
*,
|
||
progress: ProgressLogger | None = None,
|
||
) -> dict[str, Any]:
|
||
# Go 服务当前是单仓构建，逻辑更简单。
|
||
repo_root = ensure_repo_root(GOLANG_REPO_ROOT, "golang")
|
||
resolved_ref = str(git_ref or "").strip() or UPDATE_DEFAULT_GO_GIT_REF
|
||
emit_progress(progress, "build.golang", f"刷新 Go 仓库 refs：{resolved_ref}")
|
||
fetch_repo_refs(repo_root, "golang")
|
||
commit = resolve_repo_commit(repo_root, resolved_ref, "golang")
|
||
emit_progress(progress, "build.golang", f"Go ref 已解析：{resolved_ref} -> {commit[:12]}")
|
||
worktree_root = create_temp_worktree(repo_root, commit, "golang")
|
||
emit_progress(progress, "build.golang", f"已创建 Go 临时 worktree：{worktree_root}")
|
||
short_commit = commit[:12]
|
||
resolved_image_tag = derive_image_tag(resolved_ref, commit, image_tag)
|
||
|
||
outputs: list[dict[str, Any]] = []
|
||
service_images: dict[str, str] = {}
|
||
|
||
try:
|
||
for service_name in service_names:
|
||
repository_ref = service_repository_ref(service_name)
|
||
target_image_ref = f"{repository_ref}:{resolved_image_tag}"
|
||
emit_progress(progress, f"build.golang.{service_name}", f"开始构建镜像：{target_image_ref}")
|
||
build_tail = run_logged_command(
|
||
[str(worktree_root / "ci" / "build-image.sh"), target_image_ref, short_commit],
|
||
cwd=worktree_root,
|
||
label=f"golang build {service_name}",
|
||
env={"DOCKER_PLATFORM": BUILD_DOCKER_PLATFORM},
|
||
timeout_seconds=SERVICE_BUILD_TIMEOUT_SECONDS,
|
||
progress_stage=f"build.golang.{service_name}.build",
|
||
progress=progress,
|
||
)
|
||
emit_progress(progress, f"build.golang.{service_name}", f"开始推送 Harbor：{target_image_ref}")
|
||
push_tail = run_logged_command(
|
||
["docker", "push", target_image_ref],
|
||
cwd=worktree_root,
|
||
label=f"golang push {service_name}",
|
||
timeout_seconds=SERVICE_BUILD_TIMEOUT_SECONDS,
|
||
progress_stage=f"build.golang.{service_name}.push",
|
||
progress=progress,
|
||
)
|
||
digest_ref = docker_digest(target_image_ref, cwd=worktree_root)
|
||
emit_progress(progress, f"build.golang.{service_name}", f"镜像已推送完成：{digest_ref}")
|
||
service_images[service_name] = digest_ref
|
||
outputs.append(
|
||
{
|
||
"service": service_name,
|
||
"repository": repository_ref,
|
||
"tag": target_image_ref,
|
||
"image": digest_ref,
|
||
"buildTail": build_tail,
|
||
"pushTail": push_tail,
|
||
}
|
||
)
|
||
finally:
|
||
remove_temp_worktree(repo_root, worktree_root)
|
||
emit_progress(progress, "build.golang", f"已清理 Go worktree：{worktree_root}")
|
||
|
||
return {
|
||
"kind": "golang",
|
||
"repoRoot": str(repo_root),
|
||
"gitRef": resolved_ref,
|
||
"commit": commit,
|
||
"imageTag": resolved_image_tag,
|
||
"services": list(service_names),
|
||
"items": outputs,
|
||
"serviceImages": service_images,
|
||
}
|
||
|
||
|
||
def build_dashboard_cdc_images(
|
||
service_names: list[str],
|
||
git_ref: str,
|
||
image_tag: str,
|
||
*,
|
||
progress: ProgressLogger | None = None,
|
||
) -> dict[str, Any]:
|
||
# Dashboard CDC Worker 使用独立 Go 仓库和同名 ci/build-image.sh。
|
||
repo_root = ensure_dashboard_cdc_repo_root(progress=progress)
|
||
resolved_ref = str(git_ref or "").strip() or UPDATE_DEFAULT_GO_GIT_REF
|
||
emit_progress(progress, "build.dashboard-cdc", f"刷新 Dashboard CDC 仓库 refs：{resolved_ref}")
|
||
fetch_repo_refs(repo_root, "dashboard-cdc")
|
||
commit = resolve_repo_commit(repo_root, resolved_ref, "dashboard-cdc")
|
||
emit_progress(progress, "build.dashboard-cdc", f"Dashboard CDC ref 已解析：{resolved_ref} -> {commit[:12]}")
|
||
worktree_root = create_temp_worktree(repo_root, commit, "dashboard-cdc")
|
||
emit_progress(progress, "build.dashboard-cdc", f"已创建 Dashboard CDC 临时 worktree：{worktree_root}")
|
||
short_commit = commit[:12]
|
||
resolved_image_tag = derive_image_tag(resolved_ref, commit, image_tag)
|
||
|
||
outputs: list[dict[str, Any]] = []
|
||
service_images: dict[str, str] = {}
|
||
try:
|
||
for service_name in service_names:
|
||
repository_ref = service_repository_ref(service_name)
|
||
target_image_ref = f"{repository_ref}:{resolved_image_tag}"
|
||
emit_progress(progress, f"build.dashboard-cdc.{service_name}", f"开始构建镜像：{target_image_ref}")
|
||
build_tail = run_logged_command(
|
||
[str(worktree_root / "ci" / "build-image.sh"), target_image_ref, short_commit],
|
||
cwd=worktree_root,
|
||
label=f"dashboard cdc build {service_name}",
|
||
env={"DOCKER_PLATFORM": BUILD_DOCKER_PLATFORM},
|
||
timeout_seconds=SERVICE_BUILD_TIMEOUT_SECONDS,
|
||
progress_stage=f"build.dashboard-cdc.{service_name}.build",
|
||
progress=progress,
|
||
)
|
||
emit_progress(progress, f"build.dashboard-cdc.{service_name}", f"开始推送 Harbor：{target_image_ref}")
|
||
push_tail = run_logged_command(
|
||
["docker", "push", target_image_ref],
|
||
cwd=worktree_root,
|
||
label=f"dashboard cdc push {service_name}",
|
||
timeout_seconds=SERVICE_BUILD_TIMEOUT_SECONDS,
|
||
progress_stage=f"build.dashboard-cdc.{service_name}.push",
|
||
progress=progress,
|
||
)
|
||
digest_ref = docker_digest(target_image_ref, cwd=worktree_root)
|
||
emit_progress(progress, f"build.dashboard-cdc.{service_name}", f"镜像已推送完成：{digest_ref}")
|
||
service_images[service_name] = digest_ref
|
||
outputs.append(
|
||
{
|
||
"service": service_name,
|
||
"repository": repository_ref,
|
||
"tag": target_image_ref,
|
||
"image": digest_ref,
|
||
"buildTail": build_tail,
|
||
"pushTail": push_tail,
|
||
}
|
||
)
|
||
finally:
|
||
remove_temp_worktree(repo_root, worktree_root)
|
||
emit_progress(progress, "build.dashboard-cdc", f"已清理 Dashboard CDC worktree：{worktree_root}")
|
||
|
||
return {
|
||
"kind": "dashboard-cdc",
|
||
"repoRoot": str(repo_root),
|
||
"repoUrl": str(DASHBOARD_CDC_REPO_URL or "").strip(),
|
||
"gitRef": resolved_ref,
|
||
"commit": commit,
|
||
"imageTag": resolved_image_tag,
|
||
"services": list(service_names),
|
||
"items": outputs,
|
||
"serviceImages": service_images,
|
||
}
|
||
|
||
|
||
def build_frontend_bundle(
|
||
service_name: str,
|
||
git_ref: str,
|
||
*,
|
||
progress: ProgressLogger | None = None,
|
||
) -> dict[str, Any]:
|
||
# Admin 前端在 deploy 机做 pnpm install + build，最后产出静态 dist 目录。
|
||
repo_root = ensure_frontend_repo_root(progress=progress)
|
||
resolved_ref = str(git_ref or "").strip() or UPDATE_DEFAULT_ADMIN_GIT_REF
|
||
emit_progress(progress, "build.frontend", f"刷新前端仓库 refs：{resolved_ref}")
|
||
fetch_repo_refs(repo_root, "frontend")
|
||
commit = resolve_repo_commit(repo_root, resolved_ref, "frontend")
|
||
emit_progress(progress, "build.frontend", f"Frontend ref 已解析：{resolved_ref} -> {commit[:12]}")
|
||
worktree_root = prepare_persistent_build_workspace(
|
||
repo_root,
|
||
resolved_ref,
|
||
commit,
|
||
"frontend",
|
||
progress=progress,
|
||
)
|
||
|
||
shim_dir = worktree_root / ".hy-app-monitor-bin"
|
||
shim_dir.mkdir(parents=True, exist_ok=True)
|
||
pnpm_shim = shim_dir / "pnpm"
|
||
pnpm_shim.write_text("#!/bin/sh\nexec corepack pnpm \"$@\"\n", encoding="utf-8")
|
||
pnpm_shim.chmod(0o755)
|
||
|
||
frontend_env = {
|
||
"CI": "true",
|
||
"NODE_OPTIONS": "--max-old-space-size=8192",
|
||
"PATH": f"{shim_dir}:{os.environ.get('PATH', '')}",
|
||
}
|
||
emit_progress(progress, "build.frontend", f"已注入 pnpm shim：{pnpm_shim}")
|
||
emit_progress(progress, "build.frontend.install", "开始安装前端依赖")
|
||
run_logged_command(
|
||
["pnpm", "install", "--frozen-lockfile"],
|
||
cwd=worktree_root,
|
||
label="frontend pnpm install",
|
||
env=frontend_env,
|
||
timeout_seconds=SERVICE_BUILD_TIMEOUT_SECONDS,
|
||
progress_stage="build.frontend.install",
|
||
progress=progress,
|
||
)
|
||
emit_progress(progress, "build.frontend.build", f"开始构建静态站点：{ADMIN_FRONTEND_BUILD_SCRIPT}")
|
||
run_logged_command(
|
||
["pnpm", "run", ADMIN_FRONTEND_BUILD_SCRIPT],
|
||
cwd=worktree_root,
|
||
label="frontend pnpm build",
|
||
env=frontend_env,
|
||
timeout_seconds=SERVICE_BUILD_TIMEOUT_SECONDS,
|
||
progress_stage="build.frontend.build",
|
||
progress=progress,
|
||
)
|
||
|
||
artifact_root = (worktree_root / ADMIN_FRONTEND_DIST_DIR).resolve()
|
||
if not artifact_root.exists():
|
||
raise RuntimeError(f"frontend dist missing after build: {artifact_root}")
|
||
if not artifact_root.is_dir():
|
||
raise RuntimeError(f"frontend dist is not a directory: {artifact_root}")
|
||
|
||
version = frontend_version_label(resolved_ref, commit)
|
||
emit_progress(progress, "build.frontend", f"前端静态产物已生成：{artifact_root}")
|
||
return {
|
||
"kind": "frontend",
|
||
"repoRoot": str(repo_root),
|
||
"repoUrl": str(ADMIN_FRONTEND_REPO_URL or "").strip(),
|
||
"gitRef": resolved_ref,
|
||
"commit": commit,
|
||
"imageTag": version,
|
||
"services": [service_name],
|
||
"artifactPath": str(artifact_root),
|
||
"serviceArtifacts": {service_name: str(artifact_root)},
|
||
}
|
||
|
||
|
||
def build_service_images(
|
||
service_names: list[str],
|
||
*,
|
||
java_git_ref: str,
|
||
go_git_ref: str,
|
||
image_tag: str,
|
||
skip_maven_build: bool,
|
||
progress: ProgressLogger | None = None,
|
||
) -> dict[str, Any]:
|
||
# 一个请求里可能同时选了 Java 和 Go，需要分仓构建后再合并 digest。
|
||
normalized = normalize_service_names(service_names)
|
||
java_services = [service_name for service_name in normalized if service_kind(service_name) == "java"]
|
||
dashboard_cdc_services = [service_name for service_name in normalized if service_name in DASHBOARD_CDC_BUILDABLE_SERVICES]
|
||
golang_services = [
|
||
service_name
|
||
for service_name in normalized
|
||
if service_kind(service_name) == "golang" and service_name not in DASHBOARD_CDC_BUILDABLE_SERVICES
|
||
]
|
||
|
||
if not normalized:
|
||
raise RuntimeError("services are required")
|
||
|
||
if java_services:
|
||
login_repo = ensure_repo_root(JAVA_REPO_ROOT, "build")
|
||
elif golang_services:
|
||
login_repo = ensure_repo_root(GOLANG_REPO_ROOT, "build")
|
||
else:
|
||
login_repo = ensure_dashboard_cdc_repo_root(progress=progress)
|
||
docker_login_local(login_repo, progress=progress)
|
||
|
||
builds: list[dict[str, Any]] = []
|
||
service_images: dict[str, str] = {}
|
||
if java_services:
|
||
java_build = build_java_images(
|
||
java_services,
|
||
java_git_ref,
|
||
image_tag,
|
||
skip_maven_build=skip_maven_build,
|
||
progress=progress,
|
||
)
|
||
builds.append(java_build)
|
||
service_images.update(java_build["serviceImages"])
|
||
if golang_services:
|
||
golang_build = build_golang_images(golang_services, go_git_ref, image_tag, progress=progress)
|
||
builds.append(golang_build)
|
||
service_images.update(golang_build["serviceImages"])
|
||
if dashboard_cdc_services:
|
||
dashboard_cdc_build = build_dashboard_cdc_images(
|
||
dashboard_cdc_services,
|
||
go_git_ref,
|
||
image_tag,
|
||
progress=progress,
|
||
)
|
||
builds.append(dashboard_cdc_build)
|
||
service_images.update(dashboard_cdc_build["serviceImages"])
|
||
|
||
return {
|
||
"builds": builds,
|
||
"serviceImages": service_images,
|
||
}
|
||
|
||
|
||
def host_image_plan(service_names: list[str], service_images: dict[str, str]) -> list[dict[str, Any]]:
|
||
# 先按宿主机聚合 image，后面可以先预拉再做滚动重建。
|
||
records_map = service_records_by_name()
|
||
items: list[dict[str, Any]] = []
|
||
host_index: dict[str, dict[str, Any]] = {}
|
||
|
||
for service_name in service_names:
|
||
image_ref = str(service_images.get(service_name) or "").strip()
|
||
if not image_ref:
|
||
raise RuntimeError(f"missing built image for service: {service_name}")
|
||
|
||
for record in records_map.get(service_name) or []:
|
||
host_entry = host_index.get(record["host"])
|
||
if host_entry is None:
|
||
host_entry = {
|
||
"host": record["host"],
|
||
"ip": record["ip"],
|
||
"instanceId": record["instanceId"],
|
||
"services": [],
|
||
"images": [],
|
||
}
|
||
host_index[record["host"]] = host_entry
|
||
items.append(host_entry)
|
||
|
||
if service_name not in host_entry["services"]:
|
||
host_entry["services"].append(service_name)
|
||
if image_ref not in host_entry["images"]:
|
||
host_entry["images"].append(image_ref)
|
||
|
||
return items
|
||
|
||
|
||
def render_remote_login_lines() -> list[str]:
|
||
# 目标机拉 Harbor 镜像前同样需要显式登录。
|
||
require_harbor_credentials()
|
||
return [
|
||
f"printf '%s' {shlex.quote(HARBOR_PASSWORD)} | docker login {shlex.quote(HARBOR_REGISTRY)} -u {shlex.quote(HARBOR_USERNAME)} --password-stdin",
|
||
]
|
||
|
||
|
||
def remote_host_result(host_entry: dict[str, Any], script: str, command_name: str, timeout_seconds: int) -> dict[str, Any]:
|
||
# 统一执行单宿主机命令，便于预拉镜像和其他操作复用。
|
||
return run_host_script_checked(
|
||
host_entry,
|
||
script,
|
||
command_name=command_name,
|
||
timeout_seconds=timeout_seconds,
|
||
)
|
||
|
||
|
||
def prepull_host_images(host_entry: dict[str, Any]) -> dict[str, Any]:
|
||
# 先把该宿主机上要更新的镜像拉好，减少单实例重建时的等待。
|
||
lines = ["set -eu", *render_remote_login_lines()]
|
||
for image_ref in host_entry.get("images") or []:
|
||
lines.append(f"docker pull {shlex.quote(str(image_ref))}")
|
||
script = "\n".join(lines) + "\n"
|
||
result = remote_host_result(
|
||
host_entry,
|
||
script,
|
||
f"prepull-{host_entry['host']}",
|
||
timeout_seconds=max(SERVICE_PULL_TIMEOUT_SECONDS, 120),
|
||
)
|
||
return {
|
||
"status": result.get("status"),
|
||
"output": str(result.get("output") or "").strip()[-1200:],
|
||
}
|
||
|
||
|
||
def dashboard_cdc_env_path(host_name: str) -> str:
|
||
return f"{RUNTIME_BASE_DIR.rstrip('/')}/{host_name}/dashboard-cdc-worker.env"
|
||
|
||
|
||
def render_dashboard_cdc_env() -> str:
|
||
# 运行时配置从 Nacos 读取并写到目标机 env 文件，不把数据库密钥提交进 Git。
|
||
from .nacos import current_namespace, get_config_content
|
||
|
||
namespace_id, _ = current_namespace()
|
||
content = get_config_content(namespace_id, "common", "rds-config.yml")
|
||
|
||
def yaml_value(key: str) -> str:
|
||
match = re.search(rf"(?im)^\s*{re.escape(key)}\s*:\s*(.*?)\s*$", content)
|
||
if not match:
|
||
raise RuntimeError(f"missing {key} in common/rds-config.yml")
|
||
value = match.group(1).strip()
|
||
if (value.startswith('"') and value.endswith('"')) or (value.startswith("'") and value.endswith("'")):
|
||
value = value[1:-1]
|
||
return value
|
||
|
||
jdbc_url = (
|
||
yaml_value("url")
|
||
.replace("jdbc:p6spy:mysql://", "mysql://", 1)
|
||
.replace("jdbc:mysql://", "mysql://", 1)
|
||
)
|
||
parsed = urlparse(jdbc_url)
|
||
if not parsed.hostname:
|
||
raise RuntimeError("invalid rds-config url for dashboard cdc worker")
|
||
|
||
username = yaml_value("username")
|
||
password = yaml_value("password")
|
||
database = parsed.path.lstrip("/") or "likei"
|
||
addr = f"{parsed.hostname}:{parsed.port or 3306}"
|
||
dsn = (
|
||
f"{quote(username, safe='')}:{quote(password, safe='')}@tcp({addr})/{database}"
|
||
"?charset=utf8mb4&parseTime=true&loc=Asia%2FRiyadh"
|
||
)
|
||
return "\n".join(
|
||
[
|
||
f"CDC_MYSQL_ADDR={addr}",
|
||
f"CDC_MYSQL_USER={username}",
|
||
f"CDC_MYSQL_PASSWORD={password}",
|
||
f"DASHBOARD_MYSQL_DSN={dsn}",
|
||
"CDC_START_MODE=checkpoint",
|
||
"CDC_SERVER_ID=29301",
|
||
"CDC_DRY_RUN=false",
|
||
"DASHBOARD_SYS_ORIGINS=LIKEI",
|
||
"DASHBOARD_STAT_TIMEZONES=Asia/Riyadh",
|
||
"DASHBOARD_STORAGE_TIMEZONE=Asia/Riyadh",
|
||
"REDIS_ENABLED=false",
|
||
"HEALTH_ADDR=:2910",
|
||
"",
|
||
]
|
||
)
|
||
|
||
|
||
def ensure_dashboard_cdc_remote_env(host_entry: dict[str, Any]) -> dict[str, Any]:
|
||
env_text = render_dashboard_cdc_env()
|
||
env_path = dashboard_cdc_env_path(host_entry["host"])
|
||
write_remote_text(host_entry, env_path, env_text, timeout_seconds=30)
|
||
return {"path": env_path, "changed": True}
|
||
|
||
|
||
def ensure_dashboard_cdc_compose_service(host_name: str) -> dict[str, Any] | None:
|
||
# 新服务首次上线前远端还没有运行容器，不能依赖 docker inspect 自动生成模板。
|
||
model = load_local_host_compose_model(host_name)
|
||
service_map = dict(model.get("services") or {})
|
||
if DASHBOARD_CDC_SERVICE_NAME in service_map:
|
||
return None
|
||
|
||
service_map[DASHBOARD_CDC_SERVICE_NAME] = {
|
||
"image": f"{HARBOR_REGISTRY}/{HARBOR_PROJECT}/dashboard-cdc-worker:bootstrap",
|
||
"container_name": f"likei-{host_name}-{DASHBOARD_CDC_SERVICE_NAME}",
|
||
"network_mode": "host",
|
||
"restart": "unless-stopped",
|
||
"stop_signal": "SIGTERM",
|
||
"env_file": ["./dashboard-cdc-worker.env"],
|
||
"pull_policy": "always",
|
||
"working_dir": "/application",
|
||
"entrypoint": [],
|
||
"command": ["/application/service"],
|
||
"healthcheck": {
|
||
"test": ["CMD-SHELL", "wget -qO- http://127.0.0.1:2910/health | grep -q '\"code\":\"ok\"'"],
|
||
"interval": "10s",
|
||
"timeout": "10s",
|
||
"retries": 30,
|
||
},
|
||
"logging": {
|
||
"driver": "json-file",
|
||
"options": {"max-size": "200m", "max-file": "5"},
|
||
},
|
||
}
|
||
return save_local_host_compose_model(host_name, service_map)
|
||
|
||
|
||
def prepare_frontend_remote_tmp(host: dict[str, Any]) -> str:
|
||
# 先在远端站点目录旁边建一份临时上传目录，避免直接覆盖线上内容。
|
||
site_dir = str(ADMIN_FRONTEND_SITE_DIR or "").rstrip("/")
|
||
parent_dir = str(Path(site_dir).parent)
|
||
script = f"""set -eu
|
||
PARENT_DIR={shlex.quote(parent_dir)}
|
||
mkdir -p "$PARENT_DIR"
|
||
mktemp -d "$PARENT_DIR/.hy-app-monitor-site.XXXXXX"
|
||
"""
|
||
result = run_host_script_checked(
|
||
host,
|
||
script,
|
||
command_name=f"frontend-tmp-{host['host']}",
|
||
timeout_seconds=30,
|
||
)
|
||
tmp_dir = str(result.get("output") or "").strip().splitlines()[-1].strip()
|
||
if not tmp_dir:
|
||
raise RuntimeError(f"failed to create frontend temp dir on {target_display(host)}")
|
||
return tmp_dir
|
||
|
||
|
||
def activate_frontend_site(
|
||
host: dict[str, Any],
|
||
*,
|
||
tmp_dir: str,
|
||
backup_stamp: str,
|
||
) -> dict[str, str]:
|
||
# 站点目录保留原 inode，只在目录内替换文件，避免 bind mount 指向失效。
|
||
site_dir = str(ADMIN_FRONTEND_SITE_DIR or "").rstrip("/")
|
||
remote_dir = str(ADMIN_FRONTEND_REMOTE_DIR or "").rstrip("/")
|
||
compose_file = str(ADMIN_FRONTEND_COMPOSE_FILE or "").rstrip("/")
|
||
compose_service = str(ADMIN_FRONTEND_COMPOSE_SERVICE or "").strip() or "caddy"
|
||
backup_root = str(Path(site_dir).parent / ".hy-app-monitor-site-backups")
|
||
keep_count = max(1, int(FRONTEND_SITE_BACKUP_KEEP))
|
||
script = f"""set -eu
|
||
SITE_DIR={shlex.quote(site_dir)}
|
||
TMP_DIR={shlex.quote(tmp_dir)}
|
||
BACKUP_ROOT={shlex.quote(backup_root)}
|
||
BACKUP_DIR="$BACKUP_ROOT/{shlex.quote(backup_stamp)}"
|
||
REMOTE_DIR={shlex.quote(remote_dir)}
|
||
COMPOSE_FILE={shlex.quote(compose_file)}
|
||
COMPOSE_SERVICE={shlex.quote(compose_service)}
|
||
KEEP_COUNT={keep_count}
|
||
|
||
mkdir -p "$SITE_DIR" "$BACKUP_ROOT"
|
||
if [ -d "$SITE_DIR" ]; then
|
||
mkdir -p "$BACKUP_DIR"
|
||
cp -a "$SITE_DIR"/. "$BACKUP_DIR"/ 2>/dev/null || true
|
||
fi
|
||
find "$SITE_DIR" -mindepth 1 -maxdepth 1 -exec rm -rf {{}} +
|
||
cp -a "$TMP_DIR"/. "$SITE_DIR"/
|
||
rm -rf "$TMP_DIR"
|
||
|
||
if [ -d "$BACKUP_ROOT" ]; then
|
||
ls -1dt "$BACKUP_ROOT"/* 2>/dev/null | awk "NR>$KEEP_COUNT" | xargs -r rm -rf
|
||
fi
|
||
|
||
cd "$REMOTE_DIR"
|
||
docker compose -f "$COMPOSE_FILE" up -d "$COMPOSE_SERVICE" >/dev/null
|
||
docker compose -f "$COMPOSE_FILE" ps "$COMPOSE_SERVICE" || true
|
||
"""
|
||
result = run_host_script_checked(
|
||
host,
|
||
script,
|
||
command_name=f"frontend-activate-{host['host']}",
|
||
timeout_seconds=max(SERVICE_BUILD_TIMEOUT_SECONDS, 180),
|
||
)
|
||
return {
|
||
"backupPath": f"{backup_root}/{backup_stamp}",
|
||
"output": str(result.get("output") or "").strip()[-1200:],
|
||
}
|
||
|
||
|
||
def deploy_frontend_site(
|
||
service_name: str,
|
||
*,
|
||
git_ref: str,
|
||
progress: ProgressLogger | None = None,
|
||
) -> dict[str, Any]:
|
||
# 前端发布链路：构建 dist -> SCP 到目标机 -> 原地切换站点内容 -> 写版本元数据。
|
||
host = frontend_host_record(service_name)
|
||
build_result = build_frontend_bundle(service_name, git_ref, progress=progress)
|
||
artifact_path = Path(str(build_result.get("artifactPath") or "")).expanduser().resolve()
|
||
version = str(build_result.get("imageTag") or "").strip()
|
||
backup_stamp = datetime.utcnow().strftime("%Y%m%d-%H%M%S")
|
||
step_stage = f"frontend.{host['host']}.{service_name}"
|
||
started_at = time.time()
|
||
step: dict[str, Any] = {
|
||
"service": service_name,
|
||
"host": host["host"],
|
||
"ip": host["ip"],
|
||
"kind": "frontend",
|
||
"image": version,
|
||
"status": "running",
|
||
}
|
||
tmp_dir = ""
|
||
|
||
try:
|
||
tmp_dir = prepare_frontend_remote_tmp(host)
|
||
emit_progress(progress, step_stage, f"上传静态产物：{artifact_path} -> {target_display(host)}:{tmp_dir}")
|
||
copy_local_path_to_host(
|
||
host,
|
||
f"{artifact_path}/.",
|
||
f"{tmp_dir}/",
|
||
recursive=True,
|
||
timeout_seconds=max(SERVICE_BUILD_TIMEOUT_SECONDS, 180),
|
||
)
|
||
emit_progress(progress, step_stage, "上传完成，开始切换站点目录")
|
||
activate_result = activate_frontend_site(host, tmp_dir=tmp_dir, backup_stamp=backup_stamp)
|
||
except Exception as exc: # noqa: BLE001
|
||
# 上传或切换失败时尽量收尾远端临时目录，避免后续发布被历史垃圾影响。
|
||
cleanup_script = f"""set -eu
|
||
TMP_DIR={shlex.quote(tmp_dir)}
|
||
if [ -d "$TMP_DIR" ]; then
|
||
rm -rf "$TMP_DIR"
|
||
fi
|
||
"""
|
||
if tmp_dir:
|
||
try:
|
||
run_host_script_checked(
|
||
host,
|
||
cleanup_script,
|
||
command_name=f"frontend-cleanup-{host['host']}",
|
||
timeout_seconds=30,
|
||
)
|
||
except Exception: # noqa: BLE001
|
||
pass
|
||
step["status"] = "failed"
|
||
step["error"] = str(exc)
|
||
step["durationSeconds"] = round(time.time() - started_at, 2)
|
||
emit_progress(progress, step_stage, f"前端发布失败：{step['error']}", level="error")
|
||
return {
|
||
"ok": False,
|
||
"updatedAt": utc_now(),
|
||
"services": [service_name],
|
||
"build": [build_result],
|
||
"templates": [],
|
||
"preparedHosts": [],
|
||
"steps": [step],
|
||
"error": step["error"],
|
||
}
|
||
|
||
deployed_at = utc_now()
|
||
metadata = {
|
||
"service": service_name,
|
||
"kind": "frontend",
|
||
"repoUrl": str(ADMIN_FRONTEND_REPO_URL or "").strip(),
|
||
"repoRoot": str(ADMIN_FRONTEND_REPO_ROOT),
|
||
"gitRef": str(build_result.get("gitRef") or "").strip(),
|
||
"commit": str(build_result.get("commit") or "").strip(),
|
||
"version": version,
|
||
"publicUrl": ADMIN_FRONTEND_PUBLIC_URL,
|
||
"siteDir": ADMIN_FRONTEND_SITE_DIR,
|
||
"composeFile": ADMIN_FRONTEND_COMPOSE_FILE,
|
||
"deployedAt": deployed_at,
|
||
"backupPath": activate_result["backupPath"],
|
||
"deployedBy": "hy-app-monitor",
|
||
}
|
||
write_remote_text(
|
||
host,
|
||
ADMIN_FRONTEND_RELEASE_META_PATH,
|
||
json.dumps(metadata, ensure_ascii=False, indent=2) + "\n",
|
||
timeout_seconds=30,
|
||
)
|
||
emit_progress(progress, step_stage, f"版本元数据已写入：{ADMIN_FRONTEND_RELEASE_META_PATH}")
|
||
step["status"] = "success"
|
||
step["remoteStatus"] = "SUCCESS"
|
||
step["remoteOutput"] = activate_result["output"]
|
||
step["durationSeconds"] = round(time.time() - started_at, 2)
|
||
|
||
return {
|
||
"ok": True,
|
||
"updatedAt": deployed_at,
|
||
"services": [service_name],
|
||
"build": [build_result],
|
||
"templates": [],
|
||
"preparedHosts": [],
|
||
"steps": [step],
|
||
"message": "service update complete",
|
||
}
|
||
|
||
|
||
def build_update_targets_payload() -> dict[str, Any]:
|
||
# 页面需要展示当前可更新服务、宿主机，以及本地构建所依赖的仓路径。
|
||
records_map = service_records_by_name()
|
||
items: list[dict[str, Any]] = []
|
||
# 同一台主机会承载多个服务，这里先把 compose 模型按主机缓存起来，避免重复读模板。
|
||
compose_models_by_host: dict[str, dict[str, Any] | None] = {}
|
||
# 模板读取失败不能拖垮整页，这里保留每台主机的错误给后续条目复用。
|
||
compose_errors_by_host: dict[str, str] = {}
|
||
|
||
def host_compose_model(host_name: str) -> dict[str, Any] | None:
|
||
# 每台主机只解析一次本地 compose；失败时返回空并记下错误摘要。
|
||
if host_name in compose_models_by_host:
|
||
return compose_models_by_host[host_name]
|
||
if host_name in compose_errors_by_host:
|
||
return None
|
||
|
||
try:
|
||
compose_models_by_host[host_name] = load_local_host_compose_model(host_name)
|
||
return compose_models_by_host[host_name]
|
||
except Exception as exc: # noqa: BLE001
|
||
compose_errors_by_host[host_name] = short_text(str(exc), 200)
|
||
compose_models_by_host[host_name] = None
|
||
return None
|
||
|
||
for service_name in buildable_service_names():
|
||
records = records_map.get(service_name) or []
|
||
if not records:
|
||
continue
|
||
current_images: list[str] = []
|
||
compose_warnings: list[str] = []
|
||
for record in records:
|
||
model = host_compose_model(record["host"])
|
||
if model is None:
|
||
compose_error = compose_errors_by_host.get(record["host"], "compose template unavailable")
|
||
warning_text = f"{record['host']}: {compose_error}"
|
||
if warning_text not in compose_warnings:
|
||
compose_warnings.append(warning_text)
|
||
continue
|
||
image_ref = str(((model.get("services") or {}).get(service_name) or {}).get("image") or "").strip()
|
||
if image_ref and image_ref not in current_images:
|
||
current_images.append(image_ref)
|
||
display = image_release_display(current_images)
|
||
|
||
item = {
|
||
"service": service_name,
|
||
"kind": service_kind(service_name),
|
||
"repository": service_repository_ref(service_name),
|
||
"currentLabel": display["currentLabel"],
|
||
"currentTitle": display["currentTitle"],
|
||
"detailLabel": display["detailLabel"],
|
||
"detailTitle": display["detailTitle"],
|
||
"copyValue": display["copyValue"],
|
||
"currentImages": current_images,
|
||
"hosts": [
|
||
{
|
||
"host": record["host"],
|
||
"ip": record["ip"],
|
||
"instanceId": record["instanceId"],
|
||
"port": record["port"],
|
||
"path": record["path"],
|
||
}
|
||
for record in records
|
||
],
|
||
}
|
||
# 某些主机模板暂时不可读时，仍然保留服务条目，只把原因透出去。
|
||
if service_name == "gateway":
|
||
gateway_warning = gateway_clb_configuration_error()
|
||
if gateway_warning and gateway_warning not in compose_warnings:
|
||
compose_warnings.append(gateway_warning)
|
||
if compose_warnings:
|
||
item["warnings"] = compose_warnings
|
||
items.append(item)
|
||
|
||
if frontend_release_enabled():
|
||
frontend_service = ADMIN_FRONTEND_SERVICE_NAME
|
||
display = frontend_release_display(frontend_service)
|
||
items.append(
|
||
{
|
||
"service": frontend_service,
|
||
"kind": "frontend",
|
||
"repository": str(ADMIN_FRONTEND_REPO_URL or ADMIN_FRONTEND_REPO_ROOT),
|
||
"currentLabel": display["currentLabel"],
|
||
"currentTitle": display["currentTitle"],
|
||
"detailLabel": display["detailLabel"],
|
||
"detailTitle": display["detailTitle"],
|
||
"copyValue": display["copyValue"],
|
||
"currentImages": [],
|
||
"hosts": [
|
||
{
|
||
"host": ADMIN_FRONTEND_HOST,
|
||
"ip": ADMIN_FRONTEND_IP,
|
||
"instanceId": ADMIN_FRONTEND_INSTANCE_ID,
|
||
"port": 0,
|
||
"path": ADMIN_FRONTEND_SITE_DIR,
|
||
}
|
||
],
|
||
}
|
||
)
|
||
|
||
return {
|
||
"ok": True,
|
||
"updatedAt": utc_now(),
|
||
"defaults": {
|
||
"javaGitRef": UPDATE_DEFAULT_JAVA_GIT_REF,
|
||
"goGitRef": UPDATE_DEFAULT_GO_GIT_REF,
|
||
"adminGitRef": UPDATE_DEFAULT_ADMIN_GIT_REF,
|
||
"dockerPlatform": BUILD_DOCKER_PLATFORM,
|
||
},
|
||
"builder": {
|
||
"harborRegistry": HARBOR_REGISTRY,
|
||
"harborProject": HARBOR_PROJECT,
|
||
"javaRepoRoot": str(Path(JAVA_REPO_ROOT).expanduser()),
|
||
"golangRepoRoot": str(Path(GOLANG_REPO_ROOT).expanduser()),
|
||
"dashboardCdcRepoRoot": str(Path(DASHBOARD_CDC_REPO_ROOT).expanduser()),
|
||
"dashboardCdcRepoUrl": str(DASHBOARD_CDC_REPO_URL or "").strip(),
|
||
"frontendRepoRoot": str(Path(ADMIN_FRONTEND_REPO_ROOT).expanduser()),
|
||
"frontendRepoUrl": str(ADMIN_FRONTEND_REPO_URL or "").strip(),
|
||
"frontendPublicUrl": ADMIN_FRONTEND_PUBLIC_URL,
|
||
},
|
||
"items": items,
|
||
}
|
||
|
||
|
||
def deploy_runtime_services_payload(
|
||
service_names: list[str],
|
||
*,
|
||
java_git_ref: str,
|
||
go_git_ref: str,
|
||
image_tag: str,
|
||
skip_maven_build: bool,
|
||
progress: ProgressLogger | None = None,
|
||
) -> dict[str, Any]:
|
||
# Java / Go 服务沿用镜像构建、完整 compose、预拉、滚动重建这条链路。
|
||
normalized = normalize_service_names(service_names)
|
||
if not normalized:
|
||
raise RuntimeError("services are required")
|
||
|
||
records_map = service_records_by_name()
|
||
unsupported = [service_name for service_name in normalized if service_name not in buildable_service_names()]
|
||
if unsupported:
|
||
raise RuntimeError(f"unsupported update services: {', '.join(unsupported)}")
|
||
for service_name in normalized:
|
||
if service_name not in records_map:
|
||
raise RuntimeError(f"service not found in runtime topology: {service_name}")
|
||
|
||
emit_progress(progress, "prepare", f"准备更新服务：{', '.join(normalized)}")
|
||
build_result = build_service_images(
|
||
normalized,
|
||
java_git_ref=java_git_ref,
|
||
go_git_ref=go_git_ref,
|
||
image_tag=image_tag,
|
||
skip_maven_build=skip_maven_build,
|
||
progress=progress,
|
||
)
|
||
service_images = dict(build_result.get("serviceImages") or {})
|
||
emit_progress(progress, "prepare", "镜像构建完成，开始更新本地完整 compose 模板")
|
||
|
||
template_updates: list[dict[str, Any]] = []
|
||
for service_name in normalized:
|
||
for record in records_map[service_name]:
|
||
image_ref = service_images[service_name]
|
||
if service_name in DASHBOARD_CDC_BUILDABLE_SERVICES:
|
||
ensure_result = ensure_dashboard_cdc_compose_service(record["host"])
|
||
if ensure_result is not None:
|
||
template_updates.append(ensure_result)
|
||
emit_progress(progress, "compose.template", f"更新模板：{record['host']} / {service_name} -> {image_ref}")
|
||
template_updates.append(update_local_service_image(record["host"], service_name, image_ref))
|
||
|
||
host_plan = host_image_plan(normalized, service_images)
|
||
prepared_hosts: list[dict[str, Any]] = []
|
||
steps: list[dict[str, Any]] = []
|
||
|
||
try:
|
||
for host_entry in host_plan:
|
||
if any(service_name in DASHBOARD_CDC_BUILDABLE_SERVICES for service_name in host_entry.get("services") or []):
|
||
emit_progress(progress, f"compose.env.{host_entry['host']}", "写入 Dashboard CDC Worker 运行配置")
|
||
env_result = ensure_dashboard_cdc_remote_env(host_entry)
|
||
emit_progress(progress, f"compose.env.{host_entry['host']}", f"运行配置已写入：{env_result['path']}")
|
||
emit_progress(progress, f"compose.sync.{host_entry['host']}", f"同步完整 compose 到远端：{host_entry['host']}")
|
||
compose_result = sync_remote_host_compose_template(host_entry["host"], refresh_from_remote=False)
|
||
emit_progress(
|
||
progress,
|
||
f"compose.sync.{host_entry['host']}",
|
||
f"compose 同步完成：changed={bool(compose_result.get('changed'))} path={compose_result.get('path')}",
|
||
)
|
||
emit_progress(progress, f"image.prepull.{host_entry['host']}", f"预拉镜像：{', '.join(host_entry.get('images') or [])}")
|
||
prepull_result = prepull_host_images(host_entry)
|
||
emit_progress(
|
||
progress,
|
||
f"image.prepull.{host_entry['host']}",
|
||
f"预拉完成：{prepull_result['status']} {short_text(prepull_result['output'])}",
|
||
)
|
||
prepared_hosts.append(
|
||
{
|
||
"host": host_entry["host"],
|
||
"ip": host_entry["ip"],
|
||
"services": list(host_entry["services"]),
|
||
"images": list(host_entry["images"]),
|
||
"compose": {
|
||
"path": compose_result["path"],
|
||
"changed": bool(compose_result.get("changed")),
|
||
"source": str(compose_result.get("source") or ""),
|
||
"backupPath": str(compose_result.get("backupPath") or ""),
|
||
},
|
||
"prepull": {
|
||
"status": prepull_result["status"],
|
||
"output": prepull_result["output"],
|
||
},
|
||
}
|
||
)
|
||
|
||
for service_name in normalized:
|
||
for service in records_map[service_name]:
|
||
started_at = time.time()
|
||
step_stage = f"restart.{service['host']}.{service_name}"
|
||
step: dict[str, Any] = {
|
||
"service": service_name,
|
||
"host": service["host"],
|
||
"ip": service["ip"],
|
||
"kind": service["kind"],
|
||
"image": service_images[service_name],
|
||
"status": "running",
|
||
}
|
||
gateway_clb_snapshot: dict[str, Any] | None = None
|
||
gateway_clb_health_ready = False
|
||
gateway_clb_restored = False
|
||
nacos_drained = False
|
||
try:
|
||
emit_progress(progress, step_stage, f"开始滚动更新：{service['host']} / {service_name}")
|
||
|
||
if service_name == "gateway":
|
||
emit_progress(progress, step_stage, "读取 gateway 当前 CLB 绑定")
|
||
gateway_clb_snapshot = prepare_gateway_release_clb_snapshot(service)
|
||
step["clb"] = {
|
||
"loadBalancerId": gateway_clb_snapshot["loadBalancerId"],
|
||
"listenerIds": list(gateway_clb_snapshot["listenerIds"]),
|
||
"bindingCount": len(gateway_clb_snapshot["bindings"]),
|
||
"bindings": [
|
||
{
|
||
"listenerId": binding["listenerId"],
|
||
"locationId": binding["locationId"],
|
||
"domain": binding["domain"],
|
||
"url": binding["url"],
|
||
"port": binding["port"],
|
||
"originalWeight": binding["weight"],
|
||
}
|
||
for binding in gateway_clb_snapshot["bindings"]
|
||
],
|
||
}
|
||
emit_progress(progress, step_stage, f"CLB 摘流前权重：{gateway_clb_snapshot['bindingSummary']}")
|
||
drain_result = drain_gateway_release_target(gateway_clb_snapshot)
|
||
step["clb"]["drainedSummary"] = drain_result["summary"]
|
||
emit_progress(progress, step_stage, f"CLB 摘流完成：{drain_result['summary']}")
|
||
if GATEWAY_CLB_DRAIN_SECONDS > 0:
|
||
emit_progress(progress, step_stage, f"CLB 排空等待 {GATEWAY_CLB_DRAIN_SECONDS}s")
|
||
time.sleep(GATEWAY_CLB_DRAIN_SECONDS)
|
||
elif service["kind"] == "java":
|
||
drain_result = drain_java_registration(service)
|
||
if drain_result.get("checked"):
|
||
nacos_drained = True
|
||
step["nacosDrain"] = drain_result
|
||
emit_progress(
|
||
progress,
|
||
step_stage,
|
||
f"Nacos 摘流完成：enabled=false drain={drain_result.get('drainSeconds')}s",
|
||
)
|
||
|
||
restart_result = restart_service_instance(service)
|
||
step["remoteStatus"] = restart_result.get("status")
|
||
step["remoteOutput"] = str(restart_result.get("output") or "").strip()[:500]
|
||
emit_progress(
|
||
progress,
|
||
step_stage,
|
||
f"远端重建完成：{restart_result.get('status')} {short_text(step['remoteOutput'])}",
|
||
)
|
||
|
||
http_result = wait_http_ready(service, timeout_seconds=ROLLING_RESTART_TIMEOUT_SECONDS)
|
||
step["http"] = {
|
||
"ok": bool(http_result.get("ok")),
|
||
"statusCode": int(http_result.get("statusCode") or 0),
|
||
"latencyMs": int(http_result.get("latencyMs") or 0),
|
||
"detail": str(http_result.get("detail") or ""),
|
||
}
|
||
emit_progress(
|
||
progress,
|
||
step_stage,
|
||
f"HTTP 恢复：{step['http']['statusCode']} / {step['http']['latencyMs']}ms",
|
||
)
|
||
|
||
if service["kind"] == "java":
|
||
step["nacos"] = (
|
||
restore_java_registration(service)
|
||
if nacos_drained
|
||
else wait_java_registration_ready(
|
||
service,
|
||
timeout_seconds=ROLLING_RESTART_TIMEOUT_SECONDS,
|
||
)
|
||
)
|
||
emit_progress(
|
||
progress,
|
||
step_stage,
|
||
f"Nacos 注册恢复：{step['nacos'].get('source') or 'ok'}",
|
||
)
|
||
|
||
if gateway_clb_snapshot is not None:
|
||
clb_health = wait_gateway_release_target_healthy(
|
||
gateway_clb_snapshot,
|
||
timeout_seconds=ROLLING_RESTART_TIMEOUT_SECONDS,
|
||
)
|
||
gateway_clb_health_ready = True
|
||
step["clb"]["health"] = {
|
||
"ok": True,
|
||
"targets": clb_health,
|
||
}
|
||
emit_progress(progress, step_stage, "CLB 健康检查恢复，开始恢复转发权重")
|
||
restore_result = restore_gateway_release_target(gateway_clb_snapshot)
|
||
gateway_clb_restored = True
|
||
step["clb"]["restoredSummary"] = restore_result["summary"]
|
||
emit_progress(progress, step_stage, f"CLB 权重恢复完成：{restore_result['summary']}")
|
||
|
||
if ROLLING_RESTART_STABILIZE_SECONDS > 0:
|
||
emit_progress(
|
||
progress,
|
||
step_stage,
|
||
f"稳定观察 {ROLLING_RESTART_STABILIZE_SECONDS}s",
|
||
)
|
||
time.sleep(ROLLING_RESTART_STABILIZE_SECONDS)
|
||
|
||
step["status"] = "success"
|
||
step["durationSeconds"] = round(time.time() - started_at, 2)
|
||
steps.append(step)
|
||
emit_progress(
|
||
progress,
|
||
step_stage,
|
||
f"实例更新成功，耗时 {step['durationSeconds']}s",
|
||
)
|
||
except Exception as exc: # noqa: BLE001
|
||
if gateway_clb_snapshot is not None and gateway_clb_health_ready and not gateway_clb_restored:
|
||
try:
|
||
restore_result = restore_gateway_release_target(gateway_clb_snapshot)
|
||
gateway_clb_restored = True
|
||
step.setdefault("clb", {})["restoreRecovered"] = restore_result["summary"]
|
||
emit_progress(progress, step_stage, f"失败收尾时已恢复 CLB 权重：{restore_result['summary']}")
|
||
except Exception as restore_exc: # noqa: BLE001
|
||
step.setdefault("clb", {})["restoreError"] = str(restore_exc)
|
||
if nacos_drained:
|
||
try:
|
||
restore_result = update_java_registration_enabled(service, True)
|
||
step["nacosRestoreRecovered"] = restore_result
|
||
emit_progress(progress, step_stage, "失败收尾时已恢复 Nacos enabled=true")
|
||
except Exception as restore_exc: # noqa: BLE001
|
||
step["nacosRestoreError"] = str(restore_exc)
|
||
step["status"] = "failed"
|
||
step["error"] = str(exc)
|
||
step["durationSeconds"] = round(time.time() - started_at, 2)
|
||
steps.append(step)
|
||
emit_progress(progress, step_stage, f"实例更新失败：{step['error']}", level="error")
|
||
clear_operation_caches()
|
||
return {
|
||
"ok": False,
|
||
"updatedAt": utc_now(),
|
||
"services": normalized,
|
||
"build": build_result.get("builds") or [],
|
||
"templates": template_updates,
|
||
"preparedHosts": prepared_hosts,
|
||
"steps": steps,
|
||
"error": step["error"],
|
||
}
|
||
finally:
|
||
clear_operation_caches()
|
||
emit_progress(progress, "finish", "已清理监控缓存")
|
||
|
||
return {
|
||
"ok": True,
|
||
"updatedAt": utc_now(),
|
||
"services": normalized,
|
||
"build": build_result.get("builds") or [],
|
||
"templates": template_updates,
|
||
"preparedHosts": prepared_hosts,
|
||
"steps": steps,
|
||
"message": "service update complete",
|
||
}
|
||
|
||
|
||
def merge_update_results(service_names: list[str], results: list[dict[str, Any]]) -> dict[str, Any]:
|
||
# 一个任务里允许同时带镜像服务和前端服务，这里统一把结果拼回页面结构。
|
||
merged = {
|
||
"ok": True,
|
||
"updatedAt": utc_now(),
|
||
"services": list(service_names),
|
||
"build": [],
|
||
"templates": [],
|
||
"preparedHosts": [],
|
||
"steps": [],
|
||
"message": "service update complete",
|
||
}
|
||
|
||
for result in results:
|
||
if not result:
|
||
continue
|
||
merged["ok"] = merged["ok"] and bool(result.get("ok"))
|
||
merged["build"].extend(list(result.get("build") or result.get("builds") or []))
|
||
merged["templates"].extend(list(result.get("templates") or []))
|
||
merged["preparedHosts"].extend(list(result.get("preparedHosts") or []))
|
||
merged["steps"].extend(list(result.get("steps") or []))
|
||
if not result.get("ok"):
|
||
merged["error"] = str(result.get("error") or "service update failed").strip()
|
||
merged["message"] = str(result.get("message") or merged["error"] or "service update failed").strip()
|
||
return merged
|
||
if str(result.get("message") or "").strip():
|
||
merged["message"] = str(result.get("message") or "").strip()
|
||
|
||
return merged
|
||
|
||
|
||
def deploy_updated_services_payload(
|
||
service_names: list[str],
|
||
*,
|
||
java_git_ref: str,
|
||
go_git_ref: str,
|
||
admin_git_ref: str,
|
||
image_tag: str,
|
||
skip_maven_build: bool,
|
||
progress: ProgressLogger | None = None,
|
||
) -> dict[str, Any]:
|
||
# 顶层按服务类型拆分：容器服务走 Harbor/compose，Admin 前端走 pnpm build + SSH 发布。
|
||
normalized = normalize_service_names(service_names)
|
||
if not normalized:
|
||
raise RuntimeError("services are required")
|
||
|
||
unsupported = [service_name for service_name in normalized if service_name not in buildable_service_names()]
|
||
if unsupported:
|
||
raise RuntimeError(f"unsupported update services: {', '.join(unsupported)}")
|
||
|
||
runtime_services = [service_name for service_name in normalized if service_kind(service_name) in {"java", "golang"}]
|
||
frontend_services = [service_name for service_name in normalized if service_kind(service_name) == "frontend"]
|
||
records_map = service_records_by_name()
|
||
|
||
for service_name in runtime_services:
|
||
if service_name not in records_map:
|
||
raise RuntimeError(f"service not found in runtime topology: {service_name}")
|
||
|
||
emit_progress(progress, "prepare", f"准备更新服务：{', '.join(normalized)}")
|
||
results: list[dict[str, Any]] = []
|
||
|
||
if runtime_services:
|
||
runtime_result = deploy_runtime_services_payload(
|
||
runtime_services,
|
||
java_git_ref=java_git_ref,
|
||
go_git_ref=go_git_ref,
|
||
image_tag=image_tag,
|
||
skip_maven_build=skip_maven_build,
|
||
progress=progress,
|
||
)
|
||
results.append(runtime_result)
|
||
if not runtime_result.get("ok"):
|
||
return merge_update_results(normalized, results)
|
||
|
||
for service_name in frontend_services:
|
||
frontend_result = deploy_frontend_site(service_name, git_ref=admin_git_ref, progress=progress)
|
||
results.append(frontend_result)
|
||
if not frontend_result.get("ok"):
|
||
return merge_update_results(normalized, results)
|
||
|
||
emit_progress(progress, "finish", "全部服务更新完成")
|
||
return merge_update_results(normalized, results)
|