feat: speed up java service updates

2026-04-22 21:52:49 +08:00 · 2026-04-22 21:52:49 +08:00 · 67f58e4c57
commit 67f58e4c57
parent 16d5408928
4 changed files with 193 additions and 52 deletions
--- a/.env.example
+++ b/.env.example
@ -13,6 +13,9 @@ ROLLING_RESTART_POLL_SECONDS=2
 ROLLING_RESTART_STABILIZE_SECONDS=2
 SERVICE_BUILD_TIMEOUT_SECONDS=3600
 SERVICE_PULL_TIMEOUT_SECONDS=600
+SERVICE_BUILD_CACHE_ROOT=/opt/deploy/build-cache
+JAVA_MAVEN_THREADS=1C
+JAVA_IMAGE_BUILD_MAX_WORKERS=3
 NACOS_BASE_URL=http://10.2.1.17:8848
 NACOS_NAMESPACE_ID=f7a8594e-090c-4830-a3ad-b5c0f9afcc07
 NACOS_NAMESPACE_NAME=RedCircleServiceProd
--- a/README.md
+++ b/README.md
@ -48,6 +48,9 @@ HARBOR_USERNAME=***
 HARBOR_PASSWORD=***
 JAVA_REPO_ROOT=/opt/deploy/sources/chatapp3-java
 GOLANG_REPO_ROOT=/opt/deploy/sources/chatapp3-golang
+SERVICE_BUILD_CACHE_ROOT=/opt/deploy/build-cache
+JAVA_MAVEN_THREADS=1C
+JAVA_IMAGE_BUILD_MAX_WORKERS=3
 UPDATE_DEFAULT_JAVA_GIT_REF=main
 UPDATE_DEFAULT_GO_GIT_REF=main
 ```
@ -135,7 +138,7 @@ python3 ops/sync_runtime_compose.py --sync-remote

 页面里的“服务更新”会直接在当前机器上执行这条链路：

-1. 从 `JAVA_REPO_ROOT` / `GOLANG_REPO_ROOT` 指向的业务仓按 `git ref` 创建临时 `git worktree`
+1. Java 服务会在 `SERVICE_BUILD_CACHE_ROOT/<repo>/<git-ref>` 下复用固定构建工作区；Golang 仍按临时 `git worktree` 构建
 2. 本地构建镜像并推送到 Harbor
 3. 取回 Harbor `digest`
 4. 用新 `digest` 更新本地完整 compose 模板
@ -146,6 +149,8 @@ python3 ops/sync_runtime_compose.py --sync-remote
 说明：

 - `other` 现在也按 Harbor 业务镜像更新，不再继续依赖本地 `./other/service.jar` 挂载
+- Java Maven 默认走增量 `package`，并支持通过 `JAVA_MAVEN_THREADS` 开启 reactor 并行
+- 多个 Java 服务会按 `JAVA_IMAGE_BUILD_MAX_WORKERS` 受限并行执行 `docker build + push`
 - 页面里的“滚动重启”仍然保留，适合只想让配置生效、不涉及代码构建的场景
 - 这条链路默认假设 `hy-app-monitor` 就运行在打包机 / deploy 机上，并且本机已经能访问业务仓、Docker 和 Harbor

--- a/monitor/config.py
+++ b/monitor/config.py
@ -52,6 +52,12 @@ ROLLING_RESTART_STABILIZE_SECONDS = env_float("ROLLING_RESTART_STABILIZE_SECONDS
 SERVICE_BUILD_TIMEOUT_SECONDS = env_int("SERVICE_BUILD_TIMEOUT_SECONDS", 3600)
 # 服务更新时目标机预拉镜像最长等待秒数。
 SERVICE_PULL_TIMEOUT_SECONDS = env_int("SERVICE_PULL_TIMEOUT_SECONDS", 600)
+# 固定构建工作区根目录，留空时按本地 / deploy 机路径自动推导。
+SERVICE_BUILD_CACHE_ROOT = env_str("SERVICE_BUILD_CACHE_ROOT", "")
+# Java Maven 并行线程，默认按 1 个 CPU 组装 reactor。
+JAVA_MAVEN_THREADS = env_str("JAVA_MAVEN_THREADS", "1C")
+# Java 镜像 build/push 最大并行数。
+JAVA_IMAGE_BUILD_MAX_WORKERS = env_int("JAVA_IMAGE_BUILD_MAX_WORKERS", 3)

 # TAT 所在区域。
 DEPLOY_REGION = env_str("DEPLOY_REGION", "me-saudi-arabia")
--- a/monitor/service_release.py
+++ b/monitor/service_release.py
@ -1,5 +1,6 @@
 from __future__ import annotations

+from concurrent.futures import ThreadPoolExecutor, as_completed
 # 镜像 tag 需要拼时间戳。
 from datetime import datetime
 # 本地构建和 git worktree 都依赖标准库 subprocess / tempfile。
@ -26,10 +27,13 @@ from .config import (
    HARBOR_PROJECT,
    HARBOR_REGISTRY,
    HARBOR_USERNAME,
+    JAVA_IMAGE_BUILD_MAX_WORKERS,
+    JAVA_MAVEN_THREADS,
    JAVA_REPO_ROOT,
    ROOT,
    ROLLING_RESTART_STABILIZE_SECONDS,
    ROLLING_RESTART_TIMEOUT_SECONDS,
+    SERVICE_BUILD_CACHE_ROOT,
    SERVICE_BUILD_TIMEOUT_SECONDS,
    SERVICE_PULL_TIMEOUT_SECONDS,
    UPDATE_DEFAULT_GO_GIT_REF,
@ -365,6 +369,65 @@ def resolve_repo_commit(repo_root: Path, git_ref: str, label: str) -> str:
    raise RuntimeError(f"{label} git ref not found: {git_ref}")


+def build_cache_root(repo_root: Path) -> Path:
+    # 显式配置优先；未配置时按本地 / deploy 机目录结构推导默认缓存根目录。
+    override = str(SERVICE_BUILD_CACHE_ROOT or "").strip()
+    if override:
+        return Path(override).expanduser()
+    if str(repo_root).startswith("/opt/deploy/"):
+        return Path("/opt/deploy/build-cache")
+    return ROOT / "run" / "build_cache"
+
+
+def persistent_build_workspace_path(repo_root: Path, git_ref: str) -> Path:
+    # 同一仓库同一 ref 固定复用一份构建目录，最大化保留 target 缓存。
+    return build_cache_root(repo_root) / repo_root.name / sanitized_ref_fragment(git_ref)
+
+
+def prepare_persistent_build_workspace(
+    repo_root: Path,
+    git_ref: str,
+    commit: str,
+    label: str,
+    *,
+    progress: ProgressLogger | None = None,
+) -> Path:
+    # 固定构建工作区是专用缓存目录，允许 reset --hard 到目标提交。
+    workspace_root = persistent_build_workspace_path(repo_root, git_ref)
+    workspace_root.parent.mkdir(parents=True, exist_ok=True)
+
+    if not (workspace_root / ".git").exists():
+        if workspace_root.exists():
+            shutil.rmtree(workspace_root, ignore_errors=True)
+        local_command_output(
+            ["git", "clone", "--no-checkout", str(repo_root), str(workspace_root)],
+            cwd=workspace_root.parent,
+            label=f"{label} clone build workspace",
+            timeout_seconds=600,
+        )
+
+    local_command_output(
+        ["git", "-C", str(workspace_root), "remote", "set-url", "origin", str(repo_root)],
+        cwd=workspace_root,
+        label=f"{label} set build workspace origin",
+        timeout_seconds=120,
+    )
+    local_command_output(
+        ["git", "-C", str(workspace_root), "fetch", "origin", "--tags", "--prune"],
+        cwd=workspace_root,
+        label=f"{label} fetch build workspace",
+        timeout_seconds=300,
+    )
+    local_command_output(
+        ["git", "-C", str(workspace_root), "reset", "--hard", commit],
+        cwd=workspace_root,
+        label=f"{label} reset build workspace",
+        timeout_seconds=300,
+    )
+    emit_progress(progress, f"build.{label}", f"已准备固定构建工作区：{workspace_root} -> {commit[:12]}")
+    return workspace_root
+
+
 def create_temp_worktree(repo_root: Path, commit: str, label: str) -> Path:
    # 用临时 worktree 构建，避免直接改业务仓当前工作区。
    worktree_root = Path(tempfile.mkdtemp(prefix=f"hy-app-monitor-{repo_root.name}-"))
@ -489,6 +552,70 @@ def maven_module_list(service_names: list[str]) -> str:
    return ",".join(modules)


+def java_maven_package_command(service_names: list[str]) -> list[str]:
+    # Java 默认走增量 package，并允许通过 .env 配 Maven 并行线程。
+    command = ["mvn"]
+    maven_threads = str(JAVA_MAVEN_THREADS or "").strip()
+    if maven_threads:
+        command.extend(["-T", maven_threads])
+    command.extend(
+        [
+            "-pl",
+            maven_module_list(service_names),
+            "-am",
+            "-Dmaven.test.skip=true",
+            "package",
+        ]
+    )
+    return command
+
+
+def build_single_java_image(
+    service_name: str,
+    *,
+    worktree_root: Path,
+    short_commit: str,
+    resolved_image_tag: str,
+    progress: ProgressLogger | None = None,
+) -> dict[str, Any]:
+    # Maven 产物已经在固定工作区准备好，这里只负责单服务镜像构建和推送。
+    repository_ref = service_repository_ref(service_name)
+    target_image_ref = f"{repository_ref}:{resolved_image_tag}"
+    emit_progress(progress, f"build.java.{service_name}", f"开始构建镜像：{target_image_ref}")
+    build_tail = run_logged_command(
+        [str(worktree_root / "ci" / "build-service-image.sh"), service_name, target_image_ref, short_commit],
+        cwd=worktree_root,
+        label=f"java build {service_name}",
+        env={
+            "DOCKER_PLATFORM": BUILD_DOCKER_PLATFORM,
+            # 外层已经统一决定过是否 Maven 打包；镜像阶段一律复用现成产物。
+            "SKIP_MAVEN_BUILD": "1",
+        },
+        timeout_seconds=SERVICE_BUILD_TIMEOUT_SECONDS,
+        progress_stage=f"build.java.{service_name}.build",
+        progress=progress,
+    )
+    emit_progress(progress, f"build.java.{service_name}", f"开始推送 Harbor：{target_image_ref}")
+    push_tail = run_logged_command(
+        ["docker", "push", target_image_ref],
+        cwd=worktree_root,
+        label=f"java push {service_name}",
+        timeout_seconds=SERVICE_BUILD_TIMEOUT_SECONDS,
+        progress_stage=f"build.java.{service_name}.push",
+        progress=progress,
+    )
+    digest_ref = docker_digest(target_image_ref, cwd=worktree_root)
+    emit_progress(progress, f"build.java.{service_name}", f"镜像已推送完成：{digest_ref}")
+    return {
+        "service": service_name,
+        "repository": repository_ref,
+        "tag": target_image_ref,
+        "image": digest_ref,
+        "buildTail": build_tail,
+        "pushTail": push_tail,
+    }
+
+
 def build_java_images(
    service_names: list[str],
    git_ref: str,
@ -504,8 +631,13 @@ def build_java_images(
    fetch_repo_refs(repo_root, "java")
    commit = resolve_repo_commit(repo_root, resolved_ref, "java")
    emit_progress(progress, "build.java", f"Java ref 已解析：{resolved_ref} -> {commit[:12]}")
-    worktree_root = create_temp_worktree(repo_root, commit, "java")
-    emit_progress(progress, "build.java", f"已创建 Java 临时 worktree：{worktree_root}")
+    worktree_root = prepare_persistent_build_workspace(
+        repo_root,
+        resolved_ref,
+        commit,
+        "java",
+        progress=progress,
+    )
    short_commit = commit[:12]
    resolved_image_tag = derive_image_tag(resolved_ref, commit, image_tag)

@ -522,15 +654,7 @@ def build_java_images(
            )
            emit_progress(progress, "build.java.maven", f"开始 Maven 打包：{maven_module_list(service_names)}")
            run_logged_command(
-                [
-                    "mvn",
-                    "-pl",
-                    maven_module_list(service_names),
-                    "-am",
-                    "-Dmaven.test.skip=true",
-                    "clean",
-                    "package",
-                ],
+                java_maven_package_command(service_names),
                cwd=worktree_root,
                label="java maven package",
                timeout_seconds=SERVICE_BUILD_TIMEOUT_SECONDS,
@ -540,48 +664,51 @@ def build_java_images(
        else:
            emit_progress(progress, "build.java.maven", "跳过 Maven，直接复用已有 Jar")

-        for service_name in service_names:
-            repository_ref = service_repository_ref(service_name)
-            target_image_ref = f"{repository_ref}:{resolved_image_tag}"
-            emit_progress(progress, f"build.java.{service_name}", f"开始构建镜像：{target_image_ref}")
-            build_tail = run_logged_command(
-                [str(worktree_root / "ci" / "build-service-image.sh"), service_name, target_image_ref, short_commit],
-                cwd=worktree_root,
-                label=f"java build {service_name}",
-                env={
-                    "DOCKER_PLATFORM": BUILD_DOCKER_PLATFORM,
-                    # 外层已经统一决定过是否 Maven 打包；镜像阶段一律复用现成产物。
-                    "SKIP_MAVEN_BUILD": "1",
-                },
-                timeout_seconds=SERVICE_BUILD_TIMEOUT_SECONDS,
-                progress_stage=f"build.java.{service_name}.build",
-                progress=progress,
-            )
-            emit_progress(progress, f"build.java.{service_name}", f"开始推送 Harbor：{target_image_ref}")
-            push_tail = run_logged_command(
-                ["docker", "push", target_image_ref],
-                cwd=worktree_root,
-                label=f"java push {service_name}",
-                timeout_seconds=SERVICE_BUILD_TIMEOUT_SECONDS,
-                progress_stage=f"build.java.{service_name}.push",
-                progress=progress,
-            )
-            digest_ref = docker_digest(target_image_ref, cwd=worktree_root)
-            emit_progress(progress, f"build.java.{service_name}", f"镜像已推送完成：{digest_ref}")
-            service_images[service_name] = digest_ref
-            outputs.append(
-                {
-                    "service": service_name,
-                    "repository": repository_ref,
-                    "tag": target_image_ref,
-                    "image": digest_ref,
-                    "buildTail": build_tail,
-                    "pushTail": push_tail,
+        max_workers = max(1, min(JAVA_IMAGE_BUILD_MAX_WORKERS, len(service_names)))
+        emit_progress(progress, "build.java", f"Java 镜像并行度：{max_workers}")
+        if max_workers == 1:
+            for service_name in service_names:
+                item = build_single_java_image(
+                    service_name,
+                    worktree_root=worktree_root,
+                    short_commit=short_commit,
+                    resolved_image_tag=resolved_image_tag,
+                    progress=progress,
+                )
+                service_images[service_name] = item["image"]
+                outputs.append(item)
+        else:
+            items_by_service: dict[str, dict[str, Any]] = {}
+            errors: list[str] = []
+            with ThreadPoolExecutor(max_workers=max_workers, thread_name_prefix="java-image-build") as executor:
+                future_map = {
+                    executor.submit(
+                        build_single_java_image,
+                        service_name,
+                        worktree_root=worktree_root,
+                        short_commit=short_commit,
+                        resolved_image_tag=resolved_image_tag,
+                        progress=progress,
+                    ): service_name
+                    for service_name in service_names
                }
-            )
+                for future in as_completed(future_map):
+                    service_name = future_map[future]
+                    try:
+                        item = future.result()
+                    except Exception as exc:  # noqa: BLE001
+                        errors.append(f"{service_name}: {exc}")
+                        emit_progress(progress, f"build.java.{service_name}", str(exc), level="error")
+                        continue
+                    items_by_service[service_name] = item
+                    service_images[service_name] = item["image"]
+
+            if errors:
+                raise RuntimeError("java image build failed: " + "; ".join(errors))
+
+            outputs = [items_by_service[service_name] for service_name in service_names]
    finally:
-        remove_temp_worktree(repo_root, worktree_root)
-        emit_progress(progress, "build.java", f"已清理 Java worktree：{worktree_root}")
+        emit_progress(progress, "build.java", f"固定构建工作区保留复用：{worktree_root}")

    return {
        "kind": "java",