diff --git a/.env.example b/.env.example index ae098e8..797f3e1 100644 --- a/.env.example +++ b/.env.example @@ -13,6 +13,9 @@ ROLLING_RESTART_POLL_SECONDS=2 ROLLING_RESTART_STABILIZE_SECONDS=2 SERVICE_BUILD_TIMEOUT_SECONDS=3600 SERVICE_PULL_TIMEOUT_SECONDS=600 +SERVICE_BUILD_CACHE_ROOT=/opt/deploy/build-cache +JAVA_MAVEN_THREADS=1C +JAVA_IMAGE_BUILD_MAX_WORKERS=3 NACOS_BASE_URL=http://10.2.1.17:8848 NACOS_NAMESPACE_ID=f7a8594e-090c-4830-a3ad-b5c0f9afcc07 NACOS_NAMESPACE_NAME=RedCircleServiceProd diff --git a/README.md b/README.md index 7ef0e7a..3b75364 100644 --- a/README.md +++ b/README.md @@ -48,6 +48,9 @@ HARBOR_USERNAME=*** HARBOR_PASSWORD=*** JAVA_REPO_ROOT=/opt/deploy/sources/chatapp3-java GOLANG_REPO_ROOT=/opt/deploy/sources/chatapp3-golang +SERVICE_BUILD_CACHE_ROOT=/opt/deploy/build-cache +JAVA_MAVEN_THREADS=1C +JAVA_IMAGE_BUILD_MAX_WORKERS=3 UPDATE_DEFAULT_JAVA_GIT_REF=main UPDATE_DEFAULT_GO_GIT_REF=main ``` @@ -135,7 +138,7 @@ python3 ops/sync_runtime_compose.py --sync-remote 页面里的“服务更新”会直接在当前机器上执行这条链路: -1. 从 `JAVA_REPO_ROOT` / `GOLANG_REPO_ROOT` 指向的业务仓按 `git ref` 创建临时 `git worktree` +1. Java 服务会在 `SERVICE_BUILD_CACHE_ROOT//` 下复用固定构建工作区;Golang 仍按临时 `git worktree` 构建 2. 本地构建镜像并推送到 Harbor 3. 取回 Harbor `digest` 4. 用新 `digest` 更新本地完整 compose 模板 @@ -146,6 +149,8 @@ python3 ops/sync_runtime_compose.py --sync-remote 说明: - `other` 现在也按 Harbor 业务镜像更新,不再继续依赖本地 `./other/service.jar` 挂载 +- Java Maven 默认走增量 `package`,并支持通过 `JAVA_MAVEN_THREADS` 开启 reactor 并行 +- 多个 Java 服务会按 `JAVA_IMAGE_BUILD_MAX_WORKERS` 受限并行执行 `docker build + push` - 页面里的“滚动重启”仍然保留,适合只想让配置生效、不涉及代码构建的场景 - 这条链路默认假设 `hy-app-monitor` 就运行在打包机 / deploy 机上,并且本机已经能访问业务仓、Docker 和 Harbor diff --git a/monitor/config.py b/monitor/config.py index 3f4088e..5ff5fd3 100644 --- a/monitor/config.py +++ b/monitor/config.py @@ -52,6 +52,12 @@ ROLLING_RESTART_STABILIZE_SECONDS = env_float("ROLLING_RESTART_STABILIZE_SECONDS SERVICE_BUILD_TIMEOUT_SECONDS = env_int("SERVICE_BUILD_TIMEOUT_SECONDS", 3600) # 服务更新时目标机预拉镜像最长等待秒数。 SERVICE_PULL_TIMEOUT_SECONDS = env_int("SERVICE_PULL_TIMEOUT_SECONDS", 600) +# 固定构建工作区根目录,留空时按本地 / deploy 机路径自动推导。 +SERVICE_BUILD_CACHE_ROOT = env_str("SERVICE_BUILD_CACHE_ROOT", "") +# Java Maven 并行线程,默认按 1 个 CPU 组装 reactor。 +JAVA_MAVEN_THREADS = env_str("JAVA_MAVEN_THREADS", "1C") +# Java 镜像 build/push 最大并行数。 +JAVA_IMAGE_BUILD_MAX_WORKERS = env_int("JAVA_IMAGE_BUILD_MAX_WORKERS", 3) # TAT 所在区域。 DEPLOY_REGION = env_str("DEPLOY_REGION", "me-saudi-arabia") diff --git a/monitor/service_release.py b/monitor/service_release.py index db0bcfc..91f4dd7 100644 --- a/monitor/service_release.py +++ b/monitor/service_release.py @@ -1,5 +1,6 @@ from __future__ import annotations +from concurrent.futures import ThreadPoolExecutor, as_completed # 镜像 tag 需要拼时间戳。 from datetime import datetime # 本地构建和 git worktree 都依赖标准库 subprocess / tempfile。 @@ -26,10 +27,13 @@ from .config import ( HARBOR_PROJECT, HARBOR_REGISTRY, HARBOR_USERNAME, + JAVA_IMAGE_BUILD_MAX_WORKERS, + JAVA_MAVEN_THREADS, JAVA_REPO_ROOT, ROOT, ROLLING_RESTART_STABILIZE_SECONDS, ROLLING_RESTART_TIMEOUT_SECONDS, + SERVICE_BUILD_CACHE_ROOT, SERVICE_BUILD_TIMEOUT_SECONDS, SERVICE_PULL_TIMEOUT_SECONDS, UPDATE_DEFAULT_GO_GIT_REF, @@ -365,6 +369,65 @@ def resolve_repo_commit(repo_root: Path, git_ref: str, label: str) -> str: raise RuntimeError(f"{label} git ref not found: {git_ref}") +def build_cache_root(repo_root: Path) -> Path: + # 显式配置优先;未配置时按本地 / deploy 机目录结构推导默认缓存根目录。 + override = str(SERVICE_BUILD_CACHE_ROOT or "").strip() + if override: + return Path(override).expanduser() + if str(repo_root).startswith("/opt/deploy/"): + return Path("/opt/deploy/build-cache") + return ROOT / "run" / "build_cache" + + +def persistent_build_workspace_path(repo_root: Path, git_ref: str) -> Path: + # 同一仓库同一 ref 固定复用一份构建目录,最大化保留 target 缓存。 + return build_cache_root(repo_root) / repo_root.name / sanitized_ref_fragment(git_ref) + + +def prepare_persistent_build_workspace( + repo_root: Path, + git_ref: str, + commit: str, + label: str, + *, + progress: ProgressLogger | None = None, +) -> Path: + # 固定构建工作区是专用缓存目录,允许 reset --hard 到目标提交。 + workspace_root = persistent_build_workspace_path(repo_root, git_ref) + workspace_root.parent.mkdir(parents=True, exist_ok=True) + + if not (workspace_root / ".git").exists(): + if workspace_root.exists(): + shutil.rmtree(workspace_root, ignore_errors=True) + local_command_output( + ["git", "clone", "--no-checkout", str(repo_root), str(workspace_root)], + cwd=workspace_root.parent, + label=f"{label} clone build workspace", + timeout_seconds=600, + ) + + local_command_output( + ["git", "-C", str(workspace_root), "remote", "set-url", "origin", str(repo_root)], + cwd=workspace_root, + label=f"{label} set build workspace origin", + timeout_seconds=120, + ) + local_command_output( + ["git", "-C", str(workspace_root), "fetch", "origin", "--tags", "--prune"], + cwd=workspace_root, + label=f"{label} fetch build workspace", + timeout_seconds=300, + ) + local_command_output( + ["git", "-C", str(workspace_root), "reset", "--hard", commit], + cwd=workspace_root, + label=f"{label} reset build workspace", + timeout_seconds=300, + ) + emit_progress(progress, f"build.{label}", f"已准备固定构建工作区:{workspace_root} -> {commit[:12]}") + return workspace_root + + def create_temp_worktree(repo_root: Path, commit: str, label: str) -> Path: # 用临时 worktree 构建,避免直接改业务仓当前工作区。 worktree_root = Path(tempfile.mkdtemp(prefix=f"hy-app-monitor-{repo_root.name}-")) @@ -489,6 +552,70 @@ def maven_module_list(service_names: list[str]) -> str: return ",".join(modules) +def java_maven_package_command(service_names: list[str]) -> list[str]: + # Java 默认走增量 package,并允许通过 .env 配 Maven 并行线程。 + command = ["mvn"] + maven_threads = str(JAVA_MAVEN_THREADS or "").strip() + if maven_threads: + command.extend(["-T", maven_threads]) + command.extend( + [ + "-pl", + maven_module_list(service_names), + "-am", + "-Dmaven.test.skip=true", + "package", + ] + ) + return command + + +def build_single_java_image( + service_name: str, + *, + worktree_root: Path, + short_commit: str, + resolved_image_tag: str, + progress: ProgressLogger | None = None, +) -> dict[str, Any]: + # Maven 产物已经在固定工作区准备好,这里只负责单服务镜像构建和推送。 + repository_ref = service_repository_ref(service_name) + target_image_ref = f"{repository_ref}:{resolved_image_tag}" + emit_progress(progress, f"build.java.{service_name}", f"开始构建镜像:{target_image_ref}") + build_tail = run_logged_command( + [str(worktree_root / "ci" / "build-service-image.sh"), service_name, target_image_ref, short_commit], + cwd=worktree_root, + label=f"java build {service_name}", + env={ + "DOCKER_PLATFORM": BUILD_DOCKER_PLATFORM, + # 外层已经统一决定过是否 Maven 打包;镜像阶段一律复用现成产物。 + "SKIP_MAVEN_BUILD": "1", + }, + timeout_seconds=SERVICE_BUILD_TIMEOUT_SECONDS, + progress_stage=f"build.java.{service_name}.build", + progress=progress, + ) + emit_progress(progress, f"build.java.{service_name}", f"开始推送 Harbor:{target_image_ref}") + push_tail = run_logged_command( + ["docker", "push", target_image_ref], + cwd=worktree_root, + label=f"java push {service_name}", + timeout_seconds=SERVICE_BUILD_TIMEOUT_SECONDS, + progress_stage=f"build.java.{service_name}.push", + progress=progress, + ) + digest_ref = docker_digest(target_image_ref, cwd=worktree_root) + emit_progress(progress, f"build.java.{service_name}", f"镜像已推送完成:{digest_ref}") + return { + "service": service_name, + "repository": repository_ref, + "tag": target_image_ref, + "image": digest_ref, + "buildTail": build_tail, + "pushTail": push_tail, + } + + def build_java_images( service_names: list[str], git_ref: str, @@ -504,8 +631,13 @@ def build_java_images( fetch_repo_refs(repo_root, "java") commit = resolve_repo_commit(repo_root, resolved_ref, "java") emit_progress(progress, "build.java", f"Java ref 已解析:{resolved_ref} -> {commit[:12]}") - worktree_root = create_temp_worktree(repo_root, commit, "java") - emit_progress(progress, "build.java", f"已创建 Java 临时 worktree:{worktree_root}") + worktree_root = prepare_persistent_build_workspace( + repo_root, + resolved_ref, + commit, + "java", + progress=progress, + ) short_commit = commit[:12] resolved_image_tag = derive_image_tag(resolved_ref, commit, image_tag) @@ -522,15 +654,7 @@ def build_java_images( ) emit_progress(progress, "build.java.maven", f"开始 Maven 打包:{maven_module_list(service_names)}") run_logged_command( - [ - "mvn", - "-pl", - maven_module_list(service_names), - "-am", - "-Dmaven.test.skip=true", - "clean", - "package", - ], + java_maven_package_command(service_names), cwd=worktree_root, label="java maven package", timeout_seconds=SERVICE_BUILD_TIMEOUT_SECONDS, @@ -540,48 +664,51 @@ def build_java_images( else: emit_progress(progress, "build.java.maven", "跳过 Maven,直接复用已有 Jar") - for service_name in service_names: - repository_ref = service_repository_ref(service_name) - target_image_ref = f"{repository_ref}:{resolved_image_tag}" - emit_progress(progress, f"build.java.{service_name}", f"开始构建镜像:{target_image_ref}") - build_tail = run_logged_command( - [str(worktree_root / "ci" / "build-service-image.sh"), service_name, target_image_ref, short_commit], - cwd=worktree_root, - label=f"java build {service_name}", - env={ - "DOCKER_PLATFORM": BUILD_DOCKER_PLATFORM, - # 外层已经统一决定过是否 Maven 打包;镜像阶段一律复用现成产物。 - "SKIP_MAVEN_BUILD": "1", - }, - timeout_seconds=SERVICE_BUILD_TIMEOUT_SECONDS, - progress_stage=f"build.java.{service_name}.build", - progress=progress, - ) - emit_progress(progress, f"build.java.{service_name}", f"开始推送 Harbor:{target_image_ref}") - push_tail = run_logged_command( - ["docker", "push", target_image_ref], - cwd=worktree_root, - label=f"java push {service_name}", - timeout_seconds=SERVICE_BUILD_TIMEOUT_SECONDS, - progress_stage=f"build.java.{service_name}.push", - progress=progress, - ) - digest_ref = docker_digest(target_image_ref, cwd=worktree_root) - emit_progress(progress, f"build.java.{service_name}", f"镜像已推送完成:{digest_ref}") - service_images[service_name] = digest_ref - outputs.append( - { - "service": service_name, - "repository": repository_ref, - "tag": target_image_ref, - "image": digest_ref, - "buildTail": build_tail, - "pushTail": push_tail, + max_workers = max(1, min(JAVA_IMAGE_BUILD_MAX_WORKERS, len(service_names))) + emit_progress(progress, "build.java", f"Java 镜像并行度:{max_workers}") + if max_workers == 1: + for service_name in service_names: + item = build_single_java_image( + service_name, + worktree_root=worktree_root, + short_commit=short_commit, + resolved_image_tag=resolved_image_tag, + progress=progress, + ) + service_images[service_name] = item["image"] + outputs.append(item) + else: + items_by_service: dict[str, dict[str, Any]] = {} + errors: list[str] = [] + with ThreadPoolExecutor(max_workers=max_workers, thread_name_prefix="java-image-build") as executor: + future_map = { + executor.submit( + build_single_java_image, + service_name, + worktree_root=worktree_root, + short_commit=short_commit, + resolved_image_tag=resolved_image_tag, + progress=progress, + ): service_name + for service_name in service_names } - ) + for future in as_completed(future_map): + service_name = future_map[future] + try: + item = future.result() + except Exception as exc: # noqa: BLE001 + errors.append(f"{service_name}: {exc}") + emit_progress(progress, f"build.java.{service_name}", str(exc), level="error") + continue + items_by_service[service_name] = item + service_images[service_name] = item["image"] + + if errors: + raise RuntimeError("java image build failed: " + "; ".join(errors)) + + outputs = [items_by_service[service_name] for service_name in service_names] finally: - remove_temp_worktree(repo_root, worktree_root) - emit_progress(progress, "build.java", f"已清理 Java worktree:{worktree_root}") + emit_progress(progress, "build.java", f"固定构建工作区保留复用:{worktree_root}") return { "kind": "java",