This commit is contained in:
hy 2026-04-22 18:57:31 +08:00
parent 34b976b098
commit a7d8b5ebfb
7 changed files with 699 additions and 28 deletions

View File

@ -15,7 +15,8 @@ from .nacos import (
validate_nacos_config_payload, validate_nacos_config_payload,
) )
from .runtime_config import get_go_config_payload, save_go_config_payload, validate_go_config_payload from .runtime_config import get_go_config_payload, save_go_config_payload, validate_go_config_payload
from .service_release import build_update_targets_payload, deploy_updated_services_payload from .service_release import build_update_targets_payload
from .service_update_jobs import get_update_job_payload, start_update_job
from .service_ops import build_restart_targets_payload, rolling_restart_payload from .service_ops import build_restart_targets_payload, rolling_restart_payload
@ -164,6 +165,30 @@ class MonitorHandler(BaseHTTPRequestHandler):
send_body=send_body, send_body=send_body,
) )
# 返回当前服务更新任务的阶段和日志。
if parsed.path == "/api/ops/update-status":
operation_id = self.query_value(query, "id")
if not operation_id:
return self.send_error_payload(
HTTPStatus.BAD_REQUEST,
"id is required",
send_body=send_body,
)
try:
payload = get_update_job_payload(operation_id)
except KeyError:
return self.send_error_payload(HTTPStatus.NOT_FOUND, "update operation not found", send_body=send_body)
except Exception as exc: # noqa: BLE001
return self.send_error_payload(HTTPStatus.BAD_GATEWAY, str(exc), send_body=send_body)
return self.send_payload(
HTTPStatus.OK,
response_json(payload),
"application/json; charset=utf-8",
send_body=send_body,
)
# favicon 直接返回空。 # favicon 直接返回空。
if parsed.path == "/favicon.ico": if parsed.path == "/favicon.ico":
return self.send_payload(HTTPStatus.NO_CONTENT, b"", "image/x-icon", send_body=send_body) return self.send_payload(HTTPStatus.NO_CONTENT, b"", "image/x-icon", send_body=send_body)
@ -313,7 +338,7 @@ class MonitorHandler(BaseHTTPRequestHandler):
skip_maven_build = bool(payload.get("skipMavenBuild")) skip_maven_build = bool(payload.get("skipMavenBuild"))
try: try:
result = deploy_updated_services_payload( result = start_update_job(
services, services,
java_git_ref=java_git_ref, java_git_ref=java_git_ref,
go_git_ref=go_git_ref, go_git_ref=go_git_ref,
@ -324,7 +349,7 @@ class MonitorHandler(BaseHTTPRequestHandler):
return self.send_error_payload(HTTPStatus.BAD_GATEWAY, str(exc), send_body=True) return self.send_error_payload(HTTPStatus.BAD_GATEWAY, str(exc), send_body=True)
return self.send_payload( return self.send_payload(
HTTPStatus.OK, HTTPStatus.ACCEPTED,
response_json(result), response_json(result),
"application/json; charset=utf-8", "application/json; charset=utf-8",
send_body=True, send_body=True,

View File

@ -665,14 +665,31 @@ def build_nacos_payload() -> dict[str, Any]:
# 统计全健康服务数量。 # 统计全健康服务数量。
healthy_service_total = 0 healthy_service_total = 0
# 逐个 namespace 读取服务和实例。 # 只对当前目标 namespace 构建服务和实例视图,避免把 public 下的空查询混进来。
for namespace in normalized_namespaces: target_namespaces = [
namespace
for namespace in normalized_namespaces
if namespace["namespaceId"] == namespace_id or namespace["name"] == namespace_name
]
if not target_namespaces:
target_namespaces = [
{
"namespaceId": namespace_id,
"name": namespace_name,
"configCount": 0,
"type": 0,
"visible": False,
}
]
# 逐个目标 namespace 读取服务和实例。
for namespace in target_namespaces:
# 当前 namespace 的自动发现结果。 # 当前 namespace 的自动发现结果。
raw_services = list_services(namespace["namespaceId"]) raw_services = list_services(namespace["namespaceId"])
# 这里保存当前 namespace 下所有待查询服务。 # 这里保存当前 namespace 下所有待查询服务。
queries_by_key: dict[tuple[str, str], dict[str, str]] = {} queries_by_key: dict[tuple[str, str], dict[str, str]] = {}
# 只在可见 namespace或者根本没有可见 namespace 时,才补 fallback 查询。 # 当前目标 namespace 即使 service/list 为空,也要把已知注册服务补查一遍
allow_fallback_queries = namespace["visible"] or not any(item["visible"] for item in normalized_namespaces) allow_fallback_queries = True
# 先收 service/list 返回的服务。 # 先收 service/list 返回的服务。
for raw_name in raw_services: for raw_name in raw_services:

View File

@ -10,8 +10,9 @@ import shlex
import shutil import shutil
import subprocess import subprocess
import tempfile import tempfile
import threading
import time import time
from typing import Any from typing import Any, Callable
from .compose_templates import ( from .compose_templates import (
load_local_host_compose_model, load_local_host_compose_model,
@ -72,6 +73,60 @@ SERVICE_IMAGE_REPOSITORY_SUFFIXES = {
"golang": "golang", "golang": "golang",
} }
# 后台任务的日志回调统一走这个签名。
ProgressLogger = Callable[[str, str, str], None]
def emit_progress(progress: ProgressLogger | None, stage: str, message: str, *, level: str = "info") -> None:
# 所有过程日志都经这里发出,调用方传空时自动静默。
if progress is None:
return
progress(str(stage or "").strip(), str(level or "info").strip() or "info", str(message or "").rstrip())
def short_text(text: str, limit: int = 600) -> str:
# 远端输出和异常信息只保留一小段,避免页面日志爆掉。
normalized = " ".join(str(text or "").split())
if len(normalized) <= limit:
return normalized
return normalized[:limit].rstrip() + " ..."
def stream_log_file_lines(
log_path: Path,
stop_event: threading.Event,
*,
stage: str,
progress: ProgressLogger | None,
) -> None:
# 构建日志落本地文件后,这里持续追尾并把新增行推给前端。
if progress is None:
return
position = 0
remainder = ""
while not stop_event.is_set():
if log_path.exists():
with log_path.open("r", encoding="utf-8", errors="replace") as handle:
handle.seek(position)
chunk = handle.read()
position = handle.tell()
if chunk:
remainder += chunk
lines = remainder.splitlines(keepends=True)
remainder = ""
if lines and not lines[-1].endswith(("\n", "\r")):
remainder = lines.pop()
for line in lines:
text = line.rstrip()
if text:
emit_progress(progress, stage, text)
stop_event.wait(0.4)
if remainder.strip():
emit_progress(progress, stage, remainder.rstrip())
def buildable_service_names() -> list[str]: def buildable_service_names() -> list[str]:
# 页面和接口统一按这里判定哪些服务允许“构建并更新”。 # 页面和接口统一按这里判定哪些服务允许“构建并更新”。
@ -152,6 +207,8 @@ def run_logged_command(
label: str, label: str,
env: dict[str, str] | None = None, env: dict[str, str] | None = None,
timeout_seconds: int = SERVICE_BUILD_TIMEOUT_SECONDS, timeout_seconds: int = SERVICE_BUILD_TIMEOUT_SECONDS,
progress_stage: str = "",
progress: ProgressLogger | None = None,
) -> str: ) -> str:
# Maven / docker build 输出很大,单独落临时日志再取尾部更稳。 # Maven / docker build 输出很大,单独落临时日志再取尾部更稳。
merged_env = os.environ.copy() merged_env = os.environ.copy()
@ -163,6 +220,9 @@ def run_logged_command(
log_file.close() log_file.close()
try: try:
stop_event = threading.Event()
follower: threading.Thread | None = None
with log_path.open("w", encoding="utf-8") as handle: with log_path.open("w", encoding="utf-8") as handle:
process = subprocess.Popen( process = subprocess.Popen(
command, command,
@ -172,12 +232,27 @@ def run_logged_command(
text=True, text=True,
env=merged_env, env=merged_env,
) )
if progress is not None:
follower = threading.Thread(
target=stream_log_file_lines,
args=(log_path, stop_event),
kwargs={"stage": progress_stage or label, "progress": progress},
daemon=True,
)
follower.start()
try: try:
return_code = process.wait(timeout=timeout_seconds) return_code = process.wait(timeout=timeout_seconds)
except subprocess.TimeoutExpired as exc: except subprocess.TimeoutExpired as exc:
process.kill() process.kill()
process.wait() process.wait()
stop_event.set()
if follower is not None:
follower.join(timeout=1)
raise RuntimeError(f"{label} timed out: {tail_text(log_path)}") from exc raise RuntimeError(f"{label} timed out: {tail_text(log_path)}") from exc
finally:
stop_event.set()
if follower is not None:
follower.join(timeout=1)
output_tail = tail_text(log_path) output_tail = tail_text(log_path)
if return_code != 0: if return_code != 0:
@ -253,9 +328,10 @@ def require_harbor_credentials() -> None:
raise RuntimeError("missing HARBOR_USERNAME / HARBOR_PASSWORD in .env") raise RuntimeError("missing HARBOR_USERNAME / HARBOR_PASSWORD in .env")
def docker_login_local(repo_root: Path) -> None: def docker_login_local(repo_root: Path, *, progress: ProgressLogger | None = None) -> None:
# 本机构建前先登录 Harbor避免 push 时才失败。 # 本机构建前先登录 Harbor避免 push 时才失败。
require_harbor_credentials() require_harbor_credentials()
emit_progress(progress, "prepare", f"登录 Harbor{HARBOR_REGISTRY}")
local_command_output( local_command_output(
["docker", "login", HARBOR_REGISTRY, "-u", HARBOR_USERNAME, "--password-stdin"], ["docker", "login", HARBOR_REGISTRY, "-u", HARBOR_USERNAME, "--password-stdin"],
cwd=repo_root, cwd=repo_root,
@ -263,6 +339,7 @@ def docker_login_local(repo_root: Path) -> None:
input_text=HARBOR_PASSWORD, input_text=HARBOR_PASSWORD,
timeout_seconds=120, timeout_seconds=120,
) )
emit_progress(progress, "prepare", "Harbor 登录成功")
def image_repository_ref(image_ref: str) -> str: def image_repository_ref(image_ref: str) -> str:
@ -338,13 +415,23 @@ def maven_module_list(service_names: list[str]) -> str:
return ",".join(modules) return ",".join(modules)
def build_java_images(service_names: list[str], git_ref: str, image_tag: str, *, skip_maven_build: bool) -> dict[str, Any]: def build_java_images(
service_names: list[str],
git_ref: str,
image_tag: str,
*,
skip_maven_build: bool,
progress: ProgressLogger | None = None,
) -> dict[str, Any]:
# Java 服务共享一个仓库和 Maven 本地缓存,这里按仓统一构建。 # Java 服务共享一个仓库和 Maven 本地缓存,这里按仓统一构建。
repo_root = ensure_repo_root(JAVA_REPO_ROOT, "java") repo_root = ensure_repo_root(JAVA_REPO_ROOT, "java")
resolved_ref = str(git_ref or "").strip() or UPDATE_DEFAULT_JAVA_GIT_REF resolved_ref = str(git_ref or "").strip() or UPDATE_DEFAULT_JAVA_GIT_REF
emit_progress(progress, "build.java", f"刷新 Java 仓库 refs{resolved_ref}")
fetch_repo_refs(repo_root, "java") fetch_repo_refs(repo_root, "java")
commit = resolve_repo_commit(repo_root, resolved_ref, "java") commit = resolve_repo_commit(repo_root, resolved_ref, "java")
emit_progress(progress, "build.java", f"Java ref 已解析:{resolved_ref} -> {commit[:12]}")
worktree_root = create_temp_worktree(repo_root, commit, "java") worktree_root = create_temp_worktree(repo_root, commit, "java")
emit_progress(progress, "build.java", f"已创建 Java 临时 worktree{worktree_root}")
short_commit = commit[:12] short_commit = commit[:12]
resolved_image_tag = derive_image_tag(resolved_ref, commit, image_tag) resolved_image_tag = derive_image_tag(resolved_ref, commit, image_tag)
@ -353,12 +440,16 @@ def build_java_images(service_names: list[str], git_ref: str, image_tag: str, *,
try: try:
if not skip_maven_build: if not skip_maven_build:
emit_progress(progress, "build.java.maven", "开始安装私有 Maven 依赖")
run_logged_command( run_logged_command(
[str(worktree_root / "ci" / "install-private-maven.sh")], [str(worktree_root / "ci" / "install-private-maven.sh")],
cwd=worktree_root, cwd=worktree_root,
label="java install private maven", label="java install private maven",
timeout_seconds=min(SERVICE_BUILD_TIMEOUT_SECONDS, 1200), timeout_seconds=min(SERVICE_BUILD_TIMEOUT_SECONDS, 1200),
progress_stage="build.java.maven.install",
progress=progress,
) )
emit_progress(progress, "build.java.maven", f"开始 Maven 打包:{maven_module_list(service_names)}")
run_logged_command( run_logged_command(
[ [
"mvn", "mvn",
@ -372,28 +463,39 @@ def build_java_images(service_names: list[str], git_ref: str, image_tag: str, *,
cwd=worktree_root, cwd=worktree_root,
label="java maven package", label="java maven package",
timeout_seconds=SERVICE_BUILD_TIMEOUT_SECONDS, timeout_seconds=SERVICE_BUILD_TIMEOUT_SECONDS,
progress_stage="build.java.maven.package",
progress=progress,
) )
else:
emit_progress(progress, "build.java.maven", "跳过 Maven直接复用已有 Jar")
for service_name in service_names: for service_name in service_names:
repository_ref = service_repository_ref(service_name) repository_ref = service_repository_ref(service_name)
target_image_ref = f"{repository_ref}:{resolved_image_tag}" target_image_ref = f"{repository_ref}:{resolved_image_tag}"
emit_progress(progress, f"build.java.{service_name}", f"开始构建镜像:{target_image_ref}")
build_tail = run_logged_command( build_tail = run_logged_command(
[str(worktree_root / "ci" / "build-service-image.sh"), service_name, target_image_ref, short_commit], [str(worktree_root / "ci" / "build-service-image.sh"), service_name, target_image_ref, short_commit],
cwd=worktree_root, cwd=worktree_root,
label=f"java build {service_name}", label=f"java build {service_name}",
env={ env={
"DOCKER_PLATFORM": BUILD_DOCKER_PLATFORM, "DOCKER_PLATFORM": BUILD_DOCKER_PLATFORM,
"SKIP_MAVEN_BUILD": "1" if not skip_maven_build else "1", "SKIP_MAVEN_BUILD": "1" if skip_maven_build else "0",
}, },
timeout_seconds=SERVICE_BUILD_TIMEOUT_SECONDS, timeout_seconds=SERVICE_BUILD_TIMEOUT_SECONDS,
progress_stage=f"build.java.{service_name}.build",
progress=progress,
) )
emit_progress(progress, f"build.java.{service_name}", f"开始推送 Harbor{target_image_ref}")
push_tail = run_logged_command( push_tail = run_logged_command(
["docker", "push", target_image_ref], ["docker", "push", target_image_ref],
cwd=worktree_root, cwd=worktree_root,
label=f"java push {service_name}", label=f"java push {service_name}",
timeout_seconds=SERVICE_BUILD_TIMEOUT_SECONDS, timeout_seconds=SERVICE_BUILD_TIMEOUT_SECONDS,
progress_stage=f"build.java.{service_name}.push",
progress=progress,
) )
digest_ref = docker_digest(target_image_ref, cwd=worktree_root) digest_ref = docker_digest(target_image_ref, cwd=worktree_root)
emit_progress(progress, f"build.java.{service_name}", f"镜像已推送完成:{digest_ref}")
service_images[service_name] = digest_ref service_images[service_name] = digest_ref
outputs.append( outputs.append(
{ {
@ -407,6 +509,7 @@ def build_java_images(service_names: list[str], git_ref: str, image_tag: str, *,
) )
finally: finally:
remove_temp_worktree(repo_root, worktree_root) remove_temp_worktree(repo_root, worktree_root)
emit_progress(progress, "build.java", f"已清理 Java worktree{worktree_root}")
return { return {
"kind": "java", "kind": "java",
@ -420,13 +523,22 @@ def build_java_images(service_names: list[str], git_ref: str, image_tag: str, *,
} }
def build_golang_images(service_names: list[str], git_ref: str, image_tag: str) -> dict[str, Any]: def build_golang_images(
service_names: list[str],
git_ref: str,
image_tag: str,
*,
progress: ProgressLogger | None = None,
) -> dict[str, Any]:
# Go 服务当前是单仓构建,逻辑更简单。 # Go 服务当前是单仓构建,逻辑更简单。
repo_root = ensure_repo_root(GOLANG_REPO_ROOT, "golang") repo_root = ensure_repo_root(GOLANG_REPO_ROOT, "golang")
resolved_ref = str(git_ref or "").strip() or UPDATE_DEFAULT_GO_GIT_REF resolved_ref = str(git_ref or "").strip() or UPDATE_DEFAULT_GO_GIT_REF
emit_progress(progress, "build.golang", f"刷新 Go 仓库 refs{resolved_ref}")
fetch_repo_refs(repo_root, "golang") fetch_repo_refs(repo_root, "golang")
commit = resolve_repo_commit(repo_root, resolved_ref, "golang") commit = resolve_repo_commit(repo_root, resolved_ref, "golang")
emit_progress(progress, "build.golang", f"Go ref 已解析:{resolved_ref} -> {commit[:12]}")
worktree_root = create_temp_worktree(repo_root, commit, "golang") worktree_root = create_temp_worktree(repo_root, commit, "golang")
emit_progress(progress, "build.golang", f"已创建 Go 临时 worktree{worktree_root}")
short_commit = commit[:12] short_commit = commit[:12]
resolved_image_tag = derive_image_tag(resolved_ref, commit, image_tag) resolved_image_tag = derive_image_tag(resolved_ref, commit, image_tag)
@ -437,20 +549,27 @@ def build_golang_images(service_names: list[str], git_ref: str, image_tag: str)
for service_name in service_names: for service_name in service_names:
repository_ref = service_repository_ref(service_name) repository_ref = service_repository_ref(service_name)
target_image_ref = f"{repository_ref}:{resolved_image_tag}" target_image_ref = f"{repository_ref}:{resolved_image_tag}"
emit_progress(progress, f"build.golang.{service_name}", f"开始构建镜像:{target_image_ref}")
build_tail = run_logged_command( build_tail = run_logged_command(
[str(worktree_root / "ci" / "build-image.sh"), target_image_ref, short_commit], [str(worktree_root / "ci" / "build-image.sh"), target_image_ref, short_commit],
cwd=worktree_root, cwd=worktree_root,
label=f"golang build {service_name}", label=f"golang build {service_name}",
env={"DOCKER_PLATFORM": BUILD_DOCKER_PLATFORM}, env={"DOCKER_PLATFORM": BUILD_DOCKER_PLATFORM},
timeout_seconds=SERVICE_BUILD_TIMEOUT_SECONDS, timeout_seconds=SERVICE_BUILD_TIMEOUT_SECONDS,
progress_stage=f"build.golang.{service_name}.build",
progress=progress,
) )
emit_progress(progress, f"build.golang.{service_name}", f"开始推送 Harbor{target_image_ref}")
push_tail = run_logged_command( push_tail = run_logged_command(
["docker", "push", target_image_ref], ["docker", "push", target_image_ref],
cwd=worktree_root, cwd=worktree_root,
label=f"golang push {service_name}", label=f"golang push {service_name}",
timeout_seconds=SERVICE_BUILD_TIMEOUT_SECONDS, timeout_seconds=SERVICE_BUILD_TIMEOUT_SECONDS,
progress_stage=f"build.golang.{service_name}.push",
progress=progress,
) )
digest_ref = docker_digest(target_image_ref, cwd=worktree_root) digest_ref = docker_digest(target_image_ref, cwd=worktree_root)
emit_progress(progress, f"build.golang.{service_name}", f"镜像已推送完成:{digest_ref}")
service_images[service_name] = digest_ref service_images[service_name] = digest_ref
outputs.append( outputs.append(
{ {
@ -464,6 +583,7 @@ def build_golang_images(service_names: list[str], git_ref: str, image_tag: str)
) )
finally: finally:
remove_temp_worktree(repo_root, worktree_root) remove_temp_worktree(repo_root, worktree_root)
emit_progress(progress, "build.golang", f"已清理 Go worktree{worktree_root}")
return { return {
"kind": "golang", "kind": "golang",
@ -484,6 +604,7 @@ def build_service_images(
go_git_ref: str, go_git_ref: str,
image_tag: str, image_tag: str,
skip_maven_build: bool, skip_maven_build: bool,
progress: ProgressLogger | None = None,
) -> dict[str, Any]: ) -> dict[str, Any]:
# 一个请求里可能同时选了 Java 和 Go需要分仓构建后再合并 digest。 # 一个请求里可能同时选了 Java 和 Go需要分仓构建后再合并 digest。
normalized = normalize_service_names(service_names) normalized = normalize_service_names(service_names)
@ -493,16 +614,25 @@ def build_service_images(
if not normalized: if not normalized:
raise RuntimeError("services are required") raise RuntimeError("services are required")
docker_login_local(ensure_repo_root(JAVA_REPO_ROOT if java_services else GOLANG_REPO_ROOT, "build")) docker_login_local(
ensure_repo_root(JAVA_REPO_ROOT if java_services else GOLANG_REPO_ROOT, "build"),
progress=progress,
)
builds: list[dict[str, Any]] = [] builds: list[dict[str, Any]] = []
service_images: dict[str, str] = {} service_images: dict[str, str] = {}
if java_services: if java_services:
java_build = build_java_images(java_services, java_git_ref, image_tag, skip_maven_build=skip_maven_build) java_build = build_java_images(
java_services,
java_git_ref,
image_tag,
skip_maven_build=skip_maven_build,
progress=progress,
)
builds.append(java_build) builds.append(java_build)
service_images.update(java_build["serviceImages"]) service_images.update(java_build["serviceImages"])
if golang_services: if golang_services:
golang_build = build_golang_images(golang_services, go_git_ref, image_tag) golang_build = build_golang_images(golang_services, go_git_ref, image_tag, progress=progress)
builds.append(golang_build) builds.append(golang_build)
service_images.update(golang_build["serviceImages"]) service_images.update(golang_build["serviceImages"])
@ -644,6 +774,7 @@ def deploy_updated_services_payload(
go_git_ref: str, go_git_ref: str,
image_tag: str, image_tag: str,
skip_maven_build: bool, skip_maven_build: bool,
progress: ProgressLogger | None = None,
) -> dict[str, Any]: ) -> dict[str, Any]:
# 这是“服务更新”主链路:本地 build -> Harbor digest -> 完整 compose -> 预拉 -> 滚动重建。 # 这是“服务更新”主链路:本地 build -> Harbor digest -> 完整 compose -> 预拉 -> 滚动重建。
normalized = normalize_service_names(service_names) normalized = normalize_service_names(service_names)
@ -658,19 +789,24 @@ def deploy_updated_services_payload(
if service_name not in records_map: if service_name not in records_map:
raise RuntimeError(f"service not found in runtime topology: {service_name}") raise RuntimeError(f"service not found in runtime topology: {service_name}")
emit_progress(progress, "prepare", f"准备更新服务:{', '.join(normalized)}")
build_result = build_service_images( build_result = build_service_images(
normalized, normalized,
java_git_ref=java_git_ref, java_git_ref=java_git_ref,
go_git_ref=go_git_ref, go_git_ref=go_git_ref,
image_tag=image_tag, image_tag=image_tag,
skip_maven_build=skip_maven_build, skip_maven_build=skip_maven_build,
progress=progress,
) )
service_images = dict(build_result.get("serviceImages") or {}) service_images = dict(build_result.get("serviceImages") or {})
emit_progress(progress, "prepare", "镜像构建完成,开始更新本地完整 compose 模板")
template_updates: list[dict[str, Any]] = [] template_updates: list[dict[str, Any]] = []
for service_name in normalized: for service_name in normalized:
for record in records_map[service_name]: for record in records_map[service_name]:
template_updates.append(update_local_service_image(record["host"], service_name, service_images[service_name])) image_ref = service_images[service_name]
emit_progress(progress, "compose.template", f"更新模板:{record['host']} / {service_name} -> {image_ref}")
template_updates.append(update_local_service_image(record["host"], service_name, image_ref))
host_plan = host_image_plan(normalized, service_images) host_plan = host_image_plan(normalized, service_images)
prepared_hosts: list[dict[str, Any]] = [] prepared_hosts: list[dict[str, Any]] = []
@ -678,8 +814,20 @@ def deploy_updated_services_payload(
try: try:
for host_entry in host_plan: for host_entry in host_plan:
emit_progress(progress, f"compose.sync.{host_entry['host']}", f"同步完整 compose 到远端:{host_entry['host']}")
compose_result = sync_remote_host_compose_template(host_entry["host"], refresh_from_remote=False) compose_result = sync_remote_host_compose_template(host_entry["host"], refresh_from_remote=False)
emit_progress(
progress,
f"compose.sync.{host_entry['host']}",
f"compose 同步完成changed={bool(compose_result.get('changed'))} path={compose_result.get('path')}",
)
emit_progress(progress, f"image.prepull.{host_entry['host']}", f"预拉镜像:{', '.join(host_entry.get('images') or [])}")
prepull_result = prepull_host_images(host_entry) prepull_result = prepull_host_images(host_entry)
emit_progress(
progress,
f"image.prepull.{host_entry['host']}",
f"预拉完成:{prepull_result['status']} {short_text(prepull_result['output'])}",
)
prepared_hosts.append( prepared_hosts.append(
{ {
"host": host_entry["host"], "host": host_entry["host"],
@ -702,6 +850,7 @@ def deploy_updated_services_payload(
for service_name in normalized: for service_name in normalized:
for service in records_map[service_name]: for service in records_map[service_name]:
started_at = time.time() started_at = time.time()
step_stage = f"restart.{service['host']}.{service_name}"
step: dict[str, Any] = { step: dict[str, Any] = {
"service": service_name, "service": service_name,
"host": service["host"], "host": service["host"],
@ -711,9 +860,15 @@ def deploy_updated_services_payload(
"status": "running", "status": "running",
} }
try: try:
emit_progress(progress, step_stage, f"开始滚动更新:{service['host']} / {service_name}")
restart_result = restart_service_instance(service) restart_result = restart_service_instance(service)
step["tatStatus"] = restart_result.get("status") step["tatStatus"] = restart_result.get("status")
step["tatOutput"] = str(restart_result.get("output") or "").strip()[:500] step["tatOutput"] = str(restart_result.get("output") or "").strip()[:500]
emit_progress(
progress,
step_stage,
f"TAT 重建完成:{restart_result.get('status')} {short_text(step['tatOutput'])}",
)
http_result = wait_http_ready(service, timeout_seconds=ROLLING_RESTART_TIMEOUT_SECONDS) http_result = wait_http_ready(service, timeout_seconds=ROLLING_RESTART_TIMEOUT_SECONDS)
step["http"] = { step["http"] = {
@ -722,24 +877,45 @@ def deploy_updated_services_payload(
"latencyMs": int(http_result.get("latencyMs") or 0), "latencyMs": int(http_result.get("latencyMs") or 0),
"detail": str(http_result.get("detail") or ""), "detail": str(http_result.get("detail") or ""),
} }
emit_progress(
progress,
step_stage,
f"HTTP 恢复:{step['http']['statusCode']} / {step['http']['latencyMs']}ms",
)
if service["kind"] == "java": if service["kind"] == "java":
step["nacos"] = wait_java_registration_ready( step["nacos"] = wait_java_registration_ready(
service, service,
timeout_seconds=ROLLING_RESTART_TIMEOUT_SECONDS, timeout_seconds=ROLLING_RESTART_TIMEOUT_SECONDS,
) )
emit_progress(
progress,
step_stage,
f"Nacos 注册恢复:{step['nacos'].get('source') or 'ok'}",
)
if ROLLING_RESTART_STABILIZE_SECONDS > 0: if ROLLING_RESTART_STABILIZE_SECONDS > 0:
emit_progress(
progress,
step_stage,
f"稳定观察 {ROLLING_RESTART_STABILIZE_SECONDS}s",
)
time.sleep(ROLLING_RESTART_STABILIZE_SECONDS) time.sleep(ROLLING_RESTART_STABILIZE_SECONDS)
step["status"] = "success" step["status"] = "success"
step["durationSeconds"] = round(time.time() - started_at, 2) step["durationSeconds"] = round(time.time() - started_at, 2)
steps.append(step) steps.append(step)
emit_progress(
progress,
step_stage,
f"实例更新成功,耗时 {step['durationSeconds']}s",
)
except Exception as exc: # noqa: BLE001 except Exception as exc: # noqa: BLE001
step["status"] = "failed" step["status"] = "failed"
step["error"] = str(exc) step["error"] = str(exc)
step["durationSeconds"] = round(time.time() - started_at, 2) step["durationSeconds"] = round(time.time() - started_at, 2)
steps.append(step) steps.append(step)
emit_progress(progress, step_stage, f"实例更新失败:{step['error']}", level="error")
clear_operation_caches() clear_operation_caches()
return { return {
"ok": False, "ok": False,
@ -753,7 +929,9 @@ def deploy_updated_services_payload(
} }
finally: finally:
clear_operation_caches() clear_operation_caches()
emit_progress(progress, "finish", "已清理监控缓存")
emit_progress(progress, "finish", "全部服务更新完成")
return { return {
"ok": True, "ok": True,
"updatedAt": utc_now(), "updatedAt": utc_now(),

View File

@ -0,0 +1,249 @@
from __future__ import annotations
# 返回状态给 HTTP 层时直接用 deepcopy 最稳。
import copy
# 后台更新任务需要用线程跑,避免 HTTP 请求一直阻塞。
import threading
# operationId 用 uuid 生成最直接。
import uuid
from typing import Any
from .config import utc_now
from .service_release import deploy_updated_services_payload
# 内存里最多保留最近若干次更新记录,避免 deploy 机常驻进程越跑越大。
MAX_UPDATE_OPERATIONS = 20
# 单次更新最多保留最近这些日志行,超出后丢弃最旧的。
MAX_UPDATE_LOG_LINES = 1200
# 统一保护任务状态的锁。
UPDATE_JOB_LOCK = threading.Lock()
# 当前进程内的更新任务表。
UPDATE_JOBS: dict[str, dict[str, Any]] = {}
# 记录任务插入顺序,便于淘汰最旧项。
UPDATE_JOB_ORDER: list[str] = []
def normalize_services(service_names: list[str]) -> list[str]:
# 请求里允许重复服务名,这里统一去重并保序。
normalized = [str(item or "").strip() for item in service_names if str(item or "").strip()]
return list(dict.fromkeys(normalized))
def create_job_payload(
service_names: list[str],
*,
java_git_ref: str,
go_git_ref: str,
image_tag: str,
skip_maven_build: bool,
) -> dict[str, Any]:
# 后台任务一创建就落一份初始状态,前端可以马上开始轮询。
normalized = normalize_services(service_names)
operation_id = uuid.uuid4().hex[:12]
now = utc_now()
payload = {
"operationId": operation_id,
"kind": "service-update",
"status": "queued",
"stage": "queued",
"message": "service update queued",
"error": "",
"startedAt": now,
"updatedAt": now,
"finishedAt": "",
"logDroppedCount": 0,
"logs": [],
"result": None,
"request": {
"services": normalized,
"javaGitRef": str(java_git_ref or "").strip(),
"goGitRef": str(go_git_ref or "").strip(),
"imageTag": str(image_tag or "").strip(),
"skipMavenBuild": bool(skip_maven_build),
},
}
with UPDATE_JOB_LOCK:
UPDATE_JOBS[operation_id] = payload
UPDATE_JOB_ORDER.append(operation_id)
trim_jobs_unlocked()
return snapshot_job(operation_id)
def trim_jobs_unlocked() -> None:
# 只保留最近的若干次任务,老数据直接淘汰。
while len(UPDATE_JOB_ORDER) > MAX_UPDATE_OPERATIONS:
operation_id = UPDATE_JOB_ORDER.pop(0)
UPDATE_JOBS.pop(operation_id, None)
def append_job_log(operation_id: str, stage: str, message: str, *, level: str = "info") -> None:
# 每条日志都带时间和阶段,页面可以直接展示。
text = str(message or "").rstrip()
if not text:
return
with UPDATE_JOB_LOCK:
payload = UPDATE_JOBS.get(operation_id)
if payload is None:
return
payload["updatedAt"] = utc_now()
payload["stage"] = str(stage or payload.get("stage") or "running")
payload["logs"].append(
{
"time": payload["updatedAt"],
"stage": str(stage or "").strip(),
"level": str(level or "info").strip() or "info",
"message": text,
}
)
if len(payload["logs"]) > MAX_UPDATE_LOG_LINES:
overflow = len(payload["logs"]) - MAX_UPDATE_LOG_LINES
payload["logs"] = payload["logs"][overflow:]
payload["logDroppedCount"] = int(payload.get("logDroppedCount") or 0) + overflow
def patch_job(operation_id: str, **fields: Any) -> None:
# 任务元信息更新统一走这里,避免漏掉 updatedAt。
with UPDATE_JOB_LOCK:
payload = UPDATE_JOBS.get(operation_id)
if payload is None:
return
payload.update(fields)
payload["updatedAt"] = utc_now()
def finish_job(operation_id: str, *, result: dict[str, Any], status: str, error: str = "") -> None:
# 结束态统一收口,前端轮询时不用自己猜。
now = utc_now()
with UPDATE_JOB_LOCK:
payload = UPDATE_JOBS.get(operation_id)
if payload is None:
return
payload["status"] = status
payload["stage"] = status
payload["message"] = (
str(result.get("message") or "").strip()
or ("service update complete" if status == "success" else "service update failed")
)
payload["error"] = str(error or "").strip()
payload["finishedAt"] = now
payload["updatedAt"] = now
payload["result"] = result
def snapshot_job(operation_id: str) -> dict[str, Any]:
# HTTP 返回前复制一份,避免把锁外的可变对象直接暴露出去。
with UPDATE_JOB_LOCK:
payload = UPDATE_JOBS.get(operation_id)
if payload is None:
raise KeyError(operation_id)
return {
"ok": True,
"operation": {
"operationId": payload["operationId"],
"kind": payload["kind"],
"status": payload["status"],
"stage": payload["stage"],
"message": payload["message"],
"error": payload["error"],
"startedAt": payload["startedAt"],
"updatedAt": payload["updatedAt"],
"finishedAt": payload["finishedAt"],
"logDroppedCount": payload["logDroppedCount"],
"logs": copy.deepcopy(payload.get("logs") or []),
"result": copy.deepcopy(payload.get("result")),
"request": copy.deepcopy(payload.get("request") or {}),
},
}
def job_progress_logger(operation_id: str):
# service_release 侧只关心 stage / level / message落盘细节都交给这里。
def _logger(stage: str, level: str, message: str) -> None:
append_job_log(operation_id, stage, message, level=level)
return _logger
def run_update_job(
operation_id: str,
service_names: list[str],
*,
java_git_ref: str,
go_git_ref: str,
image_tag: str,
skip_maven_build: bool,
) -> None:
# 真正的更新逻辑放后台线程里执行。
patch_job(operation_id, status="running", stage="prepare", message="service update running")
append_job_log(operation_id, "prepare", f"开始更新服务:{', '.join(normalize_services(service_names))}")
try:
result = deploy_updated_services_payload(
service_names,
java_git_ref=java_git_ref,
go_git_ref=go_git_ref,
image_tag=image_tag,
skip_maven_build=skip_maven_build,
progress=job_progress_logger(operation_id),
)
if result.get("ok"):
append_job_log(operation_id, "success", "服务更新完成")
finish_job(operation_id, result=result, status="success")
return
error = str(result.get("error") or "service update failed").strip()
append_job_log(operation_id, "failed", error, level="error")
finish_job(operation_id, result=result, status="failed", error=error)
except Exception as exc: # noqa: BLE001
error = str(exc)
append_job_log(operation_id, "failed", error, level="error")
finish_job(
operation_id,
result={"ok": False, "updatedAt": utc_now(), "error": error},
status="failed",
error=error,
)
def start_update_job(
service_names: list[str],
*,
java_git_ref: str,
go_git_ref: str,
image_tag: str,
skip_maven_build: bool,
) -> dict[str, Any]:
# 创建任务后立刻拉起后台线程,并把 operationId 返回给前端。
payload = create_job_payload(
service_names,
java_git_ref=java_git_ref,
go_git_ref=go_git_ref,
image_tag=image_tag,
skip_maven_build=skip_maven_build,
)
operation_id = payload["operation"]["operationId"]
worker = threading.Thread(
target=run_update_job,
kwargs={
"operation_id": operation_id,
"service_names": list(service_names),
"java_git_ref": java_git_ref,
"go_git_ref": go_git_ref,
"image_tag": image_tag,
"skip_maven_build": skip_maven_build,
},
name=f"service-update-{operation_id}",
daemon=True,
)
worker.start()
return snapshot_job(operation_id)
def get_update_job_payload(operation_id: str) -> dict[str, Any]:
# 供 HTTP 轮询接口直接读取任务状态。
return snapshot_job(operation_id)

View File

@ -534,6 +534,38 @@ body {
margin-top: 14px; margin-top: 14px;
} }
.operation-meta {
margin-top: 14px;
}
.update-log-panel {
margin-top: 14px;
padding: 16px;
border: 1px solid var(--line);
border-radius: 22px;
background: color-mix(in oklab, var(--panel-strong) 94%, transparent);
}
.update-log-head {
margin-bottom: 12px;
}
.update-log-output {
margin: 0;
min-height: 180px;
max-height: 420px;
overflow: auto;
padding: 16px;
border: 1px solid color-mix(in oklab, var(--line) 88%, transparent);
border-radius: 18px;
background: oklch(19% 0.02 248 / 0.96);
color: oklch(92% 0.01 250);
font-size: 12px;
line-height: 1.55;
white-space: pre-wrap;
word-break: break-word;
}
.update-field, .update-field,
.update-check { .update-check {
display: flex; display: flex;

View File

@ -1,6 +1,7 @@
const { createApp } = Vue; const { createApp } = Vue;
const STORAGE_KEY = "hy-app-monitor-refresh-interval"; const STORAGE_KEY = "hy-app-monitor-refresh-interval";
const UPDATE_OPERATION_STORAGE_KEY = "hy-app-monitor-service-update-operation";
createApp({ createApp({
data() { data() {
@ -89,6 +90,13 @@ createApp({
loading: false, loading: false,
error: "", error: "",
running: false, running: false,
operationId: "",
status: "",
stage: "",
startedAt: "",
finishedAt: "",
logDroppedCount: 0,
logs: [],
message: "", message: "",
items: [], items: [],
selectedServices: [], selectedServices: [],
@ -108,6 +116,7 @@ createApp({
golangRepoRoot: "", golangRepoRoot: "",
}, },
result: null, result: null,
pollTimer: null,
}, },
restartOps: { restartOps: {
loading: false, loading: false,
@ -221,6 +230,16 @@ createApp({
updateTargetMap() { updateTargetMap() {
return Object.fromEntries((this.updateOps.items || []).map((item) => [item.service, item])); return Object.fromEntries((this.updateOps.items || []).map((item) => [item.service, item]));
}, },
updateLogText() {
return (this.updateOps.logs || [])
.map((entry) => {
const time = this.formatDateTime(entry.time);
const stage = entry.stage ? ` [${entry.stage}]` : "";
const level = entry.level && entry.level !== "info" ? ` (${entry.level})` : "";
return `${time}${stage}${level} ${entry.message || ""}`.trim();
})
.join("\n");
},
}, },
watch: { watch: {
refreshIntervalSeconds() { refreshIntervalSeconds() {
@ -314,6 +333,33 @@ createApp({
updateTargetLabel(item) { updateTargetLabel(item) {
return `${item.service} · ${item.kind} · ${item.hosts.map((host) => host.host).join(" / ")}`; return `${item.service} · ${item.kind} · ${item.hosts.map((host) => host.host).join(" / ")}`;
}, },
updateStatusBadgeClass() {
if (this.updateOps.error || this.updateOps.status === "failed") {
return "bad";
}
if (this.updateOps.running || this.updateOps.status === "running" || this.updateOps.status === "queued") {
return "warn";
}
if (this.updateOps.status === "success") {
return "ok";
}
return "neutral";
},
updateStatusLabel() {
if (this.updateOps.error || this.updateOps.status === "failed") {
return "失败";
}
if (this.updateOps.status === "queued") {
return "排队中";
}
if (this.updateOps.running || this.updateOps.status === "running") {
return "执行中";
}
if (this.updateOps.status === "success") {
return "已完成";
}
return "未开始";
},
syncRefreshSettings() { syncRefreshSettings() {
const settings = this.payload.settings || {}; const settings = this.payload.settings || {};
const options = Array.isArray(settings.refreshIntervalOptions) const options = Array.isArray(settings.refreshIntervalOptions)
@ -339,6 +385,96 @@ createApp({
this.applyRefreshTimer(); this.applyRefreshTimer();
} }
}, },
persistUpdateOperationId() {
if (this.updateOps.operationId) {
window.sessionStorage.setItem(UPDATE_OPERATION_STORAGE_KEY, this.updateOps.operationId);
return;
}
window.sessionStorage.removeItem(UPDATE_OPERATION_STORAGE_KEY);
},
stopUpdatePolling() {
if (this.updateOps.pollTimer) {
window.clearInterval(this.updateOps.pollTimer);
this.updateOps.pollTimer = null;
}
},
scrollUpdateLogToBottom() {
this.$nextTick(() => {
const node = document.getElementById("service-update-log-output");
if (node) {
node.scrollTop = node.scrollHeight;
}
});
},
applyUpdateOperation(operation) {
if (!operation) {
return;
}
this.updateOps.operationId = operation.operationId || "";
this.updateOps.status = operation.status || "";
this.updateOps.stage = operation.stage || "";
this.updateOps.startedAt = operation.startedAt || "";
this.updateOps.finishedAt = operation.finishedAt || "";
this.updateOps.logDroppedCount = Number(operation.logDroppedCount || 0);
this.updateOps.logs = Array.isArray(operation.logs) ? operation.logs : [];
this.updateOps.result = operation.result || null;
this.updateOps.running = operation.status === "queued" || operation.status === "running";
if (operation.status === "success") {
this.updateOps.message = operation.message || operation.result?.message || "服务更新完成";
this.updateOps.error = "";
} else if (operation.status === "failed") {
this.updateOps.error = operation.error || operation.result?.error || "服务更新失败";
this.updateOps.message = operation.message || "";
} else {
this.updateOps.message = operation.message || "";
}
this.persistUpdateOperationId();
this.scrollUpdateLogToBottom();
},
startUpdatePolling() {
this.stopUpdatePolling();
if (!this.updateOps.operationId) {
return;
}
this.updateOps.pollTimer = window.setInterval(() => {
this.fetchUpdateOperationStatus(this.updateOps.operationId, { silent: true });
}, 2000);
},
async fetchUpdateOperationStatus(operationId, { silent = false } = {}) {
if (!operationId) {
return;
}
const wasRunning = this.updateOps.running;
try {
const query = new URLSearchParams({ id: operationId });
const response = await fetch(`/api/ops/update-status?${query.toString()}`, { cache: "no-store" });
const payload = await this.readJsonResponse(response);
if (!response.ok) {
throw new Error(payload.error || `HTTP ${response.status}`);
}
this.applyUpdateOperation(payload.operation || {});
if (!this.updateOps.running) {
this.stopUpdatePolling();
if (wasRunning) {
await this.refresh();
await this.refreshUpdateTargets();
await this.refreshRestartTargets();
}
}
} catch (error) {
this.stopUpdatePolling();
this.updateOps.running = false;
if (!silent) {
this.updateOps.error = error instanceof Error ? error.message : String(error);
}
}
},
applyRefreshTimer() { applyRefreshTimer() {
if (this.timer) { if (this.timer) {
window.clearInterval(this.timer); window.clearInterval(this.timer);
@ -685,9 +821,19 @@ createApp({
return; return;
} }
this.stopUpdatePolling();
this.updateOps.running = true; this.updateOps.running = true;
this.updateOps.error = ""; this.updateOps.error = "";
this.updateOps.message = ""; this.updateOps.message = "已提交更新任务,等待开始执行";
this.updateOps.operationId = "";
this.updateOps.status = "queued";
this.updateOps.stage = "queued";
this.updateOps.startedAt = "";
this.updateOps.finishedAt = "";
this.updateOps.logDroppedCount = 0;
this.updateOps.logs = [];
this.updateOps.result = null;
this.persistUpdateOperationId();
try { try {
const response = await fetch("/api/ops/update-services", { const response = await fetch("/api/ops/update-services", {
@ -708,18 +854,14 @@ createApp({
throw new Error(payload.error || `HTTP ${response.status}`); throw new Error(payload.error || `HTTP ${response.status}`);
} }
this.updateOps.result = payload; this.applyUpdateOperation(payload.operation || {});
this.updateOps.message = payload.ok this.startUpdatePolling();
? (payload.message || "服务更新完成") await this.fetchUpdateOperationStatus(this.updateOps.operationId, { silent: true });
: (payload.error || "服务更新失败");
await this.refresh();
await this.refreshUpdateTargets();
await this.refreshRestartTargets();
} catch (error) { } catch (error) {
this.updateOps.error = error instanceof Error ? error.message : String(error); this.updateOps.error = error instanceof Error ? error.message : String(error);
} finally {
this.updateOps.running = false; this.updateOps.running = false;
} finally {
this.scrollUpdateLogToBottom();
} }
}, },
toggleRestartService(serviceName) { toggleRestartService(serviceName) {
@ -781,10 +923,16 @@ createApp({
this.refreshGoConfig(); this.refreshGoConfig();
this.refreshUpdateTargets(); this.refreshUpdateTargets();
this.refreshRestartTargets(); this.refreshRestartTargets();
const lastOperationId = window.sessionStorage.getItem(UPDATE_OPERATION_STORAGE_KEY) || "";
if (lastOperationId) {
this.fetchUpdateOperationStatus(lastOperationId, { silent: true });
this.startUpdatePolling();
}
}, },
beforeUnmount() { beforeUnmount() {
if (this.timer) { if (this.timer) {
window.clearInterval(this.timer); window.clearInterval(this.timer);
} }
this.stopUpdatePolling();
}, },
}).mount("#app"); }).mount("#app");

View File

@ -427,8 +427,8 @@
<h3>Service Update</h3> <h3>Service Update</h3>
<p>本机构建选中的 Java / Golang 服务,推到 Harbor更新完整 compose并只重建你选中的服务</p> <p>本机构建选中的 Java / Golang 服务,推到 Harbor更新完整 compose并只重建你选中的服务</p>
</div> </div>
<span :class="['badge', updateOps.error ? 'bad' : updateOps.running ? 'warn' : 'ok']"> <span :class="['badge', updateStatusBadgeClass()]">
{{ updateOps.error ? '存在错误' : updateOps.running ? '执行中' : '可操作' }} {{ updateStatusLabel() }}
</span> </span>
</div> </div>
@ -488,6 +488,28 @@
<p v-if="updateOps.error" class="metric-error">{{ updateOps.error }}</p> <p v-if="updateOps.error" class="metric-error">{{ updateOps.error }}</p>
<p v-if="updateOps.message" :class="updateOps.result && updateOps.result.ok ? 'ok-text' : 'metric-error'">{{ updateOps.message }}</p> <p v-if="updateOps.message" :class="updateOps.result && updateOps.result.ok ? 'ok-text' : 'metric-error'">{{ updateOps.message }}</p>
<div v-if="updateOps.operationId" class="host-tags operation-meta">
<span class="badge neutral mono">任务:{{ updateOps.operationId }}</span>
<span :class="['badge', updateStatusBadgeClass()]">状态:{{ updateStatusLabel() }}</span>
<span class="badge neutral mono">阶段:{{ updateOps.stage || '-' }}</span>
<span class="badge neutral">开始:{{ formatDateTime(updateOps.startedAt) }}</span>
<span v-if="updateOps.finishedAt" class="badge neutral">结束:{{ formatDateTime(updateOps.finishedAt) }}</span>
<span v-if="updateOps.logDroppedCount > 0" class="badge warn">已截断 {{ updateOps.logDroppedCount }} 行旧日志</span>
</div>
<section v-if="updateOps.operationId || updateOps.running || updateOps.logs.length > 0" class="update-log-panel">
<div class="panel-head update-log-head">
<div>
<h3>Update Log</h3>
<p>按阶段展示构建、推 Harbor、同步 compose、预拉镜像和滚动重建的实时输出</p>
</div>
<button class="refresh ghost" @click="fetchUpdateOperationStatus(updateOps.operationId)" :disabled="!updateOps.operationId || updateOps.running">
刷新日志
</button>
</div>
<pre id="service-update-log-output" class="update-log-output mono">{{ updateLogText || '等待更新日志...' }}</pre>
</section>
<div v-if="updateOps.result && updateOps.result.build && updateOps.result.build.length > 0" class="table-wrap"> <div v-if="updateOps.result && updateOps.result.build && updateOps.result.build.length > 0" class="table-wrap">
<table class="table compact"> <table class="table compact">
<thead> <thead>