from __future__ import annotations # 返回状态给 HTTP 层时直接用 deepcopy 最稳。 import copy # 状态持久化到 JSON 文件需要标准库 json。 import json # 日志和状态文件落到 run 目录。 from pathlib import Path # 后台更新任务需要用线程跑,避免 HTTP 请求一直阻塞。 import threading # operationId 用 uuid 生成最直接。 import re import uuid from typing import Any from core.config import ROOT, utc_now from .service_release import deploy_updated_services_payload # 内存里最多保留最近若干次更新记录,避免 deploy 机常驻进程越跑越大。 MAX_UPDATE_OPERATIONS = 20 # 任务状态和完整日志统一落到 run 目录。 UPDATE_JOB_DIR = ROOT / "run" / "service_update_jobs" # 统一保护任务状态的锁。 UPDATE_JOB_LOCK = threading.Lock() # 当前进程内的更新任务表。 UPDATE_JOBS: dict[str, dict[str, Any]] = {} # 记录任务插入顺序,便于淘汰最旧项。 UPDATE_JOB_ORDER: list[str] = [] def normalize_services(service_names: list[str]) -> list[str]: # 请求里允许重复服务名,这里统一去重并保序。 normalized = [str(item or "").strip() for item in service_names if str(item or "").strip()] return list(dict.fromkeys(normalized)) def update_job_state_path(operation_id: str) -> Path: # 每个任务一份状态 JSON。 return UPDATE_JOB_DIR / f"{operation_id}.json" def update_job_log_path(operation_id: str) -> Path: # 每个任务一份完整文本日志。 return UPDATE_JOB_DIR / f"{operation_id}.log" def ensure_update_job_dir() -> None: # 持久化目录不存在时先创建。 UPDATE_JOB_DIR.mkdir(parents=True, exist_ok=True) def valid_operation_id(operation_id: str) -> bool: # 只接受简单安全字符,避免路径拼接被滥用。 return re.fullmatch(r"[0-9A-Za-z_-]{6,64}", str(operation_id or "").strip()) is not None def append_log_file(operation_id: str, line: str) -> None: # 完整日志单独写入文本文件,供页面下载。 ensure_update_job_dir() with update_job_log_path(operation_id).open("a", encoding="utf-8") as handle: handle.write(line) def operation_payload(payload: dict[str, Any]) -> dict[str, Any]: # 对外返回和落盘都统一走这个结构,避免前后不一致。 return { "operationId": payload["operationId"], "kind": payload["kind"], "status": payload["status"], "stage": payload["stage"], "message": payload["message"], "error": payload["error"], "startedAt": payload["startedAt"], "updatedAt": payload["updatedAt"], "finishedAt": payload["finishedAt"], "logDroppedCount": payload["logDroppedCount"], "logs": copy.deepcopy(payload.get("logs") or []), "result": copy.deepcopy(payload.get("result")), "request": copy.deepcopy(payload.get("request") or {}), "logUrl": f"/api/ops/update-log?id={payload['operationId']}", } def persist_job_unlocked(payload: dict[str, Any]) -> None: # 每次状态变化都落盘,服务重启后还能恢复最近任务。 ensure_update_job_dir() state_path = update_job_state_path(payload["operationId"]) state_path.write_text( json.dumps({"ok": True, "operation": operation_payload(payload)}, ensure_ascii=False, indent=2) + "\n", encoding="utf-8", ) def recover_lost_running_job_unlocked(payload: dict[str, Any]) -> None: # 进程重启后,原后台线程已经不存在;把悬挂中的 running/queued 任务收口成失败态最直接。 if payload.get("status") not in {"queued", "running"}: return now = utc_now() message = "monitor process restarted before update status could be finalized" payload["status"] = "failed" payload["stage"] = "failed" payload["message"] = "service update interrupted" payload["error"] = message payload["updatedAt"] = now payload["finishedAt"] = now payload["logs"].append( { "time": now, "stage": "failed", "level": "error", "message": message, } ) append_log_file(payload["operationId"], f"[{now}] [failed] [error] {message}\n") persist_job_unlocked(payload) def load_job_from_disk(operation_id: str) -> dict[str, Any] | None: # 任务不在内存时,从持久化状态恢复。 if not valid_operation_id(operation_id): return None state_path = update_job_state_path(operation_id) if not state_path.exists(): return None payload = json.loads(state_path.read_text(encoding="utf-8")) operation = dict(payload.get("operation") or {}) if not operation: return None restored = { "operationId": str(operation.get("operationId") or operation_id).strip(), "kind": str(operation.get("kind") or "service-update").strip() or "service-update", "status": str(operation.get("status") or "").strip(), "stage": str(operation.get("stage") or "").strip(), "message": str(operation.get("message") or "").strip(), "error": str(operation.get("error") or "").strip(), "startedAt": str(operation.get("startedAt") or "").strip(), "updatedAt": str(operation.get("updatedAt") or "").strip(), "finishedAt": str(operation.get("finishedAt") or "").strip(), "logDroppedCount": int(operation.get("logDroppedCount") or 0), "logs": list(operation.get("logs") or []), "result": operation.get("result"), "request": dict(operation.get("request") or {}), } recover_lost_running_job_unlocked(restored) return restored def create_job_payload( service_names: list[str], *, java_git_ref: str, go_git_ref: str, admin_git_ref: str, image_tag: str, skip_maven_build: bool, ) -> dict[str, Any]: # 后台任务一创建就落一份初始状态,前端可以马上开始轮询。 normalized = normalize_services(service_names) operation_id = uuid.uuid4().hex[:12] now = utc_now() payload = { "operationId": operation_id, "kind": "service-update", "status": "queued", "stage": "queued", "message": "service update queued", "error": "", "startedAt": now, "updatedAt": now, "finishedAt": "", "logDroppedCount": 0, "logs": [], "result": None, "request": { "services": normalized, "javaGitRef": str(java_git_ref or "").strip(), "goGitRef": str(go_git_ref or "").strip(), "adminGitRef": str(admin_git_ref or "").strip(), "imageTag": str(image_tag or "").strip(), "skipMavenBuild": bool(skip_maven_build), }, } with UPDATE_JOB_LOCK: ensure_update_job_dir() UPDATE_JOBS[operation_id] = payload UPDATE_JOB_ORDER.append(operation_id) update_job_log_path(operation_id).write_text("", encoding="utf-8") persist_job_unlocked(payload) trim_jobs_unlocked() return snapshot_job(operation_id) def trim_jobs_unlocked() -> None: # 只保留最近的若干次任务,老数据直接淘汰。 while len(UPDATE_JOB_ORDER) > MAX_UPDATE_OPERATIONS: operation_id = UPDATE_JOB_ORDER.pop(0) UPDATE_JOBS.pop(operation_id, None) def append_job_log(operation_id: str, stage: str, message: str, *, level: str = "info") -> None: # 每条日志都带时间和阶段,页面和状态文件都保留完整日志。 text = str(message or "").rstrip() if not text: return with UPDATE_JOB_LOCK: payload = UPDATE_JOBS.get(operation_id) if payload is None: return payload["updatedAt"] = utc_now() payload["stage"] = str(stage or payload.get("stage") or "running") payload["logs"].append( { "time": payload["updatedAt"], "stage": str(stage or "").strip(), "level": str(level or "info").strip() or "info", "message": text, } ) append_log_file( operation_id, f"[{payload['updatedAt']}] [{str(stage or '').strip() or payload['stage']}] [{str(level or 'info').strip() or 'info'}] {text}\n", ) persist_job_unlocked(payload) def patch_job(operation_id: str, **fields: Any) -> None: # 任务元信息更新统一走这里,避免漏掉 updatedAt。 with UPDATE_JOB_LOCK: payload = UPDATE_JOBS.get(operation_id) if payload is None: return payload.update(fields) payload["updatedAt"] = utc_now() persist_job_unlocked(payload) def finish_job(operation_id: str, *, result: dict[str, Any], status: str, error: str = "") -> None: # 结束态统一收口,前端轮询时不用自己猜。 now = utc_now() with UPDATE_JOB_LOCK: payload = UPDATE_JOBS.get(operation_id) if payload is None: return payload["status"] = status payload["stage"] = status payload["message"] = ( str(result.get("message") or "").strip() or ("service update complete" if status == "success" else "service update failed") ) payload["error"] = str(error or "").strip() payload["finishedAt"] = now payload["updatedAt"] = now payload["result"] = result persist_job_unlocked(payload) def snapshot_job(operation_id: str) -> dict[str, Any]: # HTTP 返回前复制一份,避免把锁外的可变对象直接暴露出去。 with UPDATE_JOB_LOCK: payload = UPDATE_JOBS.get(operation_id) if payload is None: raise KeyError(operation_id) return {"ok": True, "operation": operation_payload(payload)} def job_progress_logger(operation_id: str): # service_release 侧只关心 stage / level / message,落盘细节都交给这里。 def _logger(stage: str, level: str, message: str) -> None: append_job_log(operation_id, stage, message, level=level) return _logger def run_update_job( operation_id: str, service_names: list[str], *, java_git_ref: str, go_git_ref: str, admin_git_ref: str, image_tag: str, skip_maven_build: bool, ) -> None: # 真正的更新逻辑放后台线程里执行。 patch_job(operation_id, status="running", stage="prepare", message="service update running") append_job_log(operation_id, "prepare", f"开始更新服务:{', '.join(normalize_services(service_names))}") try: result = deploy_updated_services_payload( service_names, java_git_ref=java_git_ref, go_git_ref=go_git_ref, admin_git_ref=admin_git_ref, image_tag=image_tag, skip_maven_build=skip_maven_build, progress=job_progress_logger(operation_id), ) if result.get("ok"): append_job_log(operation_id, "success", "服务更新完成") finish_job(operation_id, result=result, status="success") return error = str(result.get("error") or "service update failed").strip() append_job_log(operation_id, "failed", error, level="error") finish_job(operation_id, result=result, status="failed", error=error) except Exception as exc: # noqa: BLE001 error = str(exc) append_job_log(operation_id, "failed", error, level="error") finish_job( operation_id, result={"ok": False, "updatedAt": utc_now(), "error": error}, status="failed", error=error, ) def start_update_job( service_names: list[str], *, java_git_ref: str, go_git_ref: str, admin_git_ref: str, image_tag: str, skip_maven_build: bool, ) -> dict[str, Any]: # 创建任务后立刻拉起后台线程,并把 operationId 返回给前端。 payload = create_job_payload( service_names, java_git_ref=java_git_ref, go_git_ref=go_git_ref, admin_git_ref=admin_git_ref, image_tag=image_tag, skip_maven_build=skip_maven_build, ) operation_id = payload["operation"]["operationId"] worker = threading.Thread( target=run_update_job, kwargs={ "operation_id": operation_id, "service_names": list(service_names), "java_git_ref": java_git_ref, "go_git_ref": go_git_ref, "admin_git_ref": admin_git_ref, "image_tag": image_tag, "skip_maven_build": skip_maven_build, }, name=f"service-update-{operation_id}", daemon=True, ) worker.start() return snapshot_job(operation_id) def get_update_job_payload(operation_id: str) -> dict[str, Any]: # 供 HTTP 轮询接口直接读取任务状态。 try: return snapshot_job(operation_id) except KeyError: restored = load_job_from_disk(operation_id) if restored is None: raise with UPDATE_JOB_LOCK: UPDATE_JOBS[operation_id] = restored if operation_id not in UPDATE_JOB_ORDER: UPDATE_JOB_ORDER.append(operation_id) trim_jobs_unlocked() return snapshot_job(operation_id) def get_update_job_log_text(operation_id: str) -> str: # 页面下载完整日志时直接返回文本文件内容。 if not valid_operation_id(operation_id): raise KeyError(operation_id) log_path = update_job_log_path(operation_id) if not log_path.exists(): raise KeyError(operation_id) return log_path.read_text(encoding="utf-8", errors="replace")