390 lines
13 KiB
Python
390 lines
13 KiB
Python
from __future__ import annotations
|
||
|
||
# 返回状态给 HTTP 层时直接用 deepcopy 最稳。
|
||
import copy
|
||
# 状态持久化到 JSON 文件需要标准库 json。
|
||
import json
|
||
# 日志和状态文件落到 run 目录。
|
||
from pathlib import Path
|
||
# 后台更新任务需要用线程跑，避免 HTTP 请求一直阻塞。
|
||
import threading
|
||
# operationId 用 uuid 生成最直接。
|
||
import re
|
||
import uuid
|
||
from typing import Any
|
||
|
||
from core.config import ROOT, utc_now
|
||
from .service_release import deploy_updated_services_payload
|
||
|
||
|
||
# 内存里最多保留最近若干次更新记录，避免 deploy 机常驻进程越跑越大。
|
||
MAX_UPDATE_OPERATIONS = 20
|
||
# 任务状态和完整日志统一落到 run 目录。
|
||
UPDATE_JOB_DIR = ROOT / "run" / "service_update_jobs"
|
||
# 统一保护任务状态的锁。
|
||
UPDATE_JOB_LOCK = threading.Lock()
|
||
# 当前进程内的更新任务表。
|
||
UPDATE_JOBS: dict[str, dict[str, Any]] = {}
|
||
# 记录任务插入顺序，便于淘汰最旧项。
|
||
UPDATE_JOB_ORDER: list[str] = []
|
||
|
||
|
||
def normalize_services(service_names: list[str]) -> list[str]:
|
||
# 请求里允许重复服务名，这里统一去重并保序。
|
||
normalized = [str(item or "").strip() for item in service_names if str(item or "").strip()]
|
||
return list(dict.fromkeys(normalized))
|
||
|
||
|
||
def update_job_state_path(operation_id: str) -> Path:
|
||
# 每个任务一份状态 JSON。
|
||
return UPDATE_JOB_DIR / f"{operation_id}.json"
|
||
|
||
|
||
def update_job_log_path(operation_id: str) -> Path:
|
||
# 每个任务一份完整文本日志。
|
||
return UPDATE_JOB_DIR / f"{operation_id}.log"
|
||
|
||
|
||
def ensure_update_job_dir() -> None:
|
||
# 持久化目录不存在时先创建。
|
||
UPDATE_JOB_DIR.mkdir(parents=True, exist_ok=True)
|
||
|
||
|
||
def valid_operation_id(operation_id: str) -> bool:
|
||
# 只接受简单安全字符，避免路径拼接被滥用。
|
||
return re.fullmatch(r"[0-9A-Za-z_-]{6,64}", str(operation_id or "").strip()) is not None
|
||
|
||
|
||
def append_log_file(operation_id: str, line: str) -> None:
|
||
# 完整日志单独写入文本文件，供页面下载。
|
||
ensure_update_job_dir()
|
||
with update_job_log_path(operation_id).open("a", encoding="utf-8") as handle:
|
||
handle.write(line)
|
||
|
||
|
||
def operation_payload(payload: dict[str, Any]) -> dict[str, Any]:
|
||
# 对外返回和落盘都统一走这个结构，避免前后不一致。
|
||
return {
|
||
"operationId": payload["operationId"],
|
||
"kind": payload["kind"],
|
||
"status": payload["status"],
|
||
"stage": payload["stage"],
|
||
"message": payload["message"],
|
||
"error": payload["error"],
|
||
"startedAt": payload["startedAt"],
|
||
"updatedAt": payload["updatedAt"],
|
||
"finishedAt": payload["finishedAt"],
|
||
"logDroppedCount": payload["logDroppedCount"],
|
||
"logs": copy.deepcopy(payload.get("logs") or []),
|
||
"result": copy.deepcopy(payload.get("result")),
|
||
"request": copy.deepcopy(payload.get("request") or {}),
|
||
"logUrl": f"/api/ops/update-log?id={payload['operationId']}",
|
||
}
|
||
|
||
|
||
def persist_job_unlocked(payload: dict[str, Any]) -> None:
|
||
# 每次状态变化都落盘，服务重启后还能恢复最近任务。
|
||
ensure_update_job_dir()
|
||
state_path = update_job_state_path(payload["operationId"])
|
||
state_path.write_text(
|
||
json.dumps({"ok": True, "operation": operation_payload(payload)}, ensure_ascii=False, indent=2) + "\n",
|
||
encoding="utf-8",
|
||
)
|
||
|
||
|
||
def recover_lost_running_job_unlocked(payload: dict[str, Any]) -> None:
|
||
# 进程重启后，原后台线程已经不存在；把悬挂中的 running/queued 任务收口成失败态最直接。
|
||
if payload.get("status") not in {"queued", "running"}:
|
||
return
|
||
|
||
now = utc_now()
|
||
message = "monitor process restarted before update status could be finalized"
|
||
payload["status"] = "failed"
|
||
payload["stage"] = "failed"
|
||
payload["message"] = "service update interrupted"
|
||
payload["error"] = message
|
||
payload["updatedAt"] = now
|
||
payload["finishedAt"] = now
|
||
payload["logs"].append(
|
||
{
|
||
"time": now,
|
||
"stage": "failed",
|
||
"level": "error",
|
||
"message": message,
|
||
}
|
||
)
|
||
|
||
append_log_file(payload["operationId"], f"[{now}] [failed] [error] {message}\n")
|
||
persist_job_unlocked(payload)
|
||
|
||
|
||
def load_job_from_disk(operation_id: str) -> dict[str, Any] | None:
|
||
# 任务不在内存时，从持久化状态恢复。
|
||
if not valid_operation_id(operation_id):
|
||
return None
|
||
|
||
state_path = update_job_state_path(operation_id)
|
||
if not state_path.exists():
|
||
return None
|
||
|
||
payload = json.loads(state_path.read_text(encoding="utf-8"))
|
||
operation = dict(payload.get("operation") or {})
|
||
if not operation:
|
||
return None
|
||
|
||
restored = {
|
||
"operationId": str(operation.get("operationId") or operation_id).strip(),
|
||
"kind": str(operation.get("kind") or "service-update").strip() or "service-update",
|
||
"status": str(operation.get("status") or "").strip(),
|
||
"stage": str(operation.get("stage") or "").strip(),
|
||
"message": str(operation.get("message") or "").strip(),
|
||
"error": str(operation.get("error") or "").strip(),
|
||
"startedAt": str(operation.get("startedAt") or "").strip(),
|
||
"updatedAt": str(operation.get("updatedAt") or "").strip(),
|
||
"finishedAt": str(operation.get("finishedAt") or "").strip(),
|
||
"logDroppedCount": int(operation.get("logDroppedCount") or 0),
|
||
"logs": list(operation.get("logs") or []),
|
||
"result": operation.get("result"),
|
||
"request": dict(operation.get("request") or {}),
|
||
}
|
||
recover_lost_running_job_unlocked(restored)
|
||
return restored
|
||
|
||
|
||
def create_job_payload(
|
||
service_names: list[str],
|
||
*,
|
||
java_git_ref: str,
|
||
go_git_ref: str,
|
||
admin_git_ref: str,
|
||
image_tag: str,
|
||
skip_maven_build: bool,
|
||
) -> dict[str, Any]:
|
||
# 后台任务一创建就落一份初始状态，前端可以马上开始轮询。
|
||
normalized = normalize_services(service_names)
|
||
operation_id = uuid.uuid4().hex[:12]
|
||
now = utc_now()
|
||
payload = {
|
||
"operationId": operation_id,
|
||
"kind": "service-update",
|
||
"status": "queued",
|
||
"stage": "queued",
|
||
"message": "service update queued",
|
||
"error": "",
|
||
"startedAt": now,
|
||
"updatedAt": now,
|
||
"finishedAt": "",
|
||
"logDroppedCount": 0,
|
||
"logs": [],
|
||
"result": None,
|
||
"request": {
|
||
"services": normalized,
|
||
"javaGitRef": str(java_git_ref or "").strip(),
|
||
"goGitRef": str(go_git_ref or "").strip(),
|
||
"adminGitRef": str(admin_git_ref or "").strip(),
|
||
"imageTag": str(image_tag or "").strip(),
|
||
"skipMavenBuild": bool(skip_maven_build),
|
||
},
|
||
}
|
||
with UPDATE_JOB_LOCK:
|
||
ensure_update_job_dir()
|
||
UPDATE_JOBS[operation_id] = payload
|
||
UPDATE_JOB_ORDER.append(operation_id)
|
||
update_job_log_path(operation_id).write_text("", encoding="utf-8")
|
||
persist_job_unlocked(payload)
|
||
trim_jobs_unlocked()
|
||
return snapshot_job(operation_id)
|
||
|
||
|
||
def trim_jobs_unlocked() -> None:
|
||
# 只保留最近的若干次任务，老数据直接淘汰。
|
||
while len(UPDATE_JOB_ORDER) > MAX_UPDATE_OPERATIONS:
|
||
operation_id = UPDATE_JOB_ORDER.pop(0)
|
||
UPDATE_JOBS.pop(operation_id, None)
|
||
|
||
|
||
def append_job_log(operation_id: str, stage: str, message: str, *, level: str = "info") -> None:
|
||
# 每条日志都带时间和阶段，页面和状态文件都保留完整日志。
|
||
text = str(message or "").rstrip()
|
||
if not text:
|
||
return
|
||
|
||
with UPDATE_JOB_LOCK:
|
||
payload = UPDATE_JOBS.get(operation_id)
|
||
if payload is None:
|
||
return
|
||
|
||
payload["updatedAt"] = utc_now()
|
||
payload["stage"] = str(stage or payload.get("stage") or "running")
|
||
payload["logs"].append(
|
||
{
|
||
"time": payload["updatedAt"],
|
||
"stage": str(stage or "").strip(),
|
||
"level": str(level or "info").strip() or "info",
|
||
"message": text,
|
||
}
|
||
)
|
||
append_log_file(
|
||
operation_id,
|
||
f"[{payload['updatedAt']}] [{str(stage or '').strip() or payload['stage']}] [{str(level or 'info').strip() or 'info'}] {text}\n",
|
||
)
|
||
persist_job_unlocked(payload)
|
||
|
||
|
||
def patch_job(operation_id: str, **fields: Any) -> None:
|
||
# 任务元信息更新统一走这里，避免漏掉 updatedAt。
|
||
with UPDATE_JOB_LOCK:
|
||
payload = UPDATE_JOBS.get(operation_id)
|
||
if payload is None:
|
||
return
|
||
payload.update(fields)
|
||
payload["updatedAt"] = utc_now()
|
||
persist_job_unlocked(payload)
|
||
|
||
|
||
def finish_job(operation_id: str, *, result: dict[str, Any], status: str, error: str = "") -> None:
|
||
# 结束态统一收口，前端轮询时不用自己猜。
|
||
now = utc_now()
|
||
with UPDATE_JOB_LOCK:
|
||
payload = UPDATE_JOBS.get(operation_id)
|
||
if payload is None:
|
||
return
|
||
payload["status"] = status
|
||
payload["stage"] = status
|
||
payload["message"] = (
|
||
str(result.get("message") or "").strip()
|
||
or ("service update complete" if status == "success" else "service update failed")
|
||
)
|
||
payload["error"] = str(error or "").strip()
|
||
payload["finishedAt"] = now
|
||
payload["updatedAt"] = now
|
||
payload["result"] = result
|
||
persist_job_unlocked(payload)
|
||
|
||
|
||
def snapshot_job(operation_id: str) -> dict[str, Any]:
|
||
# HTTP 返回前复制一份，避免把锁外的可变对象直接暴露出去。
|
||
with UPDATE_JOB_LOCK:
|
||
payload = UPDATE_JOBS.get(operation_id)
|
||
if payload is None:
|
||
raise KeyError(operation_id)
|
||
|
||
return {"ok": True, "operation": operation_payload(payload)}
|
||
|
||
|
||
def job_progress_logger(operation_id: str):
|
||
# service_release 侧只关心 stage / level / message，落盘细节都交给这里。
|
||
def _logger(stage: str, level: str, message: str) -> None:
|
||
append_job_log(operation_id, stage, message, level=level)
|
||
|
||
return _logger
|
||
|
||
|
||
def run_update_job(
|
||
operation_id: str,
|
||
service_names: list[str],
|
||
*,
|
||
java_git_ref: str,
|
||
go_git_ref: str,
|
||
admin_git_ref: str,
|
||
image_tag: str,
|
||
skip_maven_build: bool,
|
||
) -> None:
|
||
# 真正的更新逻辑放后台线程里执行。
|
||
patch_job(operation_id, status="running", stage="prepare", message="service update running")
|
||
append_job_log(operation_id, "prepare", f"开始更新服务：{', '.join(normalize_services(service_names))}")
|
||
|
||
try:
|
||
result = deploy_updated_services_payload(
|
||
service_names,
|
||
java_git_ref=java_git_ref,
|
||
go_git_ref=go_git_ref,
|
||
admin_git_ref=admin_git_ref,
|
||
image_tag=image_tag,
|
||
skip_maven_build=skip_maven_build,
|
||
progress=job_progress_logger(operation_id),
|
||
)
|
||
if result.get("ok"):
|
||
append_job_log(operation_id, "success", "服务更新完成")
|
||
finish_job(operation_id, result=result, status="success")
|
||
return
|
||
|
||
error = str(result.get("error") or "service update failed").strip()
|
||
append_job_log(operation_id, "failed", error, level="error")
|
||
finish_job(operation_id, result=result, status="failed", error=error)
|
||
except Exception as exc: # noqa: BLE001
|
||
error = str(exc)
|
||
append_job_log(operation_id, "failed", error, level="error")
|
||
finish_job(
|
||
operation_id,
|
||
result={"ok": False, "updatedAt": utc_now(), "error": error},
|
||
status="failed",
|
||
error=error,
|
||
)
|
||
|
||
|
||
def start_update_job(
|
||
service_names: list[str],
|
||
*,
|
||
java_git_ref: str,
|
||
go_git_ref: str,
|
||
admin_git_ref: str,
|
||
image_tag: str,
|
||
skip_maven_build: bool,
|
||
) -> dict[str, Any]:
|
||
# 创建任务后立刻拉起后台线程，并把 operationId 返回给前端。
|
||
payload = create_job_payload(
|
||
service_names,
|
||
java_git_ref=java_git_ref,
|
||
go_git_ref=go_git_ref,
|
||
admin_git_ref=admin_git_ref,
|
||
image_tag=image_tag,
|
||
skip_maven_build=skip_maven_build,
|
||
)
|
||
operation_id = payload["operation"]["operationId"]
|
||
|
||
worker = threading.Thread(
|
||
target=run_update_job,
|
||
kwargs={
|
||
"operation_id": operation_id,
|
||
"service_names": list(service_names),
|
||
"java_git_ref": java_git_ref,
|
||
"go_git_ref": go_git_ref,
|
||
"admin_git_ref": admin_git_ref,
|
||
"image_tag": image_tag,
|
||
"skip_maven_build": skip_maven_build,
|
||
},
|
||
name=f"service-update-{operation_id}",
|
||
daemon=True,
|
||
)
|
||
worker.start()
|
||
return snapshot_job(operation_id)
|
||
|
||
|
||
def get_update_job_payload(operation_id: str) -> dict[str, Any]:
|
||
# 供 HTTP 轮询接口直接读取任务状态。
|
||
try:
|
||
return snapshot_job(operation_id)
|
||
except KeyError:
|
||
restored = load_job_from_disk(operation_id)
|
||
if restored is None:
|
||
raise
|
||
|
||
with UPDATE_JOB_LOCK:
|
||
UPDATE_JOBS[operation_id] = restored
|
||
if operation_id not in UPDATE_JOB_ORDER:
|
||
UPDATE_JOB_ORDER.append(operation_id)
|
||
trim_jobs_unlocked()
|
||
return snapshot_job(operation_id)
|
||
|
||
|
||
def get_update_job_log_text(operation_id: str) -> str:
|
||
# 页面下载完整日志时直接返回文本文件内容。
|
||
if not valid_operation_id(operation_id):
|
||
raise KeyError(operation_id)
|
||
|
||
log_path = update_job_log_path(operation_id)
|
||
if not log_path.exists():
|
||
raise KeyError(operation_id)
|
||
return log_path.read_text(encoding="utf-8", errors="replace")
|