from __future__ import annotations # 远端命令里会拼接 shell 参数,需要显式转义。 import shlex # 滚动重启是串行动作,轮询恢复要用时间模块。 import time from typing import Any from .compose_templates import sync_remote_host_compose_template from core.config import ( ROLLING_RESTART_POLL_SECONDS, ROLLING_RESTART_STABILIZE_SECONDS, ROLLING_RESTART_TIMEOUT_SECONDS, RUNTIME_BASE_DIR, THRESHOLDS, flatten_services, load_hosts, utc_now, ) from .http_probe import SERVICE_CACHE, probe_service from .nacos import ( NACOS_CACHE, NACOS_DISCOVERY_GROUP, NACOS_SERVICE_NAME_MAP, catalog_instances, current_namespace, list_instances, nacos_truthy, ) from .ssh_remote import run_host_script def restartable_service_records() -> list[dict[str, Any]]: # 当前只开放 Java / Golang 业务服务滚动重启。 services = flatten_services(load_hosts()) return [service for service in services if service.get("kind") in {"java", "golang"}] def build_restart_targets_payload() -> dict[str, Any]: # 页面需要按“服务 -> 宿主机”展示选择项。 grouped: dict[str, dict[str, Any]] = {} for service in restartable_service_records(): item = grouped.setdefault( service["service"], { "service": service["service"], "kind": service["kind"], "hosts": [], }, ) item["hosts"].append( { "host": service["host"], "ip": service["ip"], "instanceId": service["instanceId"], "port": service["port"], "path": service["path"], } ) return { "ok": True, "updatedAt": utc_now(), "items": list(grouped.values()), } def service_records_by_name() -> dict[str, list[dict[str, Any]]]: # 重启顺序沿用 targets.json 的主机顺序。 grouped: dict[str, list[dict[str, Any]]] = {} for service in restartable_service_records(): grouped.setdefault(service["service"], []).append(service) return grouped def remote_command_result(service: dict[str, Any], script: str, command_name: str, timeout_seconds: int) -> dict[str, Any]: # 单实例远端执行现在统一复用 SSH shell。 return run_host_script( service, script, command_name=command_name, timeout_seconds=timeout_seconds, ) def restart_service_instance(service: dict[str, Any]) -> dict[str, Any]: # 线上业务目录和 compose 文件复用现网部署结构。 host_dir = f"{RUNTIME_BASE_DIR.rstrip('/')}/{service['host']}" container_name = f"likei-{service['host']}-{service['service']}" script = f"""set -eu HOST_DIR={shlex.quote(host_dir)} cd "$HOST_DIR" docker compose up -d --force-recreate --no-deps {shlex.quote(service['service'])} docker ps --filter name={shlex.quote(container_name)} --format '{{{{.Names}}}} {{{{.Status}}}}' """ result = remote_command_result( service, script, f"restart-{service['host']}-{service['service']}", timeout_seconds=max(ROLLING_RESTART_TIMEOUT_SECONDS, 60), ) if result.get("status") != "SUCCESS": raise RuntimeError(f"restart failed on {service['host']}: {result.get('status')}") return result def wait_http_ready(service: dict[str, Any], timeout_seconds: int) -> dict[str, Any]: # 直接复用现有健康探测逻辑,不走缓存,确保拿的是最新状态。 deadline = time.time() + timeout_seconds last_result: dict[str, Any] | None = None while time.time() < deadline: last_result = probe_service(service, THRESHOLDS["latencyMs"]) if last_result.get("ok"): return last_result time.sleep(ROLLING_RESTART_POLL_SECONDS) detail = (last_result or {}).get("detail") or "service did not recover" raise RuntimeError(f"http health not ready on {service['host']} {service['service']}: {detail}") def matched_nacos_record(items: list[dict[str, Any]], service: dict[str, Any]) -> dict[str, Any] | None: # 只认当前宿主机对应的实例,避免误把另一台机器当成恢复成功。 for item in items: ip = str(item.get("ip") or "").strip() port = int(item.get("port") or 0) if ip == service["ip"] and port == int(service["port"]): return item return None def wait_java_registration_ready(service: dict[str, Any], timeout_seconds: int) -> dict[str, Any]: # 只有走 Nacos 注册的 Java 服务才做注册恢复校验。 registry_name = NACOS_SERVICE_NAME_MAP.get(service["service"]) if not registry_name: return {"checked": False, "reason": "service not managed by nacos registry map"} namespace_id, _ = current_namespace() deadline = time.time() + timeout_seconds last_record: dict[str, Any] | None = None while time.time() < deadline: try: last_record = matched_nacos_record( list_instances(namespace_id, registry_name, NACOS_DISCOVERY_GROUP), service, ) except Exception: # noqa: BLE001 last_record = None if last_record and nacos_truthy(last_record.get("healthy", False)) and nacos_truthy(last_record.get("enabled", True)): return { "checked": True, "healthy": True, "enabled": True, "source": "instance/list", } try: catalog_record = matched_nacos_record( catalog_instances(namespace_id, registry_name, NACOS_DISCOVERY_GROUP), service, ) except Exception: # noqa: BLE001 catalog_record = None if catalog_record and nacos_truthy(catalog_record.get("healthy", False)) and nacos_truthy( catalog_record.get("enabled", True) ): return { "checked": True, "healthy": True, "enabled": True, "source": "catalog/instances", } time.sleep(ROLLING_RESTART_POLL_SECONDS) raise RuntimeError(f"nacos registration not ready on {service['host']} {service['service']}") def clear_operation_caches() -> None: # 重启会直接影响服务探测和 Nacos 面板,完成后把缓存清掉。 SERVICE_CACHE.clear() NACOS_CACHE.clear() def rolling_restart_payload(service_names: list[str]) -> dict[str, Any]: # 用户可能重复点击同一个服务,这里先去重并保序。 normalized_names = [str(name or "").strip() for name in service_names if str(name or "").strip()] normalized_names = list(dict.fromkeys(normalized_names)) if not normalized_names: raise RuntimeError("services are required") records_map = service_records_by_name() steps: list[dict[str, Any]] = [] prepared_hosts: dict[str, dict[str, Any]] = {} for service_name in normalized_names: if service_name not in records_map: raise RuntimeError(f"unsupported restart service: {service_name}") for service in records_map[service_name]: started_at = time.time() step: dict[str, Any] = { "service": service_name, "host": service["host"], "ip": service["ip"], "kind": service["kind"], "status": "running", } try: # 单机第一次进入重启前,先把完整 compose 模板同步到远端。 if service["host"] not in prepared_hosts: prepared_hosts[service["host"]] = sync_remote_host_compose_template( service["host"], refresh_from_remote=True, ) step["compose"] = { "path": prepared_hosts[service["host"]]["path"], "changed": bool(prepared_hosts[service["host"]].get("changed")), "source": str(prepared_hosts[service["host"]].get("source") or ""), } restart_result = restart_service_instance(service) step["remoteStatus"] = restart_result.get("status") step["remoteOutput"] = str(restart_result.get("output") or "").strip()[:500] http_result = wait_http_ready(service, ROLLING_RESTART_TIMEOUT_SECONDS) step["http"] = { "ok": bool(http_result.get("ok")), "statusCode": int(http_result.get("statusCode") or 0), "latencyMs": int(http_result.get("latencyMs") or 0), "detail": str(http_result.get("detail") or ""), } if service["kind"] == "java": step["nacos"] = wait_java_registration_ready(service, ROLLING_RESTART_TIMEOUT_SECONDS) if ROLLING_RESTART_STABILIZE_SECONDS > 0: time.sleep(ROLLING_RESTART_STABILIZE_SECONDS) step["status"] = "success" step["durationSeconds"] = round(time.time() - started_at, 2) steps.append(step) except Exception as exc: # noqa: BLE001 step["status"] = "failed" step["error"] = str(exc) step["durationSeconds"] = round(time.time() - started_at, 2) steps.append(step) clear_operation_caches() return { "ok": False, "updatedAt": utc_now(), "services": normalized_names, "steps": steps, "error": step["error"], } clear_operation_caches() return { "ok": True, "updatedAt": utc_now(), "services": normalized_names, "steps": steps, "message": "rolling restart complete", }