269 lines
9.8 KiB
Python
269 lines
9.8 KiB
Python
from __future__ import annotations
|
|
|
|
# 远端命令里会拼接 shell 参数，需要显式转义。
|
|
import shlex
|
|
# 滚动重启是串行动作，轮询恢复要用时间模块。
|
|
import time
|
|
from typing import Any
|
|
|
|
from .compose_templates import sync_remote_host_compose_template
|
|
from core.config import (
|
|
ROLLING_RESTART_POLL_SECONDS,
|
|
ROLLING_RESTART_STABILIZE_SECONDS,
|
|
ROLLING_RESTART_TIMEOUT_SECONDS,
|
|
RUNTIME_BASE_DIR,
|
|
THRESHOLDS,
|
|
flatten_services,
|
|
load_hosts,
|
|
utc_now,
|
|
)
|
|
from .http_probe import SERVICE_CACHE, probe_service
|
|
from .nacos import (
|
|
NACOS_CACHE,
|
|
NACOS_DISCOVERY_GROUP,
|
|
NACOS_SERVICE_NAME_MAP,
|
|
catalog_instances,
|
|
current_namespace,
|
|
list_instances,
|
|
nacos_truthy,
|
|
)
|
|
from .ssh_remote import run_host_script
|
|
|
|
|
|
def restartable_service_records() -> list[dict[str, Any]]:
|
|
# 当前只开放 Java / Golang 业务服务滚动重启。
|
|
services = flatten_services(load_hosts())
|
|
return [service for service in services if service.get("kind") in {"java", "golang"}]
|
|
|
|
|
|
def build_restart_targets_payload() -> dict[str, Any]:
|
|
# 页面需要按“服务 -> 宿主机”展示选择项。
|
|
grouped: dict[str, dict[str, Any]] = {}
|
|
for service in restartable_service_records():
|
|
item = grouped.setdefault(
|
|
service["service"],
|
|
{
|
|
"service": service["service"],
|
|
"kind": service["kind"],
|
|
"hosts": [],
|
|
},
|
|
)
|
|
item["hosts"].append(
|
|
{
|
|
"host": service["host"],
|
|
"ip": service["ip"],
|
|
"instanceId": service["instanceId"],
|
|
"port": service["port"],
|
|
"path": service["path"],
|
|
}
|
|
)
|
|
|
|
return {
|
|
"ok": True,
|
|
"updatedAt": utc_now(),
|
|
"items": list(grouped.values()),
|
|
}
|
|
|
|
|
|
def service_records_by_name() -> dict[str, list[dict[str, Any]]]:
|
|
# 重启顺序沿用 targets.json 的主机顺序。
|
|
grouped: dict[str, list[dict[str, Any]]] = {}
|
|
for service in restartable_service_records():
|
|
grouped.setdefault(service["service"], []).append(service)
|
|
return grouped
|
|
|
|
|
|
def remote_command_result(service: dict[str, Any], script: str, command_name: str, timeout_seconds: int) -> dict[str, Any]:
|
|
# 单实例远端执行现在统一复用 SSH shell。
|
|
return run_host_script(
|
|
service,
|
|
script,
|
|
command_name=command_name,
|
|
timeout_seconds=timeout_seconds,
|
|
)
|
|
|
|
|
|
def restart_service_instance(service: dict[str, Any]) -> dict[str, Any]:
|
|
# 线上业务目录和 compose 文件复用现网部署结构。
|
|
host_dir = f"{RUNTIME_BASE_DIR.rstrip('/')}/{service['host']}"
|
|
container_name = f"likei-{service['host']}-{service['service']}"
|
|
script = f"""set -eu
|
|
HOST_DIR={shlex.quote(host_dir)}
|
|
cd "$HOST_DIR"
|
|
docker compose up -d --force-recreate --no-deps {shlex.quote(service['service'])}
|
|
docker ps --filter name={shlex.quote(container_name)} --format '{{{{.Names}}}} {{{{.Status}}}}'
|
|
"""
|
|
result = remote_command_result(
|
|
service,
|
|
script,
|
|
f"restart-{service['host']}-{service['service']}",
|
|
timeout_seconds=max(ROLLING_RESTART_TIMEOUT_SECONDS, 60),
|
|
)
|
|
if result.get("status") != "SUCCESS":
|
|
raise RuntimeError(f"restart failed on {service['host']}: {result.get('status')}")
|
|
return result
|
|
|
|
|
|
def wait_http_ready(service: dict[str, Any], timeout_seconds: int) -> dict[str, Any]:
|
|
# 直接复用现有健康探测逻辑，不走缓存，确保拿的是最新状态。
|
|
deadline = time.time() + timeout_seconds
|
|
last_result: dict[str, Any] | None = None
|
|
|
|
while time.time() < deadline:
|
|
last_result = probe_service(service, THRESHOLDS["latencyMs"])
|
|
if last_result.get("ok"):
|
|
return last_result
|
|
time.sleep(ROLLING_RESTART_POLL_SECONDS)
|
|
|
|
detail = (last_result or {}).get("detail") or "service did not recover"
|
|
raise RuntimeError(f"http health not ready on {service['host']} {service['service']}: {detail}")
|
|
|
|
|
|
def matched_nacos_record(items: list[dict[str, Any]], service: dict[str, Any]) -> dict[str, Any] | None:
|
|
# 只认当前宿主机对应的实例，避免误把另一台机器当成恢复成功。
|
|
for item in items:
|
|
ip = str(item.get("ip") or "").strip()
|
|
port = int(item.get("port") or 0)
|
|
if ip == service["ip"] and port == int(service["port"]):
|
|
return item
|
|
return None
|
|
|
|
|
|
def wait_java_registration_ready(service: dict[str, Any], timeout_seconds: int) -> dict[str, Any]:
|
|
# 只有走 Nacos 注册的 Java 服务才做注册恢复校验。
|
|
registry_name = NACOS_SERVICE_NAME_MAP.get(service["service"])
|
|
if not registry_name:
|
|
return {"checked": False, "reason": "service not managed by nacos registry map"}
|
|
|
|
namespace_id, _ = current_namespace()
|
|
deadline = time.time() + timeout_seconds
|
|
last_record: dict[str, Any] | None = None
|
|
|
|
while time.time() < deadline:
|
|
try:
|
|
last_record = matched_nacos_record(
|
|
list_instances(namespace_id, registry_name, NACOS_DISCOVERY_GROUP),
|
|
service,
|
|
)
|
|
except Exception: # noqa: BLE001
|
|
last_record = None
|
|
|
|
if last_record and nacos_truthy(last_record.get("healthy", False)) and nacos_truthy(last_record.get("enabled", True)):
|
|
return {
|
|
"checked": True,
|
|
"healthy": True,
|
|
"enabled": True,
|
|
"source": "instance/list",
|
|
}
|
|
|
|
try:
|
|
catalog_record = matched_nacos_record(
|
|
catalog_instances(namespace_id, registry_name, NACOS_DISCOVERY_GROUP),
|
|
service,
|
|
)
|
|
except Exception: # noqa: BLE001
|
|
catalog_record = None
|
|
|
|
if catalog_record and nacos_truthy(catalog_record.get("healthy", False)) and nacos_truthy(
|
|
catalog_record.get("enabled", True)
|
|
):
|
|
return {
|
|
"checked": True,
|
|
"healthy": True,
|
|
"enabled": True,
|
|
"source": "catalog/instances",
|
|
}
|
|
|
|
time.sleep(ROLLING_RESTART_POLL_SECONDS)
|
|
|
|
raise RuntimeError(f"nacos registration not ready on {service['host']} {service['service']}")
|
|
|
|
|
|
def clear_operation_caches() -> None:
|
|
# 重启会直接影响服务探测和 Nacos 面板，完成后把缓存清掉。
|
|
SERVICE_CACHE.clear()
|
|
NACOS_CACHE.clear()
|
|
|
|
|
|
def rolling_restart_payload(service_names: list[str]) -> dict[str, Any]:
|
|
# 用户可能重复点击同一个服务，这里先去重并保序。
|
|
normalized_names = [str(name or "").strip() for name in service_names if str(name or "").strip()]
|
|
normalized_names = list(dict.fromkeys(normalized_names))
|
|
if not normalized_names:
|
|
raise RuntimeError("services are required")
|
|
|
|
records_map = service_records_by_name()
|
|
steps: list[dict[str, Any]] = []
|
|
prepared_hosts: dict[str, dict[str, Any]] = {}
|
|
|
|
for service_name in normalized_names:
|
|
if service_name not in records_map:
|
|
raise RuntimeError(f"unsupported restart service: {service_name}")
|
|
|
|
for service in records_map[service_name]:
|
|
started_at = time.time()
|
|
step: dict[str, Any] = {
|
|
"service": service_name,
|
|
"host": service["host"],
|
|
"ip": service["ip"],
|
|
"kind": service["kind"],
|
|
"status": "running",
|
|
}
|
|
|
|
try:
|
|
# 单机第一次进入重启前，先把完整 compose 模板同步到远端。
|
|
if service["host"] not in prepared_hosts:
|
|
prepared_hosts[service["host"]] = sync_remote_host_compose_template(
|
|
service["host"],
|
|
refresh_from_remote=True,
|
|
)
|
|
step["compose"] = {
|
|
"path": prepared_hosts[service["host"]]["path"],
|
|
"changed": bool(prepared_hosts[service["host"]].get("changed")),
|
|
"source": str(prepared_hosts[service["host"]].get("source") or ""),
|
|
}
|
|
|
|
restart_result = restart_service_instance(service)
|
|
step["remoteStatus"] = restart_result.get("status")
|
|
step["remoteOutput"] = str(restart_result.get("output") or "").strip()[:500]
|
|
|
|
http_result = wait_http_ready(service, ROLLING_RESTART_TIMEOUT_SECONDS)
|
|
step["http"] = {
|
|
"ok": bool(http_result.get("ok")),
|
|
"statusCode": int(http_result.get("statusCode") or 0),
|
|
"latencyMs": int(http_result.get("latencyMs") or 0),
|
|
"detail": str(http_result.get("detail") or ""),
|
|
}
|
|
|
|
if service["kind"] == "java":
|
|
step["nacos"] = wait_java_registration_ready(service, ROLLING_RESTART_TIMEOUT_SECONDS)
|
|
|
|
if ROLLING_RESTART_STABILIZE_SECONDS > 0:
|
|
time.sleep(ROLLING_RESTART_STABILIZE_SECONDS)
|
|
|
|
step["status"] = "success"
|
|
step["durationSeconds"] = round(time.time() - started_at, 2)
|
|
steps.append(step)
|
|
except Exception as exc: # noqa: BLE001
|
|
step["status"] = "failed"
|
|
step["error"] = str(exc)
|
|
step["durationSeconds"] = round(time.time() - started_at, 2)
|
|
steps.append(step)
|
|
clear_operation_caches()
|
|
return {
|
|
"ok": False,
|
|
"updatedAt": utc_now(),
|
|
"services": normalized_names,
|
|
"steps": steps,
|
|
"error": step["error"],
|
|
}
|
|
|
|
clear_operation_caches()
|
|
return {
|
|
"ok": True,
|
|
"updatedAt": utc_now(),
|
|
"services": normalized_names,
|
|
"steps": steps,
|
|
"message": "rolling restart complete",
|
|
}
|