411 lines
16 KiB
Python
411 lines
16 KiB
Python
from __future__ import annotations
|
||
|
||
# 远端命令里会拼接 shell 参数，需要显式转义。
|
||
import shlex
|
||
# 滚动重启是串行动作，轮询恢复要用时间模块。
|
||
import time
|
||
from typing import Any
|
||
|
||
from .compose_templates import sync_remote_host_compose_template
|
||
from core.config import (
|
||
NACOS_RELEASE_DRAIN_SECONDS,
|
||
NACOS_RELEASE_DRAIN_SERVICES,
|
||
NACOS_RELEASE_DRAIN_TIMEOUT_SECONDS,
|
||
ROLLING_RESTART_POLL_SECONDS,
|
||
ROLLING_RESTART_STABILIZE_SECONDS,
|
||
ROLLING_RESTART_TIMEOUT_SECONDS,
|
||
RUNTIME_BASE_DIR,
|
||
THRESHOLDS,
|
||
flatten_services,
|
||
load_hosts,
|
||
utc_now,
|
||
)
|
||
from .http_probe import SERVICE_CACHE, probe_service
|
||
from .nacos import (
|
||
NACOS_CACHE,
|
||
NACOS_DISCOVERY_GROUP,
|
||
NACOS_SERVICE_NAME_MAP,
|
||
catalog_instances,
|
||
current_namespace,
|
||
list_instances,
|
||
nacos_truthy,
|
||
set_instance_enabled,
|
||
)
|
||
from .ssh_remote import run_host_script
|
||
|
||
|
||
def restartable_service_records() -> list[dict[str, Any]]:
|
||
# 当前只开放 Java / Golang 业务服务滚动重启。
|
||
services = flatten_services(load_hosts())
|
||
return [service for service in services if service.get("kind") in {"java", "golang"}]
|
||
|
||
|
||
def build_restart_targets_payload() -> dict[str, Any]:
|
||
# 页面需要按“服务 -> 宿主机”展示选择项。
|
||
grouped: dict[str, dict[str, Any]] = {}
|
||
for service in restartable_service_records():
|
||
item = grouped.setdefault(
|
||
service["service"],
|
||
{
|
||
"service": service["service"],
|
||
"kind": service["kind"],
|
||
"hosts": [],
|
||
},
|
||
)
|
||
item["hosts"].append(
|
||
{
|
||
"host": service["host"],
|
||
"ip": service["ip"],
|
||
"instanceId": service["instanceId"],
|
||
"port": service["port"],
|
||
"path": service["path"],
|
||
}
|
||
)
|
||
|
||
return {
|
||
"ok": True,
|
||
"updatedAt": utc_now(),
|
||
"items": list(grouped.values()),
|
||
}
|
||
|
||
|
||
def service_records_by_name() -> dict[str, list[dict[str, Any]]]:
|
||
# 重启顺序沿用 targets.json 的主机顺序。
|
||
grouped: dict[str, list[dict[str, Any]]] = {}
|
||
for service in restartable_service_records():
|
||
grouped.setdefault(service["service"], []).append(service)
|
||
return grouped
|
||
|
||
|
||
def remote_command_result(service: dict[str, Any], script: str, command_name: str, timeout_seconds: int) -> dict[str, Any]:
|
||
# 单实例远端执行现在统一复用 SSH shell。
|
||
return run_host_script(
|
||
service,
|
||
script,
|
||
command_name=command_name,
|
||
timeout_seconds=timeout_seconds,
|
||
)
|
||
|
||
|
||
def restart_service_instance(service: dict[str, Any]) -> dict[str, Any]:
|
||
# 线上业务目录和 compose 文件复用现网部署结构。
|
||
host_dir = f"{RUNTIME_BASE_DIR.rstrip('/')}/{service['host']}"
|
||
container_name = f"likei-{service['host']}-{service['service']}"
|
||
script = f"""set -eu
|
||
HOST_DIR={shlex.quote(host_dir)}
|
||
cd "$HOST_DIR"
|
||
docker compose up -d --force-recreate --no-deps {shlex.quote(service['service'])}
|
||
docker ps --filter name={shlex.quote(container_name)} --format '{{{{.Names}}}} {{{{.Status}}}}'
|
||
"""
|
||
result = remote_command_result(
|
||
service,
|
||
script,
|
||
f"restart-{service['host']}-{service['service']}",
|
||
timeout_seconds=max(ROLLING_RESTART_TIMEOUT_SECONDS, 60),
|
||
)
|
||
if result.get("status") != "SUCCESS":
|
||
raise RuntimeError(f"restart failed on {service['host']}: {result.get('status')}")
|
||
return result
|
||
|
||
|
||
def wait_http_ready(service: dict[str, Any], timeout_seconds: int) -> dict[str, Any]:
|
||
# 直接复用现有健康探测逻辑，不走缓存，确保拿的是最新状态。
|
||
deadline = time.time() + timeout_seconds
|
||
last_result: dict[str, Any] | None = None
|
||
|
||
while time.time() < deadline:
|
||
last_result = probe_service(service, THRESHOLDS["latencyMs"])
|
||
if last_result.get("ok"):
|
||
return last_result
|
||
time.sleep(ROLLING_RESTART_POLL_SECONDS)
|
||
|
||
detail = (last_result or {}).get("detail") or "service did not recover"
|
||
raise RuntimeError(f"http health not ready on {service['host']} {service['service']}: {detail}")
|
||
|
||
|
||
def matched_nacos_record(items: list[dict[str, Any]], service: dict[str, Any]) -> dict[str, Any] | None:
|
||
# 只认当前宿主机对应的实例，避免误把另一台机器当成恢复成功。
|
||
for item in items:
|
||
ip = str(item.get("ip") or "").strip()
|
||
port = int(item.get("port") or 0)
|
||
if ip == service["ip"] and port == int(service["port"]):
|
||
return item
|
||
return None
|
||
|
||
|
||
def wait_java_registration_ready(service: dict[str, Any], timeout_seconds: int) -> dict[str, Any]:
|
||
# 只有走 Nacos 注册的 Java 服务才做注册恢复校验。
|
||
registry_name = NACOS_SERVICE_NAME_MAP.get(service["service"])
|
||
if not registry_name:
|
||
return {"checked": False, "reason": "service not managed by nacos registry map"}
|
||
|
||
namespace_id, _ = current_namespace()
|
||
deadline = time.time() + timeout_seconds
|
||
last_record: dict[str, Any] | None = None
|
||
|
||
while time.time() < deadline:
|
||
try:
|
||
last_record = matched_nacos_record(
|
||
list_instances(namespace_id, registry_name, NACOS_DISCOVERY_GROUP),
|
||
service,
|
||
)
|
||
except Exception: # noqa: BLE001
|
||
last_record = None
|
||
|
||
if last_record and nacos_truthy(last_record.get("healthy", False)) and nacos_truthy(last_record.get("enabled", True)):
|
||
return {
|
||
"checked": True,
|
||
"healthy": True,
|
||
"enabled": True,
|
||
"source": "instance/list",
|
||
}
|
||
|
||
try:
|
||
catalog_record = matched_nacos_record(
|
||
catalog_instances(namespace_id, registry_name, NACOS_DISCOVERY_GROUP),
|
||
service,
|
||
)
|
||
except Exception: # noqa: BLE001
|
||
catalog_record = None
|
||
|
||
if catalog_record and nacos_truthy(catalog_record.get("healthy", False)) and nacos_truthy(
|
||
catalog_record.get("enabled", True)
|
||
):
|
||
return {
|
||
"checked": True,
|
||
"healthy": True,
|
||
"enabled": True,
|
||
"source": "catalog/instances",
|
||
}
|
||
|
||
time.sleep(ROLLING_RESTART_POLL_SECONDS)
|
||
|
||
raise RuntimeError(f"nacos registration not ready on {service['host']} {service['service']}")
|
||
|
||
|
||
def should_drain_java_registration(service: dict[str, Any]) -> bool:
|
||
# 默认只对 other 做发布前摘流；其他服务需要时可通过 .env 扩展。
|
||
return service.get("kind") == "java" and str(service.get("service") or "").strip() in NACOS_RELEASE_DRAIN_SERVICES
|
||
|
||
|
||
def current_java_registration_record(service: dict[str, Any]) -> dict[str, Any] | None:
|
||
# 返回当前 ip:port 对应的 Nacos 实例，供摘流和恢复复用。
|
||
registry_name = NACOS_SERVICE_NAME_MAP.get(service["service"])
|
||
if not registry_name:
|
||
return None
|
||
|
||
namespace_id, _ = current_namespace()
|
||
return matched_nacos_record(list_instances(namespace_id, registry_name, NACOS_DISCOVERY_GROUP), service)
|
||
|
||
|
||
def update_java_registration_enabled(service: dict[str, Any], enabled: bool) -> dict[str, Any]:
|
||
# 只更新当前服务实例的 enabled 状态，不影响同服务其他宿主机。
|
||
registry_name = NACOS_SERVICE_NAME_MAP.get(service["service"])
|
||
if not registry_name:
|
||
return {"checked": False, "reason": "service not managed by nacos registry map"}
|
||
|
||
namespace_id, _ = current_namespace()
|
||
current_record = current_java_registration_record(service)
|
||
ephemeral = current_record.get("ephemeral") if current_record and "ephemeral" in current_record else True
|
||
response_text = set_instance_enabled(
|
||
namespace_id,
|
||
registry_name,
|
||
NACOS_DISCOVERY_GROUP,
|
||
ip=str(service["ip"]),
|
||
port=int(service["port"]),
|
||
enabled=enabled,
|
||
ephemeral=bool(ephemeral),
|
||
)
|
||
NACOS_CACHE.clear()
|
||
return {
|
||
"checked": True,
|
||
"serviceName": registry_name,
|
||
"enabled": enabled,
|
||
"response": response_text,
|
||
"ephemeral": bool(ephemeral),
|
||
}
|
||
|
||
|
||
def wait_java_registration_enabled_state(service: dict[str, Any], enabled: bool, timeout_seconds: int) -> dict[str, Any]:
|
||
# 等到 Nacos 读回当前实例 enabled 状态，避免刚写完就立刻重启。
|
||
registry_name = NACOS_SERVICE_NAME_MAP.get(service["service"])
|
||
if not registry_name:
|
||
return {"checked": False, "reason": "service not managed by nacos registry map"}
|
||
|
||
namespace_id, _ = current_namespace()
|
||
deadline = time.time() + timeout_seconds
|
||
last_record: dict[str, Any] | None = None
|
||
last_source = ""
|
||
|
||
while time.time() < deadline:
|
||
last_record = matched_nacos_record(list_instances(namespace_id, registry_name, NACOS_DISCOVERY_GROUP), service)
|
||
if last_record is not None and nacos_truthy(last_record.get("enabled", True)) == enabled:
|
||
return {
|
||
"checked": True,
|
||
"enabled": enabled,
|
||
"healthy": nacos_truthy(last_record.get("healthy", False)),
|
||
"source": "instance/list",
|
||
}
|
||
if last_record is not None:
|
||
last_source = "instance/list"
|
||
|
||
# Nacos 的 instance/list 可能不返回 disabled 实例，catalog/instances 仍能看到完整状态。
|
||
catalog_record = matched_nacos_record(catalog_instances(namespace_id, registry_name, NACOS_DISCOVERY_GROUP), service)
|
||
if catalog_record is not None and nacos_truthy(catalog_record.get("enabled", True)) == enabled:
|
||
return {
|
||
"checked": True,
|
||
"enabled": enabled,
|
||
"healthy": nacos_truthy(catalog_record.get("healthy", False)),
|
||
"source": "catalog/instances",
|
||
}
|
||
if catalog_record is not None:
|
||
last_record = catalog_record
|
||
last_source = "catalog/instances"
|
||
|
||
time.sleep(ROLLING_RESTART_POLL_SECONDS)
|
||
|
||
detail = ""
|
||
if last_record is not None:
|
||
detail = (
|
||
f" last {last_source or 'nacos'} enabled={last_record.get('enabled')} "
|
||
f"healthy={last_record.get('healthy')} ip={last_record.get('ip')} port={last_record.get('port')}"
|
||
)
|
||
raise RuntimeError(f"nacos enabled={enabled} not ready on {service['host']} {service['service']}{detail}")
|
||
|
||
|
||
def drain_java_registration(service: dict[str, Any]) -> dict[str, Any]:
|
||
# 摘掉当前实例后给调用方一点缓存刷新时间，避免 live 继续打到正在重启的 other。
|
||
if not should_drain_java_registration(service):
|
||
return {"checked": False, "reason": "service not configured for nacos release drain"}
|
||
|
||
update_result = update_java_registration_enabled(service, False)
|
||
try:
|
||
state_result = wait_java_registration_enabled_state(service, False, NACOS_RELEASE_DRAIN_TIMEOUT_SECONDS)
|
||
except Exception:
|
||
# 如果摘流确认失败，尽量立刻恢复 enabled=true，避免业务实例长期停留在 Nacos 禁用状态。
|
||
update_java_registration_enabled(service, True)
|
||
raise
|
||
if NACOS_RELEASE_DRAIN_SECONDS > 0:
|
||
time.sleep(NACOS_RELEASE_DRAIN_SECONDS)
|
||
return {
|
||
**state_result,
|
||
"drainSeconds": NACOS_RELEASE_DRAIN_SECONDS,
|
||
"update": update_result,
|
||
}
|
||
|
||
|
||
def restore_java_registration(service: dict[str, Any]) -> dict[str, Any]:
|
||
# 容器 HTTP 恢复后再打开 Nacos enabled，并读回健康状态。
|
||
update_result = update_java_registration_enabled(service, True)
|
||
ready_result = wait_java_registration_ready(service, ROLLING_RESTART_TIMEOUT_SECONDS)
|
||
return {
|
||
**ready_result,
|
||
"restored": True,
|
||
"update": update_result,
|
||
}
|
||
|
||
|
||
def clear_operation_caches() -> None:
|
||
# 重启会直接影响服务探测和 Nacos 面板，完成后把缓存清掉。
|
||
SERVICE_CACHE.clear()
|
||
NACOS_CACHE.clear()
|
||
|
||
|
||
def rolling_restart_payload(service_names: list[str]) -> dict[str, Any]:
|
||
# 用户可能重复点击同一个服务，这里先去重并保序。
|
||
normalized_names = [str(name or "").strip() for name in service_names if str(name or "").strip()]
|
||
normalized_names = list(dict.fromkeys(normalized_names))
|
||
if not normalized_names:
|
||
raise RuntimeError("services are required")
|
||
|
||
records_map = service_records_by_name()
|
||
steps: list[dict[str, Any]] = []
|
||
prepared_hosts: dict[str, dict[str, Any]] = {}
|
||
|
||
for service_name in normalized_names:
|
||
if service_name not in records_map:
|
||
raise RuntimeError(f"unsupported restart service: {service_name}")
|
||
|
||
for service in records_map[service_name]:
|
||
started_at = time.time()
|
||
step: dict[str, Any] = {
|
||
"service": service_name,
|
||
"host": service["host"],
|
||
"ip": service["ip"],
|
||
"kind": service["kind"],
|
||
"status": "running",
|
||
}
|
||
nacos_drained = False
|
||
|
||
try:
|
||
# 单机第一次进入重启前，先把完整 compose 模板同步到远端。
|
||
if service["host"] not in prepared_hosts:
|
||
prepared_hosts[service["host"]] = sync_remote_host_compose_template(
|
||
service["host"],
|
||
refresh_from_remote=True,
|
||
)
|
||
step["compose"] = {
|
||
"path": prepared_hosts[service["host"]]["path"],
|
||
"changed": bool(prepared_hosts[service["host"]].get("changed")),
|
||
"source": str(prepared_hosts[service["host"]].get("source") or ""),
|
||
}
|
||
|
||
if service["kind"] == "java":
|
||
drain_result = drain_java_registration(service)
|
||
if drain_result.get("checked"):
|
||
nacos_drained = True
|
||
step["nacosDrain"] = drain_result
|
||
|
||
restart_result = restart_service_instance(service)
|
||
step["remoteStatus"] = restart_result.get("status")
|
||
step["remoteOutput"] = str(restart_result.get("output") or "").strip()[:500]
|
||
|
||
http_result = wait_http_ready(service, ROLLING_RESTART_TIMEOUT_SECONDS)
|
||
step["http"] = {
|
||
"ok": bool(http_result.get("ok")),
|
||
"statusCode": int(http_result.get("statusCode") or 0),
|
||
"latencyMs": int(http_result.get("latencyMs") or 0),
|
||
"detail": str(http_result.get("detail") or ""),
|
||
}
|
||
|
||
if service["kind"] == "java":
|
||
step["nacos"] = (
|
||
restore_java_registration(service)
|
||
if nacos_drained
|
||
else wait_java_registration_ready(service, ROLLING_RESTART_TIMEOUT_SECONDS)
|
||
)
|
||
|
||
if ROLLING_RESTART_STABILIZE_SECONDS > 0:
|
||
time.sleep(ROLLING_RESTART_STABILIZE_SECONDS)
|
||
|
||
step["status"] = "success"
|
||
step["durationSeconds"] = round(time.time() - started_at, 2)
|
||
steps.append(step)
|
||
except Exception as exc: # noqa: BLE001
|
||
if nacos_drained:
|
||
try:
|
||
step["nacosRestoreRecovered"] = update_java_registration_enabled(service, True)
|
||
except Exception as restore_exc: # noqa: BLE001
|
||
step["nacosRestoreError"] = str(restore_exc)
|
||
step["status"] = "failed"
|
||
step["error"] = str(exc)
|
||
step["durationSeconds"] = round(time.time() - started_at, 2)
|
||
steps.append(step)
|
||
clear_operation_caches()
|
||
return {
|
||
"ok": False,
|
||
"updatedAt": utc_now(),
|
||
"services": normalized_names,
|
||
"steps": steps,
|
||
"error": step["error"],
|
||
}
|
||
|
||
clear_operation_caches()
|
||
return {
|
||
"ok": True,
|
||
"updatedAt": utc_now(),
|
||
"services": normalized_names,
|
||
"steps": steps,
|
||
"message": "rolling restart complete",
|
||
}
|