From e700149a997183a4048b89c7e11ff3f452890a25 Mon Sep 17 00:00:00 2001 From: hy Date: Wed, 22 Apr 2026 15:52:52 +0800 Subject: [PATCH] =?UTF-8?q?=E5=A2=9E=E5=8A=A0=E6=BB=9A=E5=8A=A8=E9=87=8D?= =?UTF-8?q?=E5=90=AF?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- README.md | 4 +- monitor/config.py | 8 + monitor/config_validation.py | 161 ++++++++ monitor/http_app.py | 137 ++++++- monitor/nacos.py | 28 ++ monitor/runtime_config.py | 697 +++++++++++++++++++++++++++++++++++ monitor/service_ops.py | 255 +++++++++++++ requirements-ops.txt | 2 + static/app.css | 37 ++ static/app.js | 235 +++++++++++- static/index.html | 141 ++++++- 11 files changed, 1679 insertions(+), 26 deletions(-) create mode 100644 monitor/config_validation.py create mode 100644 monitor/runtime_config.py create mode 100644 monitor/service_ops.py diff --git a/README.md b/README.md index f6022b9..4fbf11c 100644 --- a/README.md +++ b/README.md @@ -106,6 +106,8 @@ python3 ops/deploy_via_tat.py - 主机：`CPU / 内存 / 磁盘 / 进程数`，带阈值颜色分级 - 服务：Java / Golang 健康检查和耗时 - Nacos：命名空间、服务、注册实例、健康状态 -- Nacos 配置：读取当前 namespace 下的配置列表，支持在线编辑保存 +- Nacos 配置：读取当前 namespace 下的配置列表，支持格式校验、在线编辑保存 +- Golang 配置：读取 app 节点当前运行中的 `service.env`，展示 `chatapp3-golang` 实际消费的环境变量并支持保存 +- 滚动重启：可按服务选择 Java / Golang 节点，逐台重启并等待健康恢复 - MongoDB：连接、内存、网络、操作计数、复制集、数据库统计 - 自动刷新：前端可切换刷新间隔，并保存在浏览器本地 diff --git a/monitor/config.py b/monitor/config.py index fbd3b93..c835e32 100644 --- a/monitor/config.py +++ b/monitor/config.py @@ -37,6 +37,14 @@ HOST_METRIC_CACHE_TTL_SECONDS = env_float("HOST_METRIC_CACHE_TTL_SECONDS", 25.0) TAT_METRIC_TIMEOUT_SECONDS = env_int("TAT_METRIC_TIMEOUT_SECONDS", 30) # TAT CPU 采样间隔。 TAT_CPU_SAMPLE_SECONDS = env_float("TAT_CPU_SAMPLE_SECONDS", 1.0) +# 业务运行目录根路径。 +RUNTIME_BASE_DIR = env_str("RUNTIME_BASE_DIR", "/opt/likei") +# 滚动重启单实例最大等待秒数。 +ROLLING_RESTART_TIMEOUT_SECONDS = env_int("ROLLING_RESTART_TIMEOUT_SECONDS", 180) +# 滚动重启探测轮询间隔。 +ROLLING_RESTART_POLL_SECONDS = env_float("ROLLING_RESTART_POLL_SECONDS", 2.0) +# 单实例恢复后额外稳定等待秒数。 +ROLLING_RESTART_STABILIZE_SECONDS = env_float("ROLLING_RESTART_STABILIZE_SECONDS", 2.0) # TAT 所在区域。 DEPLOY_REGION = env_str("DEPLOY_REGION", "me-saudi-arabia") diff --git a/monitor/config_validation.py b/monitor/config_validation.py new file mode 100644 index 0000000..4bf2aa6 --- /dev/null +++ b/monitor/config_validation.py @@ -0,0 +1,161 @@ +from __future__ import annotations + +# JSON 校验直接走标准库。 +import json +# XML 校验直接走标准库。 +import xml.etree.ElementTree as element_tree +from typing import Any + + +def normalized_config_type(config_type: str, data_id: str = "") -> str: + # 统一把类型名压平，避免前后端大小写不一致。 + lowered = (config_type or "").strip().lower() + if lowered in {"yml", "yaml"}: + return "yaml" + if lowered in {"json", "properties", "xml", "html", "text"}: + return lowered + + # 没显式类型时，按常见后缀兜底。 + suffix = data_id.strip().lower() + if suffix.endswith((".yml", ".yaml")): + return "yaml" + if suffix.endswith(".json"): + return "json" + if suffix.endswith(".properties"): + return "properties" + if suffix.endswith(".xml"): + return "xml" + if suffix.endswith(".html"): + return "html" + return "text" + + +def dependency_error(message: str) -> dict[str, Any]: + # 缺少解析依赖时也返回统一结构，前端方便直接展示。 + return { + "ok": False, + "strict": False, + "type": "", + "message": message, + } + + +def validation_success(config_type: str, message: str, *, strict: bool = True, details: dict[str, Any] | None = None) -> dict[str, Any]: + # 成功结果也统一带上 strict 标记，告诉前端是不是强校验。 + return { + "ok": True, + "strict": strict, + "type": config_type, + "message": message, + "details": details or {}, + } + + +def validation_error(config_type: str, message: str, *, details: dict[str, Any] | None = None) -> dict[str, Any]: + # 失败结构保持和成功结构一致。 + return { + "ok": False, + "strict": True, + "type": config_type, + "message": message, + "details": details or {}, + } + + +def validate_yaml(content: str) -> dict[str, Any]: + try: + # 按需导入 PyYAML，避免依赖缺失时影响其他功能启动。 + import yaml + except Exception as exc: # noqa: BLE001 + return dependency_error(f"yaml validator unavailable: {exc}") + + try: + # 空 YAML 允许解析成 null。 + parsed = yaml.safe_load(content) + except yaml.YAMLError as exc: + # PyYAML 异常里通常会带问题行列。 + problem_mark = getattr(exc, "problem_mark", None) + details = {} + if problem_mark is not None: + details = {"line": int(problem_mark.line) + 1, "column": int(problem_mark.column) + 1} + return validation_error("yaml", str(exc), details=details) + + parsed_kind = type(parsed).__name__ if parsed is not None else "null" + return validation_success("yaml", f"yaml format ok ({parsed_kind})", details={"parsedKind": parsed_kind}) + + +def validate_json_text(content: str) -> dict[str, Any]: + try: + # JSON 必须是合法文本，空串直接视为错误。 + parsed = json.loads(content) + except json.JSONDecodeError as exc: + return validation_error( + "json", + exc.msg, + details={"line": int(exc.lineno), "column": int(exc.colno)}, + ) + + parsed_kind = type(parsed).__name__ + return validation_success("json", f"json format ok ({parsed_kind})", details={"parsedKind": parsed_kind}) + + +def validate_properties_text(content: str) -> dict[str, Any]: + try: + # Java properties 校验直接走专用解析器，避免手写规则误判。 + import javaproperties + except Exception as exc: # noqa: BLE001 + return dependency_error(f"properties validator unavailable: {exc}") + + try: + # loads 会在格式异常时抛出 ParseError。 + parsed = javaproperties.loads(content) + except Exception as exc: # noqa: BLE001 + line = getattr(exc, "lineno", None) + details = {"line": int(line)} if isinstance(line, int) else {} + return validation_error("properties", str(exc), details=details) + + return validation_success("properties", f"properties format ok ({len(parsed)} keys)", details={"keysCount": len(parsed)}) + + +def validate_xml_text(content: str) -> dict[str, Any]: + try: + # XML 用标准库直接做语法解析。 + root = element_tree.fromstring(content) + except element_tree.ParseError as exc: + line, column = getattr(exc, "position", (0, 0)) + return validation_error( + "xml", + str(exc), + details={"line": int(line), "column": int(column) + 1}, + ) + + return validation_success("xml", f"xml format ok (<{root.tag}>)", details={"rootTag": root.tag}) + + +def validate_text_like(config_type: str) -> dict[str, Any]: + # text/html 先不做严格语法校验，只给出明确提示。 + return validation_success( + config_type, + f"{config_type} saved as plain text; strict syntax validation is not enabled", + strict=False, + ) + + +def validate_config_content(content: str, *, config_type: str = "", data_id: str = "") -> dict[str, Any]: + # 统一解析类型。 + normalized_type = normalized_config_type(config_type, data_id) + + # 按类型路由到对应校验器。 + if normalized_type == "yaml": + return validate_yaml(content) + if normalized_type == "json": + return validate_json_text(content) + if normalized_type == "properties": + return validate_properties_text(content) + if normalized_type == "xml": + return validate_xml_text(content) + if normalized_type in {"html", "text"}: + return validate_text_like(normalized_type) + + # 理论上不会走到这里，这里只是兜底。 + return validation_success(normalized_type, "no validator configured", strict=False) diff --git a/monitor/http_app.py b/monitor/http_app.py index 04d12a2..d6e4fee 100644 --- a/monitor/http_app.py +++ b/monitor/http_app.py @@ -8,7 +8,14 @@ from urllib.parse import parse_qs, urlparse from .config import APP_BIND, APP_PORT, STATIC_DIR, utc_now from .dashboard import build_monitor_payload -from .nacos import get_nacos_config_list_payload, get_nacos_config_payload, save_nacos_config_payload +from .nacos import ( + get_nacos_config_list_payload, + get_nacos_config_payload, + save_nacos_config_payload, + validate_nacos_config_payload, +) +from .runtime_config import get_go_config_payload, save_go_config_payload, validate_go_config_payload +from .service_ops import build_restart_targets_payload, rolling_restart_payload def response_json(payload: dict[str, Any] | list[Any]) -> bytes: @@ -114,6 +121,34 @@ class MonitorHandler(BaseHTTPRequestHandler): send_body=send_body, ) + # 返回当前线上 golang 的运行配置。 + if parsed.path == "/api/runtime/golang-config": + try: + payload = get_go_config_payload() + except Exception as exc: # noqa: BLE001 + return self.send_error_payload(HTTPStatus.BAD_GATEWAY, str(exc), send_body=send_body) + + return self.send_payload( + HTTPStatus.OK, + response_json(payload), + "application/json; charset=utf-8", + send_body=send_body, + ) + + # 返回可滚动重启的业务服务。 + if parsed.path == "/api/ops/restart-targets": + try: + payload = build_restart_targets_payload() + except Exception as exc: # noqa: BLE001 + return self.send_error_payload(HTTPStatus.BAD_GATEWAY, str(exc), send_body=send_body) + + return self.send_payload( + HTTPStatus.OK, + response_json(payload), + "application/json; charset=utf-8", + send_body=send_body, + ) + # favicon 直接返回空。 if parsed.path == "/favicon.ico": return self.send_payload(HTTPStatus.NO_CONTENT, b"", "image/x-icon", send_body=send_body) @@ -125,15 +160,55 @@ class MonitorHandler(BaseHTTPRequestHandler): # 只解析 path 段，POST 不处理复杂 query。 parsed = urlparse(self.path) - # 目前只开放 Nacos 配置保存接口。 - if parsed.path != "/api/nacos/config": - return self.send_error_payload(HTTPStatus.NOT_FOUND, "not found", send_body=True) - try: payload = self.read_json_body() except ValueError as exc: return self.send_error_payload(HTTPStatus.BAD_REQUEST, str(exc), send_body=True) + if parsed.path == "/api/nacos/config/validate": + return self.handle_nacos_validate(payload) + + if parsed.path == "/api/nacos/config": + return self.handle_nacos_save(payload) + + if parsed.path == "/api/runtime/golang-config/validate": + return self.handle_go_validate(payload) + + if parsed.path == "/api/runtime/golang-config": + return self.handle_go_save(payload) + + if parsed.path == "/api/ops/rolling-restart": + return self.handle_rolling_restart(payload) + + return self.send_error_payload(HTTPStatus.NOT_FOUND, "not found", send_body=True) + + def handle_nacos_validate(self, payload: dict[str, Any]) -> None: + group_name = str(payload.get("group") or "").strip() + data_id = str(payload.get("dataId") or "").strip() + content = payload.get("content") + config_type = str(payload.get("type") or "").strip() or None + + if not isinstance(content, str): + return self.send_error_payload(HTTPStatus.BAD_REQUEST, "content must be a string", send_body=True) + + try: + result = validate_nacos_config_payload( + content, + group_name=group_name, + data_id=data_id, + config_type=config_type, + ) + except Exception as exc: # noqa: BLE001 + return self.send_error_payload(HTTPStatus.BAD_GATEWAY, str(exc), send_body=True) + + return self.send_payload( + HTTPStatus.OK, + response_json(result), + "application/json; charset=utf-8", + send_body=True, + ) + + def handle_nacos_save(self, payload: dict[str, Any]) -> None: group_name = str(payload.get("group") or "").strip() data_id = str(payload.get("dataId") or "").strip() content = payload.get("content") @@ -143,7 +218,6 @@ class MonitorHandler(BaseHTTPRequestHandler): if not group_name or not data_id: return self.send_error_payload(HTTPStatus.BAD_REQUEST, "group and dataId are required", send_body=True) - # content 必须是字符串；允许保存空串。 if not isinstance(content, str): return self.send_error_payload(HTTPStatus.BAD_REQUEST, "content must be a string", send_body=True) @@ -159,6 +233,57 @@ class MonitorHandler(BaseHTTPRequestHandler): send_body=True, ) + def handle_go_validate(self, payload: dict[str, Any]) -> None: + content = payload.get("content") + if not isinstance(content, str): + return self.send_error_payload(HTTPStatus.BAD_REQUEST, "content must be a string", send_body=True) + + try: + result = validate_go_config_payload(content) + except Exception as exc: # noqa: BLE001 + return self.send_error_payload(HTTPStatus.BAD_GATEWAY, str(exc), send_body=True) + + return self.send_payload( + HTTPStatus.OK, + response_json(result), + "application/json; charset=utf-8", + send_body=True, + ) + + def handle_go_save(self, payload: dict[str, Any]) -> None: + content = payload.get("content") + if not isinstance(content, str): + return self.send_error_payload(HTTPStatus.BAD_REQUEST, "content must be a string", send_body=True) + + try: + result = save_go_config_payload(content) + except Exception as exc: # noqa: BLE001 + return self.send_error_payload(HTTPStatus.BAD_GATEWAY, str(exc), send_body=True) + + return self.send_payload( + HTTPStatus.OK, + response_json(result), + "application/json; charset=utf-8", + send_body=True, + ) + + def handle_rolling_restart(self, payload: dict[str, Any]) -> None: + services = payload.get("services") + if not isinstance(services, list): + return self.send_error_payload(HTTPStatus.BAD_REQUEST, "services must be a list", send_body=True) + + try: + result = rolling_restart_payload([str(item or "").strip() for item in services]) + except Exception as exc: # noqa: BLE001 + return self.send_error_payload(HTTPStatus.BAD_GATEWAY, str(exc), send_body=True) + + return self.send_payload( + HTTPStatus.OK, + response_json(result), + "application/json; charset=utf-8", + send_body=True, + ) + def query_value(self, query: dict[str, list[str]], key: str) -> str: # 统一读取单值 query 参数。 return str((query.get(key) or [""])[0]).strip() diff --git a/monitor/nacos.py b/monitor/nacos.py index eb1a9ad..5034f7a 100644 --- a/monitor/nacos.py +++ b/monitor/nacos.py @@ -9,6 +9,7 @@ import urllib.request from typing import Any from .cache import TTLCache +from .config_validation import validate_config_content from .config import ( NACOS_BASE_URL, NACOS_CACHE_TTL_SECONDS, @@ -376,9 +377,36 @@ def get_nacos_config_payload(group_name: str, data_id: str) -> dict[str, Any]: } +def validate_nacos_config_payload(content: str, *, group_name: str = "", data_id: str = "", config_type: str | None = None) -> dict[str, Any]: + # 给页面单独暴露一个“先校验再保存”的能力。 + result = validate_config_content( + content, + config_type=(config_type or "").strip(), + data_id=data_id, + ) + return { + **result, + "updatedAt": utc_now(), + "item": { + "group": group_name, + "dataId": data_id, + "type": (config_type or infer_config_type(data_id)).strip(), + }, + } + + def save_nacos_config_payload(group_name: str, data_id: str, content: str, config_type: str | None = None) -> dict[str, Any]: # 使用当前监控配置里的 namespace。 namespace_id, namespace_name = current_namespace() + # 保存前先做一次强校验，避免误把错误格式写进 Nacos。 + validation = validate_config_content( + content, + config_type=(config_type or "").strip(), + data_id=data_id, + ) + if not validation.get("ok"): + raise RuntimeError(str(validation.get("message") or f"invalid config content: {group_name}/{data_id}")) + # public 命名空间不显式传 tenant。 save_data: dict[str, Any] = { "group": group_name, diff --git a/monitor/runtime_config.py b/monitor/runtime_config.py new file mode 100644 index 0000000..4185672 --- /dev/null +++ b/monitor/runtime_config.py @@ -0,0 +1,697 @@ +from __future__ import annotations + +# 远端写文件时需要 base64 传输，避免引号和换行转义问题。 +import base64 +# 返回给前端的摘要信息会带 sha256。 +import hashlib +# env 值里有 JSON 字符串时直接复用标准库解析。 +import json +# 远端路径和 shell 参数需要安全转义。 +import shlex +from typing import Any + +from .config import RUNTIME_BASE_DIR, TAT_METRIC_TIMEOUT_SECONDS, load_hosts, utc_now +from .tencent_tat import run_shell_command + + +# Go 配置按源码里的运行分段排序，页面上也按这个顺序展示。 +GO_ENV_SPECS: list[dict[str, Any]] = [ + {"section": "HTTP", "primary": "CHATAPP_HTTP_LISTEN_ADDR", "aliases": ("INVITE_HTTP_ADDR",), "kind": "string"}, + { + "section": "HTTP", + "primary": "CHATAPP_HTTP_TIMEOUT_SECONDS", + "aliases": ("INVITE_HTTP_TIMEOUT_SECONDS",), + "kind": "int", + }, + { + "section": "HTTP", + "primary": "CHATAPP_HTTP_SHUTDOWN_TIMEOUT_SECONDS", + "aliases": ("INVITE_HTTP_SHUTDOWN_TIMEOUT_SECONDS",), + "kind": "int", + }, + { + "section": "HTTP", + "primary": "CHATAPP_HTTP_INTERNAL_CALLBACK_SECRET", + "aliases": ("GAME_INTERNAL_CALLBACK_SECRET",), + "kind": "string", + }, + {"section": "Store", "primary": "CHATAPP_STORE_MYSQL_DSN", "aliases": ("INVITE_MYSQL_DSN",), "kind": "string"}, + {"section": "Store", "primary": "CHATAPP_STORE_REDIS_ADDR", "aliases": ("INVITE_REDIS_ADDR",), "kind": "string"}, + { + "section": "Store", + "primary": "CHATAPP_STORE_REDIS_PASSWORD", + "aliases": ("INVITE_REDIS_PASSWORD",), + "kind": "string", + }, + {"section": "Store", "primary": "CHATAPP_STORE_REDIS_DB", "aliases": ("INVITE_REDIS_DB",), "kind": "int"}, + { + "section": "Java", + "primary": "CHATAPP_JAVA_AUTH_BASE_URL", + "aliases": ("INVITE_JAVA_AUTH_BASE_URL",), + "kind": "string", + }, + { + "section": "Java", + "primary": "CHATAPP_JAVA_CONSOLE_BASE_URL", + "aliases": ("JAVA_CONSOLE_BASE_URL",), + "kind": "string", + }, + { + "section": "Java", + "primary": "CHATAPP_JAVA_DEVICE_BASE_URL", + "aliases": ("INVITE_JAVA_DEVICE_BASE_URL",), + "kind": "string", + }, + { + "section": "Java", + "primary": "CHATAPP_JAVA_APP_BASE_URL", + "aliases": ("INVITE_JAVA_APP_BASE_URL",), + "kind": "string", + }, + { + "section": "Java", + "primary": "CHATAPP_JAVA_BRIDGE_BASE_URL", + "aliases": ("INVITE_JAVA_BRIDGE_BASE_URL",), + "kind": "string", + }, + { + "section": "Java", + "primary": "CHATAPP_JAVA_EXTERNAL_BASE_URL", + "aliases": ("INVITE_JAVA_EXTERNAL_BASE_URL",), + "kind": "string", + }, + { + "section": "Java", + "primary": "CHATAPP_JAVA_OTHER_BASE_URL", + "aliases": ("GAME_JAVA_OTHER_BASE_URL", "INVITE_JAVA_APP_BASE_URL"), + "kind": "string", + }, + { + "section": "Java", + "primary": "CHATAPP_JAVA_WALLET_BASE_URL", + "aliases": ("GAME_JAVA_WALLET_BASE_URL",), + "kind": "string", + }, + { + "section": "Worker", + "primary": "CHATAPP_REGISTER_REWARD_STREAM_KEY", + "aliases": ("REGISTER_REWARD_STREAM_KEY",), + "kind": "string", + }, + { + "section": "Worker", + "primary": "CHATAPP_REGISTER_REWARD_CONSUMER_GROUP", + "aliases": ("REGISTER_REWARD_CONSUMER_GROUP",), + "kind": "string", + }, + { + "section": "Worker", + "primary": "CHATAPP_REGISTER_REWARD_CONSUMER_NAME", + "aliases": ("REGISTER_REWARD_CONSUMER_NAME",), + "kind": "string", + }, + { + "section": "Worker", + "primary": "CHATAPP_REGISTER_REWARD_BATCH_SIZE", + "aliases": ("REGISTER_REWARD_BATCH_SIZE",), + "kind": "int", + }, + { + "section": "Worker", + "primary": "CHATAPP_REGISTER_REWARD_BLOCK_MILLIS", + "aliases": ("REGISTER_REWARD_BLOCK_MILLIS",), + "kind": "int", + }, + { + "section": "Worker", + "primary": "CHATAPP_REGISTER_REWARD_MIN_IDLE_MILLIS", + "aliases": ("REGISTER_REWARD_MIN_IDLE_MILLIS",), + "kind": "int", + }, + { + "section": "Invite", + "primary": "CHATAPP_INVITE_PUBLIC_BASE_URL", + "aliases": ("INVITE_PUBLIC_BASE_URL",), + "kind": "string", + }, + { + "section": "WeekStar", + "primary": "CHATAPP_WEEK_STAR_DEFAULT_SYS_ORIGIN", + "aliases": ("WEEK_STAR_DEFAULT_SYS_ORIGIN",), + "kind": "string", + }, + { + "section": "WeekStar", + "primary": "CHATAPP_WEEK_STAR_TIMEZONE", + "aliases": ("WEEK_STAR_TIMEZONE",), + "kind": "string", + }, + { + "section": "WeekStar", + "primary": "CHATAPP_WEEK_STAR_RETRY_STREAM_KEY", + "aliases": ("WEEK_STAR_RETRY_STREAM_KEY",), + "kind": "string", + }, + { + "section": "WeekStar", + "primary": "CHATAPP_WEEK_STAR_ROCKETMQ_ENABLED", + "aliases": ("WEEK_STAR_ROCKETMQ_ENABLED",), + "kind": "bool", + }, + { + "section": "WeekStar", + "primary": "CHATAPP_WEEK_STAR_ROCKETMQ_ENDPOINT", + "aliases": ("WEEK_STAR_ROCKETMQ_ENDPOINT",), + "kind": "string", + }, + { + "section": "WeekStar", + "primary": "CHATAPP_WEEK_STAR_ROCKETMQ_NAMESPACE", + "aliases": ("WEEK_STAR_ROCKETMQ_NAMESPACE",), + "kind": "string", + }, + { + "section": "WeekStar", + "primary": "CHATAPP_WEEK_STAR_ROCKETMQ_ACCESS_KEY", + "aliases": ("WEEK_STAR_ROCKETMQ_ACCESS_KEY",), + "kind": "string", + }, + { + "section": "WeekStar", + "primary": "CHATAPP_WEEK_STAR_ROCKETMQ_ACCESS_SECRET", + "aliases": ("WEEK_STAR_ROCKETMQ_ACCESS_SECRET",), + "kind": "string", + }, + { + "section": "WeekStar", + "primary": "CHATAPP_WEEK_STAR_ROCKETMQ_SECURITY_TOKEN", + "aliases": ("WEEK_STAR_ROCKETMQ_SECURITY_TOKEN",), + "kind": "string", + }, + { + "section": "WeekStar", + "primary": "CHATAPP_WEEK_STAR_ROCKETMQ_CONSUMER_GROUP", + "aliases": ("WEEK_STAR_ROCKETMQ_CONSUMER_GROUP",), + "kind": "string", + }, + { + "section": "WeekStar", + "primary": "CHATAPP_WEEK_STAR_ROCKETMQ_TOPIC", + "aliases": ("WEEK_STAR_ROCKETMQ_TOPIC",), + "kind": "string", + }, + { + "section": "WeekStar", + "primary": "CHATAPP_WEEK_STAR_ROCKETMQ_TAG", + "aliases": ("WEEK_STAR_ROCKETMQ_TAG",), + "kind": "string", + }, + { + "section": "Baishun", + "primary": "CHATAPP_BAISHUN_PLATFORM_BASE_URL", + "aliases": ("BAISHUN_PLATFORM_BASE_URL",), + "kind": "string", + }, + {"section": "Baishun", "primary": "CHATAPP_BAISHUN_APP_ID", "aliases": ("BAISHUN_APP_ID",), "kind": "int"}, + { + "section": "Baishun", + "primary": "CHATAPP_BAISHUN_APP_NAME", + "aliases": ("BAISHUN_APP_NAME",), + "kind": "string", + }, + { + "section": "Baishun", + "primary": "CHATAPP_BAISHUN_APP_CHANNEL", + "aliases": ("BAISHUN_APP_CHANNEL",), + "kind": "string", + }, + {"section": "Baishun", "primary": "CHATAPP_BAISHUN_APP_KEY", "aliases": ("BAISHUN_APP_KEY",), "kind": "string"}, + {"section": "Baishun", "primary": "CHATAPP_BAISHUN_GSP", "aliases": ("BAISHUN_GSP",), "kind": "int"}, + { + "section": "Baishun", + "primary": "CHATAPP_BAISHUN_LAUNCH_CODE_TTL_SECONDS", + "aliases": ("BAISHUN_LAUNCH_CODE_TTL_SECONDS",), + "kind": "int", + }, + { + "section": "Baishun", + "primary": "CHATAPP_BAISHUN_SS_TOKEN_TTL_SECONDS", + "aliases": ("BAISHUN_SS_TOKEN_TTL_SECONDS",), + "kind": "int", + }, + { + "section": "LuckyGift", + "primary": "CHATAPP_LUCKY_GIFT_ENABLED", + "aliases": ("LUCKY_GIFT_ENABLED", "LUCKY_GIFT_GO_ENABLED"), + "kind": "bool", + }, + { + "section": "LuckyGift", + "primary": "CHATAPP_LUCKY_GIFT_TIMEOUT_MILLIS", + "aliases": ("LUCKY_GIFT_TIMEOUT_MILLIS", "LUCKY_GIFT_GO_TIMEOUT_MILLIS"), + "kind": "int", + }, + { + "section": "LuckyGift", + "primary": "CHATAPP_LUCKY_GIFT_REWARD_STREAM_KEY", + "aliases": ("LUCKY_GIFT_REWARD_STREAM_KEY",), + "kind": "string", + }, + { + "section": "LuckyGift", + "primary": "CHATAPP_LUCKY_GIFT_CONFIGS_JSON", + "aliases": ("LUCKY_GIFT_CONFIGS_JSON", "LUCKY_GIFT_CONFIGS"), + "kind": "json", + }, + { + "section": "LuckyGift", + "primary": "CHATAPP_LUCKY_GIFT_STANDARD_ID", + "aliases": ("LUCKY_GIFT_STANDARD_ID",), + "kind": "int", + }, + { + "section": "LuckyGift", + "primary": "CHATAPP_LUCKY_GIFT_URL", + "aliases": ("LUCKY_GIFT_URL", "LUCKY_GIFT_API_DEFAULT_URL"), + "kind": "string", + }, + { + "section": "LuckyGift", + "primary": "CHATAPP_LUCKY_GIFT_APP_ID", + "aliases": ("LUCKY_GIFT_APP_ID", "LUCKY_GIFT_API_DEFAULT_APP_ID"), + "kind": "string", + }, + { + "section": "LuckyGift", + "primary": "CHATAPP_LUCKY_GIFT_APP_CHANNEL", + "aliases": ("LUCKY_GIFT_APP_CHANNEL", "LUCKY_GIFT_API_DEFAULT_APP_CHANNEL"), + "kind": "string", + }, + { + "section": "LuckyGift", + "primary": "CHATAPP_LUCKY_GIFT_APP_KEY", + "aliases": ("LUCKY_GIFT_APP_KEY", "LUCKY_GIFT_API_DEFAULT_APP_KEY"), + "kind": "string", + }, +] + +# 把所有可接受的 key 映射回唯一主键，方便校验和保存时去重。 +GO_ENV_PRIMARY_BY_KEY = { + key: spec["primary"] + for spec in GO_ENV_SPECS + for key in (spec["primary"], *spec["aliases"]) +} +# 同时保留按主键索引的 spec，后续查类型更直接。 +GO_ENV_SPECS_BY_PRIMARY = {spec["primary"]: spec for spec in GO_ENV_SPECS} + +# 远端文件不存在时，用固定标记避免和 shell 错误混淆。 +REMOTE_FILE_MISSING_MARKER = "__HY_APP_MONITOR_FILE_MISSING__" + + +def go_service_hosts() -> list[dict[str, Any]]: + # 只挑出当前监控目标里真正承载 golang 服务的主机。 + hosts = [ + host + for host in load_hosts() + if any(str(service.get("name") or "").strip() == "golang" for service in host.get("services") or []) + ] + if not hosts: + raise RuntimeError("no golang hosts found in config/targets.json") + return hosts + + +def service_env_path(host_name: str) -> str: + # 线上运行目录沿用现有部署结构。 + return f"{RUNTIME_BASE_DIR.rstrip('/')}/{host_name}/service.env" + + +def parse_env_assignment(raw_line: str) -> tuple[str, str] | None: + # 保留原始内容用于判断注释和空行。 + stripped = raw_line.strip() + if not stripped or stripped.startswith("#"): + return None + if stripped.startswith("export "): + stripped = stripped[7:].strip() + if "=" not in stripped: + return None + + # 只按第一个等号切分，避免 URL / JSON 被截断。 + key, value = stripped.split("=", 1) + key = key.strip() + value = value.strip() + if not key: + return None + return key, decode_env_value(value) + + +def decode_env_value(raw_value: str) -> str: + # 双引号优先按 JSON 字符串解码，兼容反斜杠和 Unicode。 + if len(raw_value) >= 2 and raw_value[0] == raw_value[-1] == '"': + try: + return str(json.loads(raw_value)) + except Exception: # noqa: BLE001 + return raw_value[1:-1] + + # 单引号值按最简单规则去壳。 + if len(raw_value) >= 2 and raw_value[0] == raw_value[-1] == "'": + return raw_value[1:-1] + + # 其余值原样返回。 + return raw_value + + +def render_env_value(value: str) -> str: + # 统一用 JSON 风格双引号输出，避免空格和特殊字符破坏 env 语法。 + return json.dumps(value, ensure_ascii=False) + + +def parse_env_text(raw: str) -> dict[str, str]: + # env 文件最终按 key/value 平铺。 + values: dict[str, str] = {} + for raw_line in raw.splitlines(): + parsed = parse_env_assignment(raw_line) + if parsed is None: + continue + key, value = parsed + values[key] = value + return values + + +def remote_host_result(host: dict[str, Any], script: str, command_name: str, *, timeout_seconds: int = TAT_METRIC_TIMEOUT_SECONDS) -> dict[str, Any]: + # 统一执行单机 TAT 命令，并把错误折叠成 RuntimeError。 + result_map = run_shell_command( + [host["instanceId"]], + script, + command_name=command_name, + timeout_seconds=timeout_seconds, + ) + result = result_map[host["instanceId"]] + if result.get("status") != "SUCCESS": + raise RuntimeError(f"{host['host']} {command_name} failed: {result.get('status')}") + return result + + +def read_remote_text(host: dict[str, Any], path: str) -> str: + quoted_path = shlex.quote(path) + script = f"""set -eu +FILE={quoted_path} +if [ ! -f "$FILE" ]; then + printf '%s' {shlex.quote(REMOTE_FILE_MISSING_MARKER)} + exit 0 +fi +cat "$FILE" +""" + result = remote_host_result(host, script, f"read-{host['host']}-service-env") + output = str(result.get("output") or "") + if output.strip() == REMOTE_FILE_MISSING_MARKER: + raise RuntimeError(f"{host['host']} missing file: {path}") + return output + + +def write_remote_text(host: dict[str, Any], path: str, content: str) -> None: + # 把文本整体 base64 编码后再下发，避免 shell here-doc 被内容污染。 + encoded = base64.b64encode(content.encode("utf-8")).decode("ascii") + quoted_path = shlex.quote(path) + script = f"""set -eu +FILE={quoted_path} +TMP="${{FILE}}.hyapp.$$" +mkdir -p "$(dirname "$FILE")" +printf '%s' {shlex.quote(encoded)} | base64 -d > "$TMP" +mv "$TMP" "$FILE" +""" + remote_host_result(host, script, f"write-{host['host']}-service-env") + + +def host_sha256(content: str) -> str: + # 前端需要一个稳定摘要判断读写前后是否一致。 + return hashlib.sha256(content.encode("utf-8")).hexdigest() + + +def first_present_value(values: dict[str, str], keys: tuple[str, ...]) -> str: + # 配置读取语义和 getEnvAny 对齐，优先取第一个非空值。 + for key in keys: + candidate = values.get(key, "") + if candidate != "": + return candidate + + # 如果全是空串，但确实存在某个 key，也保留空串语义。 + for key in keys: + if key in values: + return values[key] + return "" + + +def extract_go_values(values: dict[str, str]) -> dict[str, str]: + # 统一抽成“主键 -> 当前值”，方便页面和保存逻辑复用。 + extracted: dict[str, str] = {} + for spec in GO_ENV_SPECS: + keys = (spec["primary"], *spec["aliases"]) + extracted[spec["primary"]] = first_present_value(values, keys) + return extracted + + +def render_go_editor_content(values: dict[str, str]) -> str: + # 生成带分段注释的可编辑 env 文本，用户一眼能看出结构。 + lines: list[str] = [] + current_section = "" + for spec in GO_ENV_SPECS: + if spec["section"] != current_section: + if lines: + lines.append("") + current_section = spec["section"] + lines.append(f"# {current_section}") + lines.append(f"{spec['primary']}={render_env_value(values.get(spec['primary'], ''))}") + return "\n".join(lines).rstrip() + "\n" + + +def validate_scalar_kind(primary_key: str, value: str, kind: str) -> None: + # 空串允许保存，运行时会按 fallback / 默认值处理。 + if value == "": + return + if kind == "string": + return + if kind == "int": + int(value) + return + if kind == "bool": + if value.strip().lower() not in {"1", "0", "true", "false", "yes", "no", "on", "off"}: + raise ValueError(f"{primary_key} must be a boolean") + return + if kind == "json": + json.loads(value) + return + + +def validate_go_editor_content(content: str) -> dict[str, Any]: + # 这里按“可编辑文本”逐行解析，保留重复和未知 key 的校验能力。 + values: dict[str, str] = {} + seen_primaries: set[str] = set() + + for line_no, raw_line in enumerate(content.splitlines(), start=1): + parsed = parse_env_assignment(raw_line) + if parsed is None: + continue + + key, value = parsed + primary_key = GO_ENV_PRIMARY_BY_KEY.get(key) + if not primary_key: + return { + "ok": False, + "strict": True, + "type": "env", + "message": f"unsupported golang env key: {key}", + "details": {"line": line_no, "key": key}, + } + + if primary_key in seen_primaries: + return { + "ok": False, + "strict": True, + "type": "env", + "message": f"duplicated golang env key: {primary_key}", + "details": {"line": line_no, "key": primary_key}, + } + + spec = GO_ENV_SPECS_BY_PRIMARY[primary_key] + try: + validate_scalar_kind(primary_key, value, spec["kind"]) + except Exception as exc: # noqa: BLE001 + return { + "ok": False, + "strict": True, + "type": "env", + "message": str(exc), + "details": {"line": line_no, "key": primary_key}, + } + + seen_primaries.add(primary_key) + values[primary_key] = value + + return { + "ok": True, + "strict": True, + "type": "env", + "message": f"golang env format ok ({len(values)} keys)", + "details": {"keysCount": len(values)}, + "values": values, + } + + +def expand_host_updates(current_values: dict[str, str], editor_values: dict[str, str]) -> dict[str, str]: + # 保存时尽量沿用宿主机现有 key 形态，只在缺失时补主键。 + updates: dict[str, str] = {} + for primary_key, value in editor_values.items(): + spec = GO_ENV_SPECS_BY_PRIMARY[primary_key] + host_keys = [key for key in (primary_key, *spec["aliases"]) if key in current_values] + if not host_keys: + host_keys = [primary_key] + for key in host_keys: + updates[key] = value + return updates + + +def merge_env_text(raw_content: str, updates: dict[str, str]) -> str: + # 在原文件基础上按行覆盖，尽量保留注释和无关配置。 + output_lines: list[str] = [] + seen_keys: set[str] = set() + + for raw_line in raw_content.splitlines(): + parsed = parse_env_assignment(raw_line) + if parsed is None: + output_lines.append(raw_line) + continue + + key, _ = parsed + if key not in updates: + output_lines.append(raw_line) + continue + + output_lines.append(f"{key}={render_env_value(updates[key])}") + seen_keys.add(key) + + # 原文件没有的 key 统一追加到末尾。 + missing_keys = [key for key in updates if key not in seen_keys] + if missing_keys: + if output_lines and output_lines[-1].strip(): + output_lines.append("") + output_lines.append("# hy-app-monitor golang config") + for key in missing_keys: + output_lines.append(f"{key}={render_env_value(updates[key])}") + + return "\n".join(output_lines).rstrip() + "\n" + + +def go_host_snapshots() -> list[dict[str, Any]]: + # 逐台读取实际运行中的 service.env。 + snapshots: list[dict[str, Any]] = [] + for host in go_service_hosts(): + env_path = service_env_path(host["host"]) + raw_content = read_remote_text(host, env_path) + env_values = parse_env_text(raw_content) + go_values = extract_go_values(env_values) + snapshots.append( + { + "host": host["host"], + "ip": host["ip"], + "instanceId": host["instanceId"], + "path": env_path, + "sha256": host_sha256(raw_content), + "lineCount": len(raw_content.splitlines()) if raw_content else 0, + "content": raw_content, + "values": env_values, + "goValues": go_values, + } + ) + return snapshots + + +def go_diff_keys(snapshots: list[dict[str, Any]]) -> list[str]: + # 同步看两台 app 机器上的 Go 配置是否已经一致。 + if not snapshots: + return [] + baseline = snapshots[0]["goValues"] + diff: list[str] = [] + for key in baseline: + if any(snapshot["goValues"].get(key, "") != baseline.get(key, "") for snapshot in snapshots[1:]): + diff.append(key) + return diff + + +def get_go_config_payload() -> dict[str, Any]: + # 读取所有 golang 宿主机当前运行配置。 + snapshots = go_host_snapshots() + diff_keys = go_diff_keys(snapshots) + baseline_values = snapshots[0]["goValues"] if snapshots else {} + + return { + "ok": True, + "updatedAt": utc_now(), + "hosts": [ + { + "host": snapshot["host"], + "ip": snapshot["ip"], + "instanceId": snapshot["instanceId"], + "path": snapshot["path"], + "sha256": snapshot["sha256"], + "lineCount": snapshot["lineCount"], + } + for snapshot in snapshots + ], + "diffKeys": diff_keys, + "editorContent": render_go_editor_content(baseline_values), + "keysCount": len([line for line in render_go_editor_content(baseline_values).splitlines() if "=" in line]), + } + + +def validate_go_config_payload(content: str) -> dict[str, Any]: + # 对页面编辑器内容做显式格式校验。 + result = validate_go_editor_content(content) + if result.get("ok"): + result["updatedAt"] = utc_now() + return result + + +def save_go_config_payload(content: str) -> dict[str, Any]: + # 保存前先做一次强校验，避免把非法配置下发到线上。 + validation = validate_go_editor_content(content) + if not validation.get("ok"): + raise RuntimeError(str(validation.get("message") or "invalid golang env content")) + + editor_values = dict(validation.get("values") or {}) + if not editor_values: + raise RuntimeError("no golang env keys found in editor content") + + saved_hosts: list[dict[str, Any]] = [] + for snapshot in go_host_snapshots(): + host_updates = expand_host_updates(snapshot["values"], editor_values) + merged_content = merge_env_text(snapshot["content"], host_updates) + write_remote_text( + {"host": snapshot["host"], "instanceId": snapshot["instanceId"]}, + snapshot["path"], + merged_content, + ) + + # 写完立即回读，确认关键值确实落盘。 + verified_content = read_remote_text( + {"host": snapshot["host"], "instanceId": snapshot["instanceId"]}, + snapshot["path"], + ) + verified_values = extract_go_values(parse_env_text(verified_content)) + for primary_key, expected_value in editor_values.items(): + if verified_values.get(primary_key, "") != expected_value: + raise RuntimeError(f"verify golang config failed on {snapshot['host']}: {primary_key}") + + saved_hosts.append( + { + "host": snapshot["host"], + "ip": snapshot["ip"], + "instanceId": snapshot["instanceId"], + "path": snapshot["path"], + "sha256": host_sha256(verified_content), + } + ) + + payload = get_go_config_payload() + payload["savedHosts"] = saved_hosts + payload["message"] = "golang service.env saved" + return payload diff --git a/monitor/service_ops.py b/monitor/service_ops.py new file mode 100644 index 0000000..a861bd8 --- /dev/null +++ b/monitor/service_ops.py @@ -0,0 +1,255 @@ +from __future__ import annotations + +# 远端命令里会拼接 shell 参数，需要显式转义。 +import shlex +# 滚动重启是串行动作，轮询恢复要用时间模块。 +import time +from typing import Any + +from .config import ( + ROLLING_RESTART_POLL_SECONDS, + ROLLING_RESTART_STABILIZE_SECONDS, + ROLLING_RESTART_TIMEOUT_SECONDS, + RUNTIME_BASE_DIR, + THRESHOLDS, + flatten_services, + load_hosts, + utc_now, +) +from .http_probe import SERVICE_CACHE, probe_service +from .nacos import ( + NACOS_CACHE, + NACOS_DISCOVERY_GROUP, + NACOS_SERVICE_NAME_MAP, + catalog_instances, + current_namespace, + list_instances, + nacos_truthy, +) +from .tencent_tat import run_shell_command + + +def restartable_service_records() -> list[dict[str, Any]]: + # 当前只开放 Java / Golang 业务服务滚动重启。 + services = flatten_services(load_hosts()) + return [service for service in services if service.get("kind") in {"java", "golang"}] + + +def build_restart_targets_payload() -> dict[str, Any]: + # 页面需要按“服务 -> 宿主机”展示选择项。 + grouped: dict[str, dict[str, Any]] = {} + for service in restartable_service_records(): + item = grouped.setdefault( + service["service"], + { + "service": service["service"], + "kind": service["kind"], + "hosts": [], + }, + ) + item["hosts"].append( + { + "host": service["host"], + "ip": service["ip"], + "instanceId": service["instanceId"], + "port": service["port"], + "path": service["path"], + } + ) + + return { + "ok": True, + "updatedAt": utc_now(), + "items": list(grouped.values()), + } + + +def service_records_by_name() -> dict[str, list[dict[str, Any]]]: + # 重启顺序沿用 targets.json 的主机顺序。 + grouped: dict[str, list[dict[str, Any]]] = {} + for service in restartable_service_records(): + grouped.setdefault(service["service"], []).append(service) + return grouped + + +def remote_command_result(service: dict[str, Any], script: str, command_name: str, timeout_seconds: int) -> dict[str, Any]: + # 单实例远端执行统一复用 TAT shell。 + result_map = run_shell_command( + [service["instanceId"]], + script, + command_name=command_name, + timeout_seconds=timeout_seconds, + ) + return result_map[service["instanceId"]] + + +def restart_service_instance(service: dict[str, Any]) -> dict[str, Any]: + # 线上业务目录和 compose 文件复用现网部署结构。 + host_dir = f"{RUNTIME_BASE_DIR.rstrip('/')}/{service['host']}" + container_name = f"likei-{service['host']}-{service['service']}" + script = f"""set -eu +HOST_DIR={shlex.quote(host_dir)} +cd "$HOST_DIR" +docker compose up -d --force-recreate --no-deps {shlex.quote(service['service'])} +docker ps --filter name={shlex.quote(container_name)} --format '{{{{.Names}}}} {{{{.Status}}}}' +""" + result = remote_command_result( + service, + script, + f"restart-{service['host']}-{service['service']}", + timeout_seconds=max(ROLLING_RESTART_TIMEOUT_SECONDS, 60), + ) + if result.get("status") != "SUCCESS": + raise RuntimeError(f"restart failed on {service['host']}: {result.get('status')}") + return result + + +def wait_http_ready(service: dict[str, Any], timeout_seconds: int) -> dict[str, Any]: + # 直接复用现有健康探测逻辑，不走缓存，确保拿的是最新状态。 + deadline = time.time() + timeout_seconds + last_result: dict[str, Any] | None = None + + while time.time() < deadline: + last_result = probe_service(service, THRESHOLDS["latencyMs"]) + if last_result.get("ok"): + return last_result + time.sleep(ROLLING_RESTART_POLL_SECONDS) + + detail = (last_result or {}).get("detail") or "service did not recover" + raise RuntimeError(f"http health not ready on {service['host']} {service['service']}: {detail}") + + +def matched_nacos_record(items: list[dict[str, Any]], service: dict[str, Any]) -> dict[str, Any] | None: + # 只认当前宿主机对应的实例，避免误把另一台机器当成恢复成功。 + for item in items: + ip = str(item.get("ip") or "").strip() + port = int(item.get("port") or 0) + if ip == service["ip"] and port == int(service["port"]): + return item + return None + + +def wait_java_registration_ready(service: dict[str, Any], timeout_seconds: int) -> dict[str, Any]: + # 只有走 Nacos 注册的 Java 服务才做注册恢复校验。 + registry_name = NACOS_SERVICE_NAME_MAP.get(service["service"]) + if not registry_name: + return {"checked": False, "reason": "service not managed by nacos registry map"} + + namespace_id, _ = current_namespace() + deadline = time.time() + timeout_seconds + last_record: dict[str, Any] | None = None + + while time.time() < deadline: + try: + last_record = matched_nacos_record( + list_instances(namespace_id, registry_name, NACOS_DISCOVERY_GROUP), + service, + ) + except Exception: # noqa: BLE001 + last_record = None + + if last_record and nacos_truthy(last_record.get("healthy", False)) and nacos_truthy(last_record.get("enabled", True)): + return { + "checked": True, + "healthy": True, + "enabled": True, + "source": "instance/list", + } + + try: + catalog_record = matched_nacos_record( + catalog_instances(namespace_id, registry_name, NACOS_DISCOVERY_GROUP), + service, + ) + except Exception: # noqa: BLE001 + catalog_record = None + + if catalog_record and nacos_truthy(catalog_record.get("healthy", False)) and nacos_truthy( + catalog_record.get("enabled", True) + ): + return { + "checked": True, + "healthy": True, + "enabled": True, + "source": "catalog/instances", + } + + time.sleep(ROLLING_RESTART_POLL_SECONDS) + + raise RuntimeError(f"nacos registration not ready on {service['host']} {service['service']}") + + +def clear_operation_caches() -> None: + # 重启会直接影响服务探测和 Nacos 面板，完成后把缓存清掉。 + SERVICE_CACHE.clear() + NACOS_CACHE.clear() + + +def rolling_restart_payload(service_names: list[str]) -> dict[str, Any]: + # 用户可能重复点击同一个服务，这里先去重并保序。 + normalized_names = [str(name or "").strip() for name in service_names if str(name or "").strip()] + normalized_names = list(dict.fromkeys(normalized_names)) + if not normalized_names: + raise RuntimeError("services are required") + + records_map = service_records_by_name() + steps: list[dict[str, Any]] = [] + + for service_name in normalized_names: + if service_name not in records_map: + raise RuntimeError(f"unsupported restart service: {service_name}") + + for service in records_map[service_name]: + started_at = time.time() + step: dict[str, Any] = { + "service": service_name, + "host": service["host"], + "ip": service["ip"], + "kind": service["kind"], + "status": "running", + } + + try: + restart_result = restart_service_instance(service) + step["tatStatus"] = restart_result.get("status") + step["tatOutput"] = str(restart_result.get("output") or "").strip()[:500] + + http_result = wait_http_ready(service, ROLLING_RESTART_TIMEOUT_SECONDS) + step["http"] = { + "ok": bool(http_result.get("ok")), + "statusCode": int(http_result.get("statusCode") or 0), + "latencyMs": int(http_result.get("latencyMs") or 0), + "detail": str(http_result.get("detail") or ""), + } + + if service["kind"] == "java": + step["nacos"] = wait_java_registration_ready(service, ROLLING_RESTART_TIMEOUT_SECONDS) + + if ROLLING_RESTART_STABILIZE_SECONDS > 0: + time.sleep(ROLLING_RESTART_STABILIZE_SECONDS) + + step["status"] = "success" + step["durationSeconds"] = round(time.time() - started_at, 2) + steps.append(step) + except Exception as exc: # noqa: BLE001 + step["status"] = "failed" + step["error"] = str(exc) + step["durationSeconds"] = round(time.time() - started_at, 2) + steps.append(step) + clear_operation_caches() + return { + "ok": False, + "updatedAt": utc_now(), + "services": normalized_names, + "steps": steps, + "error": step["error"], + } + + clear_operation_caches() + return { + "ok": True, + "updatedAt": utc_now(), + "services": normalized_names, + "steps": steps, + "message": "rolling restart complete", + } diff --git a/requirements-ops.txt b/requirements-ops.txt index 1ec6d22..83f353d 100644 --- a/requirements-ops.txt +++ b/requirements-ops.txt @@ -1,2 +1,4 @@ tencentcloud-sdk-python>=3.0.0 pymongo>=4.8.0 +PyYAML>=6.0.2 +javaproperties>=0.8.2 diff --git a/static/app.css b/static/app.css index 980c1a3..2daa211 100644 --- a/static/app.css +++ b/static/app.css @@ -170,6 +170,15 @@ body { transform: translateY(-1px); } +.refresh.ghost { + background: color-mix(in oklab, var(--panel-strong) 82%, transparent); +} + +.refresh.danger { + color: var(--bad); + border-color: color-mix(in oklab, var(--bad) 55%, var(--line)); +} + .filter-chip.active { border-color: color-mix(in oklab, var(--accent) 65%, white 15%); background: color-mix(in oklab, var(--accent) 18%, var(--panel-strong)); @@ -500,6 +509,24 @@ body { min-width: 320px; } +.host-tags, +.restart-pills, +.action-group { + display: flex; + gap: 10px; + align-items: center; + flex-wrap: wrap; +} + +.restart-pills { + flex: 1 1 auto; +} + +.restart-pills .config-item { + width: auto; + min-width: 220px; +} + .config-layout { display: grid; grid-template-columns: minmax(260px, 340px) minmax(0, 1fr); @@ -670,4 +697,14 @@ body { flex-direction: column; align-items: stretch; } + + .host-tags, + .restart-pills, + .action-group { + width: 100%; + } + + .restart-pills .config-item { + width: 100%; + } } diff --git a/static/app.js b/static/app.js index 12bc268..805022b 100644 --- a/static/app.js +++ b/static/app.js @@ -57,6 +57,8 @@ createApp({ selectedKey: "", detailLoading: false, detailError: "", + validating: false, + validationMessage: "", saving: false, saveMessage: "", loadedContent: "", @@ -71,6 +73,27 @@ createApp({ content: "", }, }, + goConfig: { + loading: false, + error: "", + validating: false, + validationMessage: "", + saving: false, + saveMessage: "", + loadedContent: "", + editorContent: "", + diffKeys: [], + hosts: [], + }, + restartOps: { + loading: false, + error: "", + running: false, + message: "", + items: [], + selectedServices: [], + result: null, + }, timer: null, }; }, @@ -116,8 +139,8 @@ createApp({ const shouldShowBecauseDown = this.filter === "down" && !(host.metrics && host.metrics.ok); const shouldShowBecauseKeyword = keyword ? hostMatched || filteredItems.length > 0 : true; const shouldShow = - shouldShowBecauseKeyword && - (this.filter === "all" || filteredItems.length > 0 || shouldShowBecauseDown || hostMatched); + shouldShowBecauseKeyword + && (this.filter === "all" || filteredItems.length > 0 || shouldShowBecauseDown || hostMatched); return { ...host, @@ -162,12 +185,15 @@ createApp({ .includes(keyword); }); }, - selectedNacosConfig() { - return (this.nacosConfig.items || []).find((item) => this.configKey(item.group, item.dataId) === this.nacosConfig.selectedKey) || null; - }, nacosConfigDirty() { return this.nacosConfig.editor.content !== this.nacosConfig.loadedContent; }, + goConfigDirty() { + return this.goConfig.editorContent !== this.goConfig.loadedContent; + }, + restartTargetMap() { + return Object.fromEntries((this.restartOps.items || []).map((item) => [item.service, item])); + }, }, watch: { refreshIntervalSeconds() { @@ -188,9 +214,6 @@ createApp({ toneClass(level) { return `tone-${level || "neutral"}`; }, - boolTone(ok) { - return ok ? "tone-ok" : "tone-bad"; - }, configKey(group, dataId) { return `${group}@@${dataId}`; }, @@ -258,6 +281,9 @@ createApp({ } return `${remainSeconds}s`; }, + restartTargetLabel(item) { + return `${item.service} · ${item.kind} · ${item.hosts.map((host) => host.host).join(" / ")}`; + }, syncRefreshSettings() { const settings = this.payload.settings || {}; const options = Array.isArray(settings.refreshIntervalOptions) @@ -292,6 +318,7 @@ createApp({ resetNacosEditor() { this.nacosConfig.selectedKey = ""; this.nacosConfig.detailError = ""; + this.nacosConfig.validationMessage = ""; this.nacosConfig.saveMessage = ""; this.nacosConfig.loadedContent = ""; this.nacosConfig.editor = { @@ -316,6 +343,23 @@ createApp({ throw new Error(text); } }, + async refresh() { + this.loading = true; + this.error = ""; + try { + const response = await fetch("/api/monitor/services", { cache: "no-store" }); + const payload = await this.readJsonResponse(response); + if (!response.ok) { + throw new Error(payload.error || `HTTP ${response.status}`); + } + this.payload = payload; + this.syncRefreshSettings(); + } catch (error) { + this.error = error instanceof Error ? error.message : String(error); + } finally { + this.loading = false; + } + }, async refreshNacosConfigs({ preserveSelection = true, reloadSelected = false } = {}) { const previousKey = preserveSelection ? this.nacosConfig.selectedKey : ""; this.nacosConfig.loading = true; @@ -369,6 +413,7 @@ createApp({ this.nacosConfig.detailLoading = true; this.nacosConfig.detailError = ""; + this.nacosConfig.validationMessage = ""; this.nacosConfig.saveMessage = ""; try { @@ -396,6 +441,42 @@ createApp({ this.nacosConfig.detailLoading = false; } }, + async validateNacosConfig() { + if (!this.nacosConfig.editor.dataId) { + return; + } + + this.nacosConfig.validating = true; + this.nacosConfig.detailError = ""; + this.nacosConfig.validationMessage = ""; + + try { + const response = await fetch("/api/nacos/config/validate", { + method: "POST", + headers: { + "Content-Type": "application/json", + }, + body: JSON.stringify({ + group: this.nacosConfig.editor.group, + dataId: this.nacosConfig.editor.dataId, + type: this.nacosConfig.editor.type, + content: this.nacosConfig.editor.content, + }), + }); + const payload = await this.readJsonResponse(response); + if (!response.ok) { + throw new Error(payload.error || `HTTP ${response.status}`); + } + if (!payload.ok) { + throw new Error(payload.message || "config validation failed"); + } + this.nacosConfig.validationMessage = payload.message || "格式校验通过"; + } catch (error) { + this.nacosConfig.detailError = error instanceof Error ? error.message : String(error); + } finally { + this.nacosConfig.validating = false; + } + }, async saveNacosConfig() { if (!this.nacosConfig.editor.group || !this.nacosConfig.editor.dataId) { return; @@ -432,21 +513,141 @@ createApp({ this.nacosConfig.saving = false; } }, - async refresh() { - this.loading = true; - this.error = ""; + async refreshGoConfig() { + this.goConfig.loading = true; + this.goConfig.error = ""; + this.goConfig.saveMessage = ""; + try { - const response = await fetch("/api/monitor/services", { cache: "no-store" }); + const response = await fetch("/api/runtime/golang-config", { cache: "no-store" }); const payload = await this.readJsonResponse(response); if (!response.ok) { throw new Error(payload.error || `HTTP ${response.status}`); } - this.payload = payload; - this.syncRefreshSettings(); + this.goConfig.hosts = Array.isArray(payload.hosts) ? payload.hosts : []; + this.goConfig.diffKeys = Array.isArray(payload.diffKeys) ? payload.diffKeys : []; + this.goConfig.loadedContent = typeof payload.editorContent === "string" ? payload.editorContent : ""; + this.goConfig.editorContent = this.goConfig.loadedContent; } catch (error) { - this.error = error instanceof Error ? error.message : String(error); + this.goConfig.error = error instanceof Error ? error.message : String(error); } finally { - this.loading = false; + this.goConfig.loading = false; + } + }, + async validateGoConfig() { + this.goConfig.validating = true; + this.goConfig.error = ""; + this.goConfig.validationMessage = ""; + + try { + const response = await fetch("/api/runtime/golang-config/validate", { + method: "POST", + headers: { + "Content-Type": "application/json", + }, + body: JSON.stringify({ content: this.goConfig.editorContent }), + }); + const payload = await this.readJsonResponse(response); + if (!response.ok) { + throw new Error(payload.error || `HTTP ${response.status}`); + } + if (!payload.ok) { + throw new Error(payload.message || "golang config validation failed"); + } + this.goConfig.validationMessage = payload.message || "格式校验通过"; + } catch (error) { + this.goConfig.error = error instanceof Error ? error.message : String(error); + } finally { + this.goConfig.validating = false; + } + }, + async saveGoConfig() { + this.goConfig.saving = true; + this.goConfig.error = ""; + this.goConfig.saveMessage = ""; + + try { + const response = await fetch("/api/runtime/golang-config", { + method: "POST", + headers: { + "Content-Type": "application/json", + }, + body: JSON.stringify({ content: this.goConfig.editorContent }), + }); + const payload = await this.readJsonResponse(response); + if (!response.ok) { + throw new Error(payload.error || `HTTP ${response.status}`); + } + this.goConfig.loadedContent = this.goConfig.editorContent; + this.goConfig.saveMessage = payload.message || "golang 配置已保存"; + await this.refreshGoConfig(); + } catch (error) { + this.goConfig.error = error instanceof Error ? error.message : String(error); + } finally { + this.goConfig.saving = false; + } + }, + async refreshRestartTargets() { + this.restartOps.loading = true; + this.restartOps.error = ""; + + try { + const response = await fetch("/api/ops/restart-targets", { cache: "no-store" }); + const payload = await this.readJsonResponse(response); + if (!response.ok) { + throw new Error(payload.error || `HTTP ${response.status}`); + } + this.restartOps.items = Array.isArray(payload.items) ? payload.items : []; + const available = new Set(this.restartOps.items.map((item) => item.service)); + this.restartOps.selectedServices = this.restartOps.selectedServices.filter((service) => available.has(service)); + } catch (error) { + this.restartOps.error = error instanceof Error ? error.message : String(error); + } finally { + this.restartOps.loading = false; + } + }, + toggleRestartService(serviceName) { + const next = new Set(this.restartOps.selectedServices); + if (next.has(serviceName)) { + next.delete(serviceName); + } else { + next.add(serviceName); + } + this.restartOps.selectedServices = [...next]; + }, + async runRollingRestart() { + if (this.restartOps.selectedServices.length === 0) { + return; + } + + this.restartOps.running = true; + this.restartOps.error = ""; + this.restartOps.message = ""; + + try { + const response = await fetch("/api/ops/rolling-restart", { + method: "POST", + headers: { + "Content-Type": "application/json", + }, + body: JSON.stringify({ services: this.restartOps.selectedServices }), + }); + const payload = await this.readJsonResponse(response); + if (!response.ok) { + throw new Error(payload.error || `HTTP ${response.status}`); + } + + this.restartOps.result = payload; + this.restartOps.message = payload.ok + ? (payload.message || "滚动重启完成") + : (payload.error || "滚动重启失败"); + + await this.refresh(); + await this.refreshRestartTargets(); + } catch (error) { + this.restartOps.error = error instanceof Error ? error.message : String(error); + } finally { + this.restartOps.running = false; } }, }, @@ -461,6 +662,8 @@ createApp({ } this.refresh(); this.refreshNacosConfigs({ preserveSelection: false, reloadSelected: true }); + this.refreshGoConfig(); + this.refreshRestartTargets(); }, beforeUnmount() { if (this.timer) { diff --git a/static/index.html b/static/index.html index 14b6a22..0d86ceb 100644 --- a/static/index.html +++ b/static/index.html @@ -339,13 +339,19 @@
{{ nacosConfig.saveMessage }} + {{ nacosConfig.validationMessage }} 配置加载中 有未保存修改 内容已同步 - +
+ + +
@@ -356,6 +362,135 @@ +
+

Golang 配置

+
+
+
+
+

Golang Runtime Env

+

读取 app 节点当前运行中的 service.env，只暴露 chatapp3-golang 实际消费的环境变量

+
+ + {{ goConfig.error ? '存在错误' : goConfig.diffKeys.length > 0 ? '双机有差异' : '已同步' }} + +
+ +
+
+ + {{ host.host }} · {{ host.path }} + +
+ +
+ +

{{ goConfig.error }}

+

+ 当前 app 节点存在差异 key：{{ goConfig.diffKeys.join(', ') }} +

+ +
+ + +
+ {{ goConfig.saveMessage }} + {{ goConfig.validationMessage }} + 有未保存修改 + 内容已同步 + +
+ + +
+
+
+
+ +
+

滚动重启

+
+
+
+
+

Rolling Restart

+

选择要发布生效的 Java / Golang 服务，按宿主机顺序逐台重启并等待健康恢复

+
+ + {{ restartOps.error ? '存在错误' : restartOps.running ? '执行中' : '可操作' }} + +
+ +
+
+ +
+
+ + +
+
+ +

{{ restartOps.error }}

+

{{ restartOps.message }}

+ +
+ + + + + + + + + + + + + + + + + + + + + + + +
服务主机类型状态HTTPNacos耗时
{{ step.service }}{{ step.host }}{{ step.kind }}{{ step.status }} + {{ step.http.statusCode }} / {{ step.http.latencyMs }}ms + - + + {{ step.nacos.source || 'ok' }} + - + {{ step.durationSeconds || '-' }}s
+
+
+

MongoDB 指标