from __future__ import annotations # SSH 主机指标改成并发采集,避免逐台串行拖慢首页。 import concurrent.futures import json from typing import Any from core.cache import TTLCache from core.config import HOST_METRIC_CACHE_TTL_SECONDS, HOST_METRIC_CPU_SAMPLE_SECONDS, HOST_METRIC_TIMEOUT_SECONDS, utc_now from .ssh_remote import run_host_script # 主机指标单独缓存,避免页面频繁刷新时不断压业务主机。 HOST_METRIC_CACHE = TTLCache[dict[str, dict[str, Any]]](HOST_METRIC_CACHE_TTL_SECONDS) def build_metrics_script() -> str: # 远端脚本只依赖 Python 标准库,避免每台机器额外装 agent。 return f"""set -euo pipefail python3 - <<'PY' import json import os import shutil import time # 读取 /proc/stat 的 CPU 统计值。 def read_cpu(): with open('/proc/stat', 'r', encoding='utf-8') as handle: parts = handle.readline().split()[1:8] values = [int(item) for item in parts] idle = values[3] + values[4] total = sum(values) return idle, total # 第一次采样。 idle_1, total_1 = read_cpu() # 等待一个很短的采样窗口。 time.sleep({HOST_METRIC_CPU_SAMPLE_SECONDS}) # 第二次采样。 idle_2, total_2 = read_cpu() # 计算 CPU 使用率。 total_diff = max(total_2 - total_1, 1) idle_diff = max(idle_2 - idle_1, 0) cpu_percent = round((1 - idle_diff / total_diff) * 100, 2) # 读取内存信息。 memory_info = {{}} with open('/proc/meminfo', 'r', encoding='utf-8') as handle: for line in handle: key, value = line.split(':', 1) memory_info[key] = int(value.strip().split()[0]) # 计算内存用量。 memory_total_gb = round(memory_info.get('MemTotal', 0) / 1024 / 1024, 2) memory_available_gb = round(memory_info.get('MemAvailable', 0) / 1024 / 1024, 2) memory_used_gb = round(max(memory_total_gb - memory_available_gb, 0), 2) memory_percent = round((memory_used_gb / memory_total_gb) * 100, 2) if memory_total_gb else 0.0 # 读取根分区磁盘占用。 disk = shutil.disk_usage('/') disk_total_gb = round(disk.total / 1024 / 1024 / 1024, 2) disk_used_gb = round(disk.used / 1024 / 1024 / 1024, 2) disk_percent = round((disk.used / disk.total) * 100, 2) if disk.total else 0.0 # 通过 /proc 估算当前进程数。 process_count = len([name for name in os.listdir('/proc') if name.isdigit()]) # 输出统一 JSON。 print(json.dumps({{ 'ok': True, 'hostname': os.uname().nodename, 'cpuPercent': cpu_percent, 'memoryPercent': memory_percent, 'memoryUsedGb': memory_used_gb, 'memoryTotalGb': memory_total_gb, 'diskPercent': disk_percent, 'diskUsedGb': disk_used_gb, 'diskTotalGb': disk_total_gb, 'processCount': process_count }})) PY """ def metrics_error_payload(error: str, instance_id: str = "") -> dict[str, Any]: # 返回统一的主机指标错误结构。 return { "ok": False, "error": error, "updatedAt": utc_now(), "instanceId": instance_id, } def collect_single_host_metrics(host: dict[str, Any]) -> tuple[str, dict[str, Any]]: # SSH 远端执行失败时也按统一结构回传,避免单机问题拖垮整个汇总接口。 instance_id = str(host.get("instanceId") or "").strip() result = run_host_script( host, build_metrics_script(), command_name=f"host-metrics-{host['host']}", timeout_seconds=HOST_METRIC_TIMEOUT_SECONDS, ) status = str(result.get("status") or "").strip() output = str(result.get("output") or "") if status == "SUCCESS": try: payload = json.loads(output or "{}") payload["updatedAt"] = utc_now() payload["instanceId"] = instance_id return host["host"], payload except Exception as exc: # noqa: BLE001 return host["host"], metrics_error_payload(f"metrics parse failed: {exc}", instance_id) return host["host"], metrics_error_payload(output.strip() or status or "metrics failed", instance_id) def collect_host_metrics(hosts: list[dict[str, Any]]) -> dict[str, dict[str, Any]]: # 没有主机时直接返回空结构。 if not hosts: return {} # 主机没有基本 SSH 信息时先记错误,剩余主机继续采集。 payload_by_host: dict[str, dict[str, Any]] = {} available_hosts: list[dict[str, Any]] = [] for host in hosts: if str(host.get("ip") or host.get("sshHost") or "").strip(): available_hosts.append(host) continue payload_by_host[host["host"]] = metrics_error_payload( "missing ip/sshHost in config/targets.json", host.get("instanceId", ""), ) # SSH 无法像 TAT 那样一次多机下发,这里按主机并发执行。 workers = max(1, min(len(available_hosts) or 1, 8)) with concurrent.futures.ThreadPoolExecutor(max_workers=workers) as executor: future_map = {executor.submit(collect_single_host_metrics, host): host for host in available_hosts} for future in concurrent.futures.as_completed(future_map): host = future_map[future] try: host_name, payload = future.result() except Exception as exc: # noqa: BLE001 payload_by_host[host["host"]] = metrics_error_payload( f"metrics collection failed: {exc}", host.get("instanceId", ""), ) continue payload_by_host[host_name] = payload # 返回按 host 索引的主机指标。 return payload_by_host def get_host_metrics(hosts: list[dict[str, Any]]) -> dict[str, dict[str, Any]]: # 定义真正的采集构建函数。 def builder() -> dict[str, dict[str, Any]]: try: # 优先返回真实采集结果。 return collect_host_metrics(hosts) except Exception as exc: # noqa: BLE001 # 任意异常都降级成错误占位,避免拖垮整个页面。 return { host["host"]: metrics_error_payload(f"metrics collection failed: {exc}", host.get("instanceId", "")) for host in hosts } # 通过缓存保护 SSH 调用。 return HOST_METRIC_CACHE.get_or_build(builder)