hy-app-monitor/monitors/yumi/host_metrics.py

from __future__ import annotations

# SSH 主机指标改成并发采集，避免逐台串行拖慢首页。
import concurrent.futures
import json
from typing import Any

from core.cache import TTLCache
from core.config import HOST_METRIC_CACHE_TTL_SECONDS, HOST_METRIC_CPU_SAMPLE_SECONDS, HOST_METRIC_TIMEOUT_SECONDS, utc_now
from .ssh_remote import run_host_script


# 主机指标单独缓存，避免页面频繁刷新时不断压业务主机。
HOST_METRIC_CACHE = TTLCache[dict[str, dict[str, Any]]](HOST_METRIC_CACHE_TTL_SECONDS)


def build_metrics_script() -> str:
    # 远端脚本只依赖 Python 标准库，避免每台机器额外装 agent。
    return f"""set -euo pipefail
python3 - <<'PY'
import json
import os
import shutil
import time

# 读取 /proc/stat 的 CPU 统计值。
def read_cpu():
    with open('/proc/stat', 'r', encoding='utf-8') as handle:
        parts = handle.readline().split()[1:8]
    values = [int(item) for item in parts]
    idle = values[3] + values[4]
    total = sum(values)
    return idle, total

# 第一次采样。
idle_1, total_1 = read_cpu()
# 等待一个很短的采样窗口。
time.sleep({HOST_METRIC_CPU_SAMPLE_SECONDS})
# 第二次采样。
idle_2, total_2 = read_cpu()

# 计算 CPU 使用率。
total_diff = max(total_2 - total_1, 1)
idle_diff = max(idle_2 - idle_1, 0)
cpu_percent = round((1 - idle_diff / total_diff) * 100, 2)

# 读取内存信息。
memory_info = {{}}
with open('/proc/meminfo', 'r', encoding='utf-8') as handle:
    for line in handle:
        key, value = line.split(':', 1)
        memory_info[key] = int(value.strip().split()[0])

# 计算内存用量。
memory_total_gb = round(memory_info.get('MemTotal', 0) / 1024 / 1024, 2)
memory_available_gb = round(memory_info.get('MemAvailable', 0) / 1024 / 1024, 2)
memory_used_gb = round(max(memory_total_gb - memory_available_gb, 0), 2)
memory_percent = round((memory_used_gb / memory_total_gb) * 100, 2) if memory_total_gb else 0.0

# 读取根分区磁盘占用。
disk = shutil.disk_usage('/')
disk_total_gb = round(disk.total / 1024 / 1024 / 1024, 2)
disk_used_gb = round(disk.used / 1024 / 1024 / 1024, 2)
disk_percent = round((disk.used / disk.total) * 100, 2) if disk.total else 0.0

# 通过 /proc 估算当前进程数。
process_count = len([name for name in os.listdir('/proc') if name.isdigit()])

# 输出统一 JSON。
print(json.dumps({{
    'ok': True,
    'hostname': os.uname().nodename,
    'cpuPercent': cpu_percent,
    'memoryPercent': memory_percent,
    'memoryUsedGb': memory_used_gb,
    'memoryTotalGb': memory_total_gb,
    'diskPercent': disk_percent,
    'diskUsedGb': disk_used_gb,
    'diskTotalGb': disk_total_gb,
    'processCount': process_count
}}))
PY
"""


def metrics_error_payload(error: str, instance_id: str = "") -> dict[str, Any]:
    # 返回统一的主机指标错误结构。
    return {
        "ok": False,
        "error": error,
        "updatedAt": utc_now(),
        "instanceId": instance_id,
    }


def collect_single_host_metrics(host: dict[str, Any]) -> tuple[str, dict[str, Any]]:
    # SSH 远端执行失败时也按统一结构回传，避免单机问题拖垮整个汇总接口。
    instance_id = str(host.get("instanceId") or "").strip()
    result = run_host_script(
        host,
        build_metrics_script(),
        command_name=f"host-metrics-{host['host']}",
        timeout_seconds=HOST_METRIC_TIMEOUT_SECONDS,
    )
    status = str(result.get("status") or "").strip()
    output = str(result.get("output") or "")

    if status == "SUCCESS":
        try:
            payload = json.loads(output or "{}")
            payload["updatedAt"] = utc_now()
            payload["instanceId"] = instance_id
            return host["host"], payload
        except Exception as exc:  # noqa: BLE001
            return host["host"], metrics_error_payload(f"metrics parse failed: {exc}", instance_id)

    return host["host"], metrics_error_payload(output.strip() or status or "metrics failed", instance_id)


def collect_host_metrics(hosts: list[dict[str, Any]]) -> dict[str, dict[str, Any]]:
    # 没有主机时直接返回空结构。
    if not hosts:
        return {}

    # 主机没有基本 SSH 信息时先记错误，剩余主机继续采集。
    payload_by_host: dict[str, dict[str, Any]] = {}
    available_hosts: list[dict[str, Any]] = []
    for host in hosts:
        if str(host.get("ip") or host.get("sshHost") or "").strip():
            available_hosts.append(host)
            continue
        payload_by_host[host["host"]] = metrics_error_payload(
            "missing ip/sshHost in config/targets.json",
            host.get("instanceId", ""),
        )

    # SSH 无法像 TAT 那样一次多机下发，这里按主机并发执行。
    workers = max(1, min(len(available_hosts) or 1, 8))

    with concurrent.futures.ThreadPoolExecutor(max_workers=workers) as executor:
        future_map = {executor.submit(collect_single_host_metrics, host): host for host in available_hosts}
        for future in concurrent.futures.as_completed(future_map):
            host = future_map[future]
            try:
                host_name, payload = future.result()
            except Exception as exc:  # noqa: BLE001
                payload_by_host[host["host"]] = metrics_error_payload(
                    f"metrics collection failed: {exc}",
                    host.get("instanceId", ""),
                )
                continue
            payload_by_host[host_name] = payload

    # 返回按 host 索引的主机指标。
    return payload_by_host


def get_host_metrics(hosts: list[dict[str, Any]]) -> dict[str, dict[str, Any]]:
    # 定义真正的采集构建函数。
    def builder() -> dict[str, dict[str, Any]]:
        try:
            # 优先返回真实采集结果。
            return collect_host_metrics(hosts)
        except Exception as exc:  # noqa: BLE001
            # 任意异常都降级成错误占位，避免拖垮整个页面。
            return {
                host["host"]: metrics_error_payload(f"metrics collection failed: {exc}", host.get("instanceId", ""))
                for host in hosts
            }

    # 通过缓存保护 SSH 调用。
    return HOST_METRIC_CACHE.get_or_build(builder)