165 lines
5.6 KiB
Python
165 lines
5.6 KiB
Python
from __future__ import annotations
|
|
|
|
import json
|
|
from typing import Any
|
|
|
|
from .cache import TTLCache
|
|
from .config import HOST_METRIC_CACHE_TTL_SECONDS, TAT_CPU_SAMPLE_SECONDS, TAT_METRIC_TIMEOUT_SECONDS, utc_now
|
|
from .tencent_tat import run_shell_command
|
|
|
|
|
|
# 主机指标单独缓存，避免页面频繁刷新时不断调用 TAT。
|
|
HOST_METRIC_CACHE = TTLCache[dict[str, dict[str, Any]]](HOST_METRIC_CACHE_TTL_SECONDS)
|
|
|
|
|
|
def build_metrics_script() -> str:
|
|
# 远端脚本只依赖 Python 标准库，避免每台机器额外装 agent。
|
|
return f"""set -euo pipefail
|
|
python3 - <<'PY'
|
|
import json
|
|
import os
|
|
import shutil
|
|
import time
|
|
|
|
# 读取 /proc/stat 的 CPU 统计值。
|
|
def read_cpu():
|
|
with open('/proc/stat', 'r', encoding='utf-8') as handle:
|
|
parts = handle.readline().split()[1:8]
|
|
values = [int(item) for item in parts]
|
|
idle = values[3] + values[4]
|
|
total = sum(values)
|
|
return idle, total
|
|
|
|
# 第一次采样。
|
|
idle_1, total_1 = read_cpu()
|
|
# 等待一个很短的采样窗口。
|
|
time.sleep({TAT_CPU_SAMPLE_SECONDS})
|
|
# 第二次采样。
|
|
idle_2, total_2 = read_cpu()
|
|
|
|
# 计算 CPU 使用率。
|
|
total_diff = max(total_2 - total_1, 1)
|
|
idle_diff = max(idle_2 - idle_1, 0)
|
|
cpu_percent = round((1 - idle_diff / total_diff) * 100, 2)
|
|
|
|
# 读取内存信息。
|
|
memory_info = {{}}
|
|
with open('/proc/meminfo', 'r', encoding='utf-8') as handle:
|
|
for line in handle:
|
|
key, value = line.split(':', 1)
|
|
memory_info[key] = int(value.strip().split()[0])
|
|
|
|
# 计算内存用量。
|
|
memory_total_gb = round(memory_info.get('MemTotal', 0) / 1024 / 1024, 2)
|
|
memory_available_gb = round(memory_info.get('MemAvailable', 0) / 1024 / 1024, 2)
|
|
memory_used_gb = round(max(memory_total_gb - memory_available_gb, 0), 2)
|
|
memory_percent = round((memory_used_gb / memory_total_gb) * 100, 2) if memory_total_gb else 0.0
|
|
|
|
# 读取根分区磁盘占用。
|
|
disk = shutil.disk_usage('/')
|
|
disk_total_gb = round(disk.total / 1024 / 1024 / 1024, 2)
|
|
disk_used_gb = round(disk.used / 1024 / 1024 / 1024, 2)
|
|
disk_percent = round((disk.used / disk.total) * 100, 2) if disk.total else 0.0
|
|
|
|
# 通过 /proc 估算当前进程数。
|
|
process_count = len([name for name in os.listdir('/proc') if name.isdigit()])
|
|
|
|
# 输出统一 JSON。
|
|
print(json.dumps({{
|
|
'ok': True,
|
|
'hostname': os.uname().nodename,
|
|
'cpuPercent': cpu_percent,
|
|
'memoryPercent': memory_percent,
|
|
'memoryUsedGb': memory_used_gb,
|
|
'memoryTotalGb': memory_total_gb,
|
|
'diskPercent': disk_percent,
|
|
'diskUsedGb': disk_used_gb,
|
|
'diskTotalGb': disk_total_gb,
|
|
'processCount': process_count
|
|
}}))
|
|
PY
|
|
"""
|
|
|
|
|
|
def metrics_error_payload(error: str, instance_id: str = "") -> dict[str, Any]:
|
|
# 返回统一的主机指标错误结构。
|
|
return {
|
|
"ok": False,
|
|
"error": error,
|
|
"updatedAt": utc_now(),
|
|
"instanceId": instance_id,
|
|
}
|
|
|
|
|
|
def collect_host_metrics(hosts: list[dict[str, Any]]) -> dict[str, dict[str, Any]]:
|
|
# 建立实例 ID 到 host 的映射。
|
|
host_by_instance = {host["instanceId"]: host for host in hosts if host.get("instanceId")}
|
|
# 去重后的实例 ID 列表。
|
|
instance_ids = list(host_by_instance.keys())
|
|
|
|
# 没有实例 ID 时直接返回错误。
|
|
if not instance_ids:
|
|
return {
|
|
host["host"]: metrics_error_payload("missing instanceId in config/targets.json")
|
|
for host in hosts
|
|
}
|
|
|
|
# 通过 TAT 批量执行主机指标采集脚本。
|
|
task_results = run_shell_command(
|
|
instance_ids,
|
|
build_metrics_script(),
|
|
command_name="hy-app-monitor-host-metrics",
|
|
timeout_seconds=TAT_METRIC_TIMEOUT_SECONDS,
|
|
)
|
|
|
|
# 这里按 host name 回填最终结果。
|
|
payload_by_host: dict[str, dict[str, Any]] = {}
|
|
|
|
# 逐台主机解析结果。
|
|
for host in hosts:
|
|
# 读取实例 ID。
|
|
instance_id = host.get("instanceId", "")
|
|
# 读取该实例的远端执行结果。
|
|
task = task_results.get(instance_id, {})
|
|
# 读取执行状态。
|
|
status = task.get("status", "")
|
|
# 读取远端输出。
|
|
output = task.get("output", "")
|
|
|
|
# 成功时尝试解析 JSON。
|
|
if status == "SUCCESS":
|
|
try:
|
|
# 把 JSON 输出反序列化成 Python 结构。
|
|
payload = json.loads(output or "{}")
|
|
# 补齐统一字段。
|
|
payload["updatedAt"] = utc_now()
|
|
payload["instanceId"] = instance_id
|
|
payload_by_host[host["host"]] = payload
|
|
continue
|
|
except Exception as exc: # noqa: BLE001
|
|
payload_by_host[host["host"]] = metrics_error_payload(f"metrics parse failed: {exc}", instance_id)
|
|
continue
|
|
|
|
# 失败时保留错误文本或状态码。
|
|
payload_by_host[host["host"]] = metrics_error_payload(output.strip() or status or "metrics failed", instance_id)
|
|
|
|
# 返回按 host 索引的主机指标。
|
|
return payload_by_host
|
|
|
|
|
|
def get_host_metrics(hosts: list[dict[str, Any]]) -> dict[str, dict[str, Any]]:
|
|
# 定义真正的采集构建函数。
|
|
def builder() -> dict[str, dict[str, Any]]:
|
|
try:
|
|
# 优先返回真实采集结果。
|
|
return collect_host_metrics(hosts)
|
|
except Exception as exc: # noqa: BLE001
|
|
# 任意异常都降级成错误占位，避免拖垮整个页面。
|
|
return {
|
|
host["host"]: metrics_error_payload(f"metrics collection failed: {exc}", host.get("instanceId", ""))
|
|
for host in hosts
|
|
}
|
|
|
|
# 通过缓存保护 TAT 调用。
|
|
return HOST_METRIC_CACHE.get_or_build(builder)
|