172 lines
5.8 KiB
Python
172 lines
5.8 KiB
Python
from __future__ import annotations
|
||
|
||
import concurrent.futures
|
||
from typing import Any
|
||
import urllib.error
|
||
import urllib.request
|
||
import time
|
||
|
||
from core.cache import TTLCache
|
||
from core.config import PROBE_CACHE_TTL_SECONDS, PROBE_MAX_WORKERS, PROBE_TIMEOUT_SECONDS, flatten_services, utc_now
|
||
|
||
|
||
# 服务探测使用独立缓存，避免页面轮询直接把内网接口打爆。
|
||
SERVICE_CACHE = TTLCache[dict[str, Any]](PROBE_CACHE_TTL_SECONDS)
|
||
|
||
|
||
def healthy_from_body(service_kind: str, status_code: int, body_text: str) -> bool:
|
||
# 先把响应体转成小写，后续判断更简单。
|
||
lowered = body_text.lower()
|
||
|
||
# Golang 服务按现有 /health 的返回结构判断。
|
||
if service_kind == "golang":
|
||
return status_code == 200 and (
|
||
'"code":"ok"' in lowered
|
||
or '"code": "ok"' in lowered
|
||
or '"status":"ok"' in lowered
|
||
or '"status": "ok"' in lowered
|
||
)
|
||
|
||
# 某些 Java 健康接口虽然返回 401/403，但说明进程本身活着。
|
||
if status_code in {401, 403}:
|
||
return True
|
||
|
||
# 除此之外，HTTP 非 200 直接判失败。
|
||
if status_code != 200:
|
||
return False
|
||
|
||
# Spring Boot actuator 常见格式。
|
||
if '"status":"up"' in lowered or '"status": "up"' in lowered:
|
||
return True
|
||
|
||
# 兼容只返回裸字符串的健康接口。
|
||
if body_text.strip() == "UP":
|
||
return True
|
||
|
||
# 最后做一次保守模糊匹配。
|
||
return "up" in lowered and "status" in lowered
|
||
|
||
|
||
def classify_latency(latency_ms: int, ok: bool, thresholds: dict[str, float | int]) -> str:
|
||
# 服务本身不健康时直接标成 bad。
|
||
if not ok:
|
||
return "bad"
|
||
|
||
# 拿到 warn 阈值。
|
||
warn = float(thresholds["warn"])
|
||
# 拿到 bad 阈值。
|
||
bad = float(thresholds["bad"])
|
||
|
||
# 超过 bad 阈值记为 bad。
|
||
if latency_ms >= bad:
|
||
return "bad"
|
||
|
||
# 超过 warn 阈值记为 warn。
|
||
if latency_ms >= warn:
|
||
return "warn"
|
||
|
||
# 其余都记为 ok。
|
||
return "ok"
|
||
|
||
|
||
def probe_service(service: dict[str, Any], latency_thresholds: dict[str, float | int]) -> dict[str, Any]:
|
||
# 拼出健康检查地址。
|
||
url = f"http://{service['ip']}:{service['port']}{service['path']}"
|
||
# 记录开始时间。
|
||
started = time.perf_counter()
|
||
# 构造 HTTP 请求。
|
||
request = urllib.request.Request(url, headers={"User-Agent": "hy-app-monitor/1.0"})
|
||
|
||
# 准备统一的基础返回结构。
|
||
base = {
|
||
**service,
|
||
"url": url,
|
||
"statusCode": 0,
|
||
"latencyMs": 0,
|
||
"ok": False,
|
||
"level": "bad",
|
||
"detail": "",
|
||
}
|
||
|
||
try:
|
||
# 发起 HTTP 探测。
|
||
with urllib.request.urlopen(request, timeout=PROBE_TIMEOUT_SECONDS) as response:
|
||
# 读取响应体。
|
||
body_text = response.read().decode("utf-8", errors="replace")
|
||
# 计算请求耗时。
|
||
latency_ms = int((time.perf_counter() - started) * 1000)
|
||
# 判断服务是否健康。
|
||
ok = healthy_from_body(service["kind"], response.status, body_text)
|
||
# 计算当前服务的颜色级别。
|
||
level = classify_latency(latency_ms, ok, latency_thresholds)
|
||
|
||
# 返回成功探测结果。
|
||
return {
|
||
**base,
|
||
"statusCode": int(response.status),
|
||
"latencyMs": latency_ms,
|
||
"ok": ok,
|
||
"level": level,
|
||
"detail": body_text[:220].replace("\n", " ").strip(),
|
||
}
|
||
except urllib.error.HTTPError as exc:
|
||
# HTTPError 里仍然尽量读取响应体。
|
||
body_text = exc.read().decode("utf-8", errors="replace")
|
||
# 计算失败请求耗时。
|
||
latency_ms = int((time.perf_counter() - started) * 1000)
|
||
# 继续按状态码和响应体判断服务活性。
|
||
ok = healthy_from_body(service["kind"], exc.code, body_text)
|
||
# 按最终健康度和耗时标记级别。
|
||
level = classify_latency(latency_ms, ok, latency_thresholds)
|
||
|
||
# 返回 HTTP 错误场景结果。
|
||
return {
|
||
**base,
|
||
"statusCode": int(exc.code),
|
||
"latencyMs": latency_ms,
|
||
"ok": ok,
|
||
"level": level,
|
||
"detail": body_text[:220].replace("\n", " ").strip() or str(exc),
|
||
}
|
||
except Exception as exc: # noqa: BLE001
|
||
# 其他异常统一按失败返回。
|
||
latency_ms = int((time.perf_counter() - started) * 1000)
|
||
|
||
# 返回失败结果，便于前端排查。
|
||
return {
|
||
**base,
|
||
"latencyMs": latency_ms,
|
||
"detail": str(exc),
|
||
}
|
||
|
||
|
||
def build_service_payload(hosts: list[dict[str, Any]], latency_thresholds: dict[str, float | int]) -> dict[str, Any]:
|
||
# 先把 hosts 展开成 services。
|
||
services = flatten_services(hosts)
|
||
# 服务数量可能为 0，这里仍然保证线程池至少 1 个 worker。
|
||
workers = max(1, min(PROBE_MAX_WORKERS, len(services) or 1))
|
||
|
||
# 并发探测所有服务。
|
||
with concurrent.futures.ThreadPoolExecutor(max_workers=workers) as executor:
|
||
items = list(executor.map(lambda service: probe_service(service, latency_thresholds), services))
|
||
|
||
# 统计健康服务数量。
|
||
ok_count = sum(1 for item in items if item["ok"])
|
||
|
||
# 返回服务维度监控结果。
|
||
return {
|
||
"updatedAt": utc_now(),
|
||
"summary": {
|
||
"total": len(items),
|
||
"ok": ok_count,
|
||
"down": len(items) - ok_count,
|
||
"hostTotal": len(hosts),
|
||
},
|
||
"items": items,
|
||
}
|
||
|
||
|
||
def get_service_payload(hosts: list[dict[str, Any]], latency_thresholds: dict[str, float | int]) -> dict[str, Any]:
|
||
# 通过缓存保护服务探测。
|
||
return SERVICE_CACHE.get_or_build(lambda: build_service_payload(hosts, latency_thresholds))
|