from __future__ import annotations import concurrent.futures from typing import Any import urllib.error import urllib.request import time from core.cache import TTLCache from core.config import PROBE_CACHE_TTL_SECONDS, PROBE_MAX_WORKERS, PROBE_TIMEOUT_SECONDS, flatten_services, utc_now # 服务探测使用独立缓存,避免页面轮询直接把内网接口打爆。 SERVICE_CACHE = TTLCache[dict[str, Any]](PROBE_CACHE_TTL_SECONDS) def healthy_from_body(service_kind: str, status_code: int, body_text: str) -> bool: # 先把响应体转成小写,后续判断更简单。 lowered = body_text.lower() # Golang 服务按现有 /health 的返回结构判断。 if service_kind == "golang": return status_code == 200 and ( '"code":"ok"' in lowered or '"code": "ok"' in lowered or '"status":"ok"' in lowered or '"status": "ok"' in lowered ) # 某些 Java 健康接口虽然返回 401/403,但说明进程本身活着。 if status_code in {401, 403}: return True # 除此之外,HTTP 非 200 直接判失败。 if status_code != 200: return False # Spring Boot actuator 常见格式。 if '"status":"up"' in lowered or '"status": "up"' in lowered: return True # 兼容只返回裸字符串的健康接口。 if body_text.strip() == "UP": return True # 最后做一次保守模糊匹配。 return "up" in lowered and "status" in lowered def classify_latency(latency_ms: int, ok: bool, thresholds: dict[str, float | int]) -> str: # 服务本身不健康时直接标成 bad。 if not ok: return "bad" # 拿到 warn 阈值。 warn = float(thresholds["warn"]) # 拿到 bad 阈值。 bad = float(thresholds["bad"]) # 超过 bad 阈值记为 bad。 if latency_ms >= bad: return "bad" # 超过 warn 阈值记为 warn。 if latency_ms >= warn: return "warn" # 其余都记为 ok。 return "ok" def probe_service(service: dict[str, Any], latency_thresholds: dict[str, float | int]) -> dict[str, Any]: # 拼出健康检查地址。 url = f"http://{service['ip']}:{service['port']}{service['path']}" # 记录开始时间。 started = time.perf_counter() # 构造 HTTP 请求。 request = urllib.request.Request(url, headers={"User-Agent": "hy-app-monitor/1.0"}) # 准备统一的基础返回结构。 base = { **service, "url": url, "statusCode": 0, "latencyMs": 0, "ok": False, "level": "bad", "detail": "", } try: # 发起 HTTP 探测。 with urllib.request.urlopen(request, timeout=PROBE_TIMEOUT_SECONDS) as response: # 读取响应体。 body_text = response.read().decode("utf-8", errors="replace") # 计算请求耗时。 latency_ms = int((time.perf_counter() - started) * 1000) # 判断服务是否健康。 ok = healthy_from_body(service["kind"], response.status, body_text) # 计算当前服务的颜色级别。 level = classify_latency(latency_ms, ok, latency_thresholds) # 返回成功探测结果。 return { **base, "statusCode": int(response.status), "latencyMs": latency_ms, "ok": ok, "level": level, "detail": body_text[:220].replace("\n", " ").strip(), } except urllib.error.HTTPError as exc: # HTTPError 里仍然尽量读取响应体。 body_text = exc.read().decode("utf-8", errors="replace") # 计算失败请求耗时。 latency_ms = int((time.perf_counter() - started) * 1000) # 继续按状态码和响应体判断服务活性。 ok = healthy_from_body(service["kind"], exc.code, body_text) # 按最终健康度和耗时标记级别。 level = classify_latency(latency_ms, ok, latency_thresholds) # 返回 HTTP 错误场景结果。 return { **base, "statusCode": int(exc.code), "latencyMs": latency_ms, "ok": ok, "level": level, "detail": body_text[:220].replace("\n", " ").strip() or str(exc), } except Exception as exc: # noqa: BLE001 # 其他异常统一按失败返回。 latency_ms = int((time.perf_counter() - started) * 1000) # 返回失败结果,便于前端排查。 return { **base, "latencyMs": latency_ms, "detail": str(exc), } def build_service_payload(hosts: list[dict[str, Any]], latency_thresholds: dict[str, float | int]) -> dict[str, Any]: # 先把 hosts 展开成 services。 services = flatten_services(hosts) # 服务数量可能为 0,这里仍然保证线程池至少 1 个 worker。 workers = max(1, min(PROBE_MAX_WORKERS, len(services) or 1)) # 并发探测所有服务。 with concurrent.futures.ThreadPoolExecutor(max_workers=workers) as executor: items = list(executor.map(lambda service: probe_service(service, latency_thresholds), services)) # 统计健康服务数量。 ok_count = sum(1 for item in items if item["ok"]) # 返回服务维度监控结果。 return { "updatedAt": utc_now(), "summary": { "total": len(items), "ok": ok_count, "down": len(items) - ok_count, "hostTotal": len(hosts), }, "items": items, } def get_service_payload(hosts: list[dict[str, Any]], latency_thresholds: dict[str, float | int]) -> dict[str, Any]: # 通过缓存保护服务探测。 return SERVICE_CACHE.get_or_build(lambda: build_service_payload(hosts, latency_thresholds))