From 0edf38041ec693fe823a085f4059d6a9c38aeaa4 Mon Sep 17 00:00:00 2001 From: zhx Date: Wed, 8 Jul 2026 17:47:58 +0800 Subject: [PATCH] =?UTF-8?q?=E9=83=A8=E7=BD=B2?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../deploy-likei-services.sh | 42 +- .deploy/prod-deploy/ops/aslan-ops.service | 21 + .deploy/prod-deploy/ops/aslan_ops.py | 607 ++++++++++++++++++ .../sync-and-deploy.sh | 14 +- .deploy/test-deploy/deploy-likei-services.sh | 345 ++++++++++ .../test-deploy/ops/aslan-test-ops.service | 21 + .deploy/test-deploy/ops/aslan_ops.py | 607 ++++++++++++++++++ .deploy/test-deploy/sync-and-deploy.sh | 84 +++ 8 files changed, 1728 insertions(+), 13 deletions(-) rename .deploy/{aslan-deploy => prod-deploy}/deploy-likei-services.sh (85%) create mode 100644 .deploy/prod-deploy/ops/aslan-ops.service create mode 100644 .deploy/prod-deploy/ops/aslan_ops.py rename .deploy/{aslan-deploy => prod-deploy}/sync-and-deploy.sh (84%) create mode 100755 .deploy/test-deploy/deploy-likei-services.sh create mode 100644 .deploy/test-deploy/ops/aslan-test-ops.service create mode 100644 .deploy/test-deploy/ops/aslan_ops.py create mode 100755 .deploy/test-deploy/sync-and-deploy.sh diff --git a/.deploy/aslan-deploy/deploy-likei-services.sh b/.deploy/prod-deploy/deploy-likei-services.sh similarity index 85% rename from .deploy/aslan-deploy/deploy-likei-services.sh rename to .deploy/prod-deploy/deploy-likei-services.sh index 7e06ba34..e797cd4a 100755 --- a/.deploy/aslan-deploy/deploy-likei-services.sh +++ b/.deploy/prod-deploy/deploy-likei-services.sh @@ -4,14 +4,19 @@ set -euo pipefail BASE_DIR="${BASE_DIR:-/opt/aslan-deploy}" SRC_DIR="${SRC_DIR:-$BASE_DIR/source/likei-services}" KUBECONFIG="${KUBECONFIG:-$BASE_DIR/kubeconfig}" +LOCK_FILE="${LOCK_FILE:-$BASE_DIR/deploy.lock}" NAMESPACE="${NAMESPACE:-prod}" MODE="${MODE:-preload}" SKIP_BUILD="${SKIP_BUILD:-0}" USE_GIT_SOURCE="${USE_GIT_SOURCE:-0}" GIT_REMOTE_URL="${GIT_REMOTE_URL:-git@gitea.haiyihy.com:hy/aslan-server.git}" -GIT_REF="${GIT_REF:-aslan_test}" +GIT_REF="${GIT_REF:-aslan_prod}" +MAVEN_GOALS="${MAVEN_GOALS:-clean package}" +MAVEN_PROFILE="${MAVEN_PROFILE:-prod}" MAVEN_EXTRA_ARGS="${MAVEN_EXTRA_ARGS:-}" BUILD_TS="${BUILD_TS:-$(date +%Y%m%dv%H%M%S)}" +IMAGE_REGISTRY="${IMAGE_REGISTRY:-tuokemi-con-registry.ap-southeast-1.cr.aliyuncs.com}" +IMAGE_PROJECT="${IMAGE_PROJECT:-atyou-prod}" BASE_IMAGE="tuokemi-con-registry.ap-southeast-1.cr.aliyuncs.com/public_mirror_images/eclipse-temurin:17-jdk-jammy" FALLBACK_BASE_IMAGE="${FALLBACK_BASE_IMAGE:-eclipse-temurin:17-jdk-jammy}" @@ -23,7 +28,7 @@ Usage: Examples: /opt/aslan-deploy/deploy-likei-services.sh status /opt/aslan-deploy/deploy-likei-services.sh --mode preload other external console - USE_GIT_SOURCE=1 GIT_REF=aslan_test /opt/aslan-deploy/deploy-likei-services.sh --mode preload other + USE_GIT_SOURCE=1 GIT_REF=aslan_prod /opt/aslan-deploy/deploy-likei-services.sh --mode preload other MODE=push ALIYUN_USER=xxx ALIYUN_PASS=xxx /opt/aslan-deploy/deploy-likei-services.sh other Notes: @@ -44,6 +49,23 @@ need_cmd() { } } +acquire_deploy_lock() { + need_cmd flock + exec 9>"$LOCK_FILE" + # Only one deploy should mutate source, build images, preload nodes, and roll deployments at a time. + flock -n 9 || { + echo "another deploy is already running: $LOCK_FILE" >&2 + exit 5 + } +} + +validate_git_ref() { + if [[ ! "$GIT_REF" =~ ^[A-Za-z0-9._/-]{1,80}$ || "$GIT_REF" == -* || "$GIT_REF" == *..* ]]; then + echo "invalid GIT_REF: $GIT_REF" >&2 + exit 2 + fi +} + ensure_base_image() { if docker image inspect "$BASE_IMAGE" >/dev/null 2>&1; then return @@ -62,6 +84,7 @@ update_source_from_git() { fi need_cmd git + validate_git_ref mkdir -p "$(dirname "$SRC_DIR")" if [[ ! -d "$SRC_DIR/.git" ]]; then rm -rf "$SRC_DIR" @@ -73,7 +96,7 @@ update_source_from_git() { log "git reset source to origin/$GIT_REF from $GIT_REMOTE_URL" git -C "$SRC_DIR" remote set-url origin "$GIT_REMOTE_URL" git -C "$SRC_DIR" fetch origin "$GIT_REF" - git -C "$SRC_DIR" checkout "$GIT_REF" + git -C "$SRC_DIR" checkout -B "$GIT_REF" "origin/$GIT_REF" git -C "$SRC_DIR" reset --hard "origin/$GIT_REF" } @@ -97,9 +120,9 @@ service_dir() { image_repo() { case "$1" in - other) echo "tuokemi-con-registry.ap-southeast-1.cr.aliyuncs.com/atyou-prod/other" ;; - external) echo "tuokemi-con-registry.ap-southeast-1.cr.aliyuncs.com/atyou-prod/external" ;; - console) echo "tuokemi-con-registry.ap-southeast-1.cr.aliyuncs.com/atyou-prod/console" ;; + other) echo "$IMAGE_REGISTRY/$IMAGE_PROJECT/other" ;; + external) echo "$IMAGE_REGISTRY/$IMAGE_PROJECT/external" ;; + console) echo "$IMAGE_REGISTRY/$IMAGE_PROJECT/console" ;; *) echo "unsupported service: $1" >&2; exit 2 ;; esac } @@ -121,13 +144,14 @@ build_service() { if [[ "$SKIP_BUILD" != "1" ]]; then log "maven package $svc ($module)" # shellcheck disable=SC2086 - mvn clean package -pl "$module" -am -P prod -Dmaven.test.skip=true $MAVEN_EXTRA_ARGS + # MAVEN_GOALS=package keeps the existing target cache for code-only fast builds; use the default clean package for safer full builds. + mvn $MAVEN_GOALS -pl "$module" -am -P "$MAVEN_PROFILE" -Dmaven.test.skip=true $MAVEN_EXTRA_ARGS fi ensure_base_image log "docker build $image" docker build \ - --build-arg "SERVICE_VERSION=atyou-prod:${tag}" \ + --build-arg "SERVICE_VERSION=${IMAGE_PROJECT}:${tag}" \ -f "$dir/Dockerfile" \ -t "$image" \ "$dir" @@ -274,6 +298,7 @@ main() { while [[ "$#" -gt 0 ]]; do case "$1" in --mode) + [[ -n "${2:-}" ]] || { echo "--mode requires an argument" >&2; exit 2; } MODE="$2" shift 2 ;; @@ -304,6 +329,7 @@ main() { need_cmd kubectl [[ -f "$KUBECONFIG" ]] || { echo "missing kubeconfig: $KUBECONFIG" >&2; exit 2; } + acquire_deploy_lock update_source_from_git local services=("$@") diff --git a/.deploy/prod-deploy/ops/aslan-ops.service b/.deploy/prod-deploy/ops/aslan-ops.service new file mode 100644 index 00000000..5471d049 --- /dev/null +++ b/.deploy/prod-deploy/ops/aslan-ops.service @@ -0,0 +1,21 @@ +[Unit] +Description=Aslan deployment operations panel +After=network-online.target docker.service +Wants=network-online.target + +[Service] +Type=simple +User=ubuntu +Group=ubuntu +WorkingDirectory=/opt/aslan-deploy +Environment=PATH=/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin +Environment=HOME=/home/ubuntu +Environment=ASLAN_ALLOWED_BRANCHES=aslan_prod +EnvironmentFile=/etc/aslan-ops.env +ExecStart=/usr/bin/python3 /opt/aslan-deploy/ops/aslan_ops.py +Restart=always +RestartSec=5 +NoNewPrivileges=true + +[Install] +WantedBy=multi-user.target diff --git a/.deploy/prod-deploy/ops/aslan_ops.py b/.deploy/prod-deploy/ops/aslan_ops.py new file mode 100644 index 00000000..e6f90f99 --- /dev/null +++ b/.deploy/prod-deploy/ops/aslan_ops.py @@ -0,0 +1,607 @@ +#!/usr/bin/env python3 +import base64 +import hashlib +import hmac +import html +import json +import os +import re +import secrets +import subprocess +import threading +import time +from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer +from pathlib import Path +from urllib.parse import parse_qs, urlparse + + +BASE_DIR = Path(os.environ.get("ASLAN_DEPLOY_BASE", "/opt/aslan-deploy")) +RUN_DIR = Path(os.environ.get("ASLAN_OPS_RUN_DIR", str(BASE_DIR / "ops" / "runs"))) +KUBECONFIG = Path(os.environ.get("KUBECONFIG", str(BASE_DIR / "kubeconfig"))) +DEPLOY_SCRIPT = Path(os.environ.get("ASLAN_DEPLOY_SCRIPT", str(BASE_DIR / "deploy-likei-services.sh"))) +SOURCE_DIR = Path(os.environ.get("ASLAN_SOURCE_DIR", str(BASE_DIR / "source" / "likei-services"))) +NAMESPACE = os.environ.get("ASLAN_NAMESPACE", "prod") +GIT_REF = os.environ.get("ASLAN_GIT_REF", "aslan_prod") +REMOTE_URL = os.environ.get("ASLAN_GIT_REMOTE_URL", "git@gitea.haiyihy.com:hy/aslan-server.git") +HOST = os.environ.get("ASLAN_OPS_HOST", "0.0.0.0") +PORT = int(os.environ.get("ASLAN_OPS_PORT", "18080")) +USERNAME = os.environ.get("ASLAN_OPS_USER", "admin") +PASSWORD = os.environ.get("ASLAN_OPS_PASSWORD", "") +SESSION_SECRET = os.environ.get("ASLAN_OPS_SESSION_SECRET", PASSWORD or secrets.token_hex(32)) +ALLOWED_BRANCHES = { + branch.strip() + for branch in os.environ.get("ASLAN_ALLOWED_BRANCHES", GIT_REF).split(",") + if branch.strip() +} + +SERVICES = { + "other": { + "label": "Other", + "port": "5800", + "health": "/actuator/health", + }, + "external": { + "label": "External", + "port": "5200", + "health": "/actuator/health", + }, + "console": { + "label": "Console", + "port": "5300", + "health": "/console/actuator/health", + }, +} + +BRANCH_RE = re.compile(r"^[A-Za-z0-9._/-]{1,80}$") + +job_lock = threading.Lock() +active_job = None +login_failures = {} + + +def now_ts(): + return time.strftime("%Y-%m-%d %H:%M:%S") + + +def json_response(handler, status, payload): + body = json.dumps(payload, ensure_ascii=False).encode("utf-8") + handler.send_response(status) + handler.send_header("Content-Type", "application/json; charset=utf-8") + handler.send_header("Cache-Control", "no-store") + handler.send_header("X-Frame-Options", "DENY") + handler.send_header("Content-Length", str(len(body))) + handler.end_headers() + handler.wfile.write(body) + + +def text_response(handler, status, body, content_type="text/plain; charset=utf-8"): + data = body.encode("utf-8") + handler.send_response(status) + handler.send_header("Content-Type", content_type) + handler.send_header("Cache-Control", "no-store") + handler.send_header("X-Frame-Options", "DENY") + handler.send_header("Content-Length", str(len(data))) + handler.end_headers() + handler.wfile.write(data) + + +def redirect_response(handler, location, cookies=None): + handler.send_response(302) + for cookie in cookies or []: + handler.send_header("Set-Cookie", cookie) + handler.send_header("Location", location) + handler.send_header("Content-Length", "0") + handler.end_headers() + + +def sign_session(value): + mac = hmac.new(SESSION_SECRET.encode("utf-8"), value.encode("utf-8"), hashlib.sha256).hexdigest() + return f"{value}.{mac}" + + +def verify_session(signed): + if not signed or "." not in signed: + return False + value, mac = signed.rsplit(".", 1) + expected = hmac.new(SESSION_SECRET.encode("utf-8"), value.encode("utf-8"), hashlib.sha256).hexdigest() + if not hmac.compare_digest(mac, expected): + return False + try: + username, issued = value.split(":", 1) + return username == USERNAME and time.time() - int(issued) < 86400 + except ValueError: + return False + + +def parse_cookies(header): + cookies = {} + for part in (header or "").split(";"): + if "=" not in part: + continue + key, value = part.strip().split("=", 1) + cookies[key] = value + return cookies + + +def too_many_login_failures(ip): + now = time.time() + window = [ts for ts in login_failures.get(ip, []) if now - ts < 300] + login_failures[ip] = window + return len(window) >= 5 + + +def record_login_failure(ip): + window = login_failures.get(ip, []) + window.append(time.time()) + login_failures[ip] = window[-10:] + + +def run_json(cmd, timeout=20): + result = subprocess.run(cmd, text=True, capture_output=True, timeout=timeout) + if result.returncode != 0: + raise RuntimeError(result.stderr.strip() or result.stdout.strip() or f"command failed: {cmd[0]}") + return json.loads(result.stdout) + + +def run_text(cmd, timeout=10): + result = subprocess.run(cmd, text=True, capture_output=True, timeout=timeout) + if result.returncode != 0: + return "" + return result.stdout.strip() + + +def deployment_status(): + names = list(SERVICES) + payload = run_json([ + "kubectl", + "--kubeconfig", + str(KUBECONFIG), + "-n", + NAMESPACE, + "get", + "deploy", + *names, + "-o", + "json", + ]) + items = [] + for item in payload.get("items", []): + name = item["metadata"]["name"] + spec = item.get("spec", {}) + status = item.get("status", {}) + containers = item.get("spec", {}).get("template", {}).get("spec", {}).get("containers", []) + image = next((c.get("image", "") for c in containers if c.get("name") == name), containers[0].get("image", "") if containers else "") + conditions = {c.get("type"): c.get("status") for c in status.get("conditions", [])} + items.append({ + "name": name, + "label": SERVICES.get(name, {}).get("label", name), + "ready": f"{status.get('readyReplicas', 0)}/{spec.get('replicas', 0)}", + "updated": status.get("updatedReplicas", 0), + "available": status.get("availableReplicas", 0), + "image": image, + "port": SERVICES.get(name, {}).get("port", ""), + "health": SERVICES.get(name, {}).get("health", ""), + "healthy": conditions.get("Available") == "True", + }) + order = {name: idx for idx, name in enumerate(names)} + return sorted(items, key=lambda x: order.get(x["name"], 99)) + + +def pod_status(): + selector = "app in (other,external,console)" + payload = run_json([ + "kubectl", + "--kubeconfig", + str(KUBECONFIG), + "-n", + NAMESPACE, + "get", + "pods", + "-l", + selector, + "-o", + "json", + ]) + pods = [] + for item in payload.get("items", []): + status = item.get("status", {}) + containers = status.get("containerStatuses", []) + ready_count = sum(1 for c in containers if c.get("ready")) + pods.append({ + "name": item["metadata"]["name"], + "app": item["metadata"].get("labels", {}).get("app", ""), + "ready": f"{ready_count}/{len(containers)}", + "phase": status.get("phase", ""), + "restarts": sum(c.get("restartCount", 0) for c in containers), + "node": item.get("spec", {}).get("nodeName", ""), + "ip": status.get("podIP", ""), + "age": item["metadata"].get("creationTimestamp", ""), + }) + return pods + + +def git_status(): + head = run_text(["git", "-C", str(SOURCE_DIR), "rev-parse", "HEAD"]) + branch = run_text(["git", "-C", str(SOURCE_DIR), "branch", "--show-current"]) + short = run_text(["git", "-C", str(SOURCE_DIR), "status", "--short", "--branch"]) + return {"head": head, "branch": branch, "status": short} + + +def list_jobs(): + RUN_DIR.mkdir(parents=True, exist_ok=True) + jobs = [] + for meta_path in sorted(RUN_DIR.glob("*.json"), key=lambda p: p.stat().st_mtime, reverse=True)[:20]: + try: + jobs.append(json.loads(meta_path.read_text())) + except json.JSONDecodeError: + continue + return jobs + + +def save_job(job): + RUN_DIR.mkdir(parents=True, exist_ok=True) + (RUN_DIR / f"{job['id']}.json").write_text(json.dumps(job, ensure_ascii=False, indent=2)) + + +def stream_process(job, cmd, env): + log_path = RUN_DIR / f"{job['id']}.log" + try: + with log_path.open("a", buffering=1) as log: + log.write(f"[{now_ts()}] start {' '.join(cmd)}\n") + process = subprocess.Popen( + cmd, + cwd=str(BASE_DIR), + env=env, + stdout=subprocess.PIPE, + stderr=subprocess.STDOUT, + text=True, + bufsize=1, + ) + job["pid"] = process.pid + job["status"] = "running" + save_job(job) + for line in process.stdout: + log.write(line) + rc = process.wait() + job["status"] = "success" if rc == 0 else "failed" + job["returnCode"] = rc + job["finishedAt"] = now_ts() + save_job(job) + log.write(f"[{now_ts()}] finished rc={rc}\n") + except Exception as exc: + job["status"] = "failed" + job["returnCode"] = -1 + job["finishedAt"] = now_ts() + job["error"] = str(exc) + save_job(job) + with log_path.open("a", buffering=1) as log: + log.write(f"[{now_ts()}] runner error: {exc}\n") + finally: + global active_job + with job_lock: + active_job = None + + +def start_deploy(services, branch, fast): + global active_job + invalid = [svc for svc in services if svc not in SERVICES] + if invalid: + raise ValueError(f"unsupported services: {', '.join(invalid)}") + if not services: + raise ValueError("select at least one service") + if not BRANCH_RE.match(branch): + raise ValueError("invalid branch name") + if branch not in ALLOWED_BRANCHES: + raise ValueError(f"branch is not allowed: {branch}") + if not DEPLOY_SCRIPT.exists(): + raise ValueError(f"missing deploy script: {DEPLOY_SCRIPT}") + + with job_lock: + if active_job: + raise RuntimeError(f"deploy already running: {active_job['id']}") + job_id = time.strftime("%Y%m%d%H%M%S") + "-" + secrets.token_hex(3) + job = { + "id": job_id, + "services": services, + "branch": branch, + "fast": fast, + "status": "queued", + "createdAt": now_ts(), + "finishedAt": "", + "returnCode": None, + } + active_job = job + save_job(job) + + env = os.environ.copy() + env.update({ + "USE_GIT_SOURCE": "1", + "GIT_REF": branch, + "GIT_REMOTE_URL": REMOTE_URL, + "MODE": "preload", + "KUBECONFIG": str(KUBECONFIG), + "NAMESPACE": NAMESPACE, + "ASLAN_NAMESPACE": NAMESPACE, + "HOME": str(Path.home()), + }) + if fast: + # Fast mode avoids Maven clean so unchanged modules and already downloaded dependencies can reuse local target output. + env["MAVEN_GOALS"] = "package" + cmd = [str(DEPLOY_SCRIPT), "--mode", "preload", *services] + thread = threading.Thread(target=stream_process, args=(job, cmd, env), daemon=True) + thread.start() + return job + + +def login_page(error=""): + message = f"
{html.escape(error)}
" if error else "" + return f""" + + + + + Aslan Ops + + +

Aslan Ops

{message}
+""" + + +INDEX_HTML = """ + + + + + Aslan Ops + + + +
+

Aslan Ops

+ +
+
+
+

微服务

+
+
+
+

部署

固定 preload 到 TKE 节点,不走 registry push
+
+
+ + + +
+
+
+
+

部署日志

无运行任务
+

+      
+
+

Pod

+
名称状态节点
+
+
+
+

最近任务

+
ID服务分支状态时间
+
+
+ + +""" + + +class Handler(BaseHTTPRequestHandler): + server_version = "AslanOps/1.0" + + def authenticated(self): + cookies = parse_cookies(self.headers.get("Cookie")) + return verify_session(cookies.get("aslan_ops")) + + def require_auth(self): + if self.authenticated(): + return True + redirect_response(self, "/login") + return False + + def read_body(self, max_bytes=65536): + size = int(self.headers.get("Content-Length", "0")) + if size > max_bytes: + raise ValueError("request body too large") + return self.rfile.read(size).decode("utf-8") + + def do_GET(self): + parsed = urlparse(self.path) + if parsed.path == "/login": + return text_response(self, 200, login_page(), "text/html; charset=utf-8") + if parsed.path == "/logout": + redirect_response(self, "/login", ["aslan_ops=; Max-Age=0; HttpOnly; SameSite=Lax; Path=/"]) + return + if not self.require_auth(): + return + if parsed.path == "/": + return text_response(self, 200, INDEX_HTML, "text/html; charset=utf-8") + if parsed.path == "/api/status": + try: + return json_response(self, 200, { + "deployments": deployment_status(), + "pods": pod_status(), + "git": git_status(), + "jobs": list_jobs(), + }) + except Exception as exc: + return json_response(self, 500, {"error": str(exc)}) + match = re.match(r"^/api/jobs/([A-Za-z0-9-]+)/log$", parsed.path) + if match: + job_id = match.group(1) + meta_path = RUN_DIR / f"{job_id}.json" + log_path = RUN_DIR / f"{job_id}.log" + if not meta_path.exists(): + return json_response(self, 404, {"error": "job not found"}) + job = json.loads(meta_path.read_text()) + log = log_path.read_text(errors="replace")[-120000:] if log_path.exists() else "" + return json_response(self, 200, {"job": job, "log": log}) + return text_response(self, 404, "not found") + + def do_POST(self): + parsed = urlparse(self.path) + if parsed.path == "/login": + if too_many_login_failures(self.client_address[0]): + return text_response(self, 429, login_page("登录失败次数过多,稍后再试"), "text/html; charset=utf-8") + try: + form = parse_qs(self.read_body(max_bytes=4096)) + except ValueError as exc: + return text_response(self, 413, str(exc)) + username = form.get("username", [""])[0] + password = form.get("password", [""])[0] + if hmac.compare_digest(username, USERNAME) and hmac.compare_digest(password, PASSWORD): + login_failures.pop(self.client_address[0], None) + session = sign_session(f"{USERNAME}:{int(time.time())}") + redirect_response(self, "/", [f"aslan_ops={session}; HttpOnly; SameSite=Lax; Path=/"]) + return + record_login_failure(self.client_address[0]) + return text_response(self, 401, login_page("账号或密码错误"), "text/html; charset=utf-8") + if not self.require_auth(): + return + if parsed.path == "/api/deploy": + try: + if self.headers.get("X-Aslan-Ops") != "1": + return json_response(self, 403, {"error": "missing deploy header"}) + payload = json.loads(self.read_body(max_bytes=4096) or "{}") + services = payload.get("services") or [] + branch = payload.get("branch") or GIT_REF + fast = bool(payload.get("fast", True)) + job = start_deploy(services, branch, fast) + return json_response(self, 202, {"job": job}) + except Exception as exc: + return json_response(self, 400, {"error": str(exc)}) + return text_response(self, 404, "not found") + + def log_message(self, fmt, *args): + print(f"[{now_ts()}] {self.address_string()} {fmt % args}", flush=True) + + +def main(): + if not PASSWORD: + raise SystemExit("ASLAN_OPS_PASSWORD is required") + RUN_DIR.mkdir(parents=True, exist_ok=True) + server = ThreadingHTTPServer((HOST, PORT), Handler) + print(f"aslan ops listening on {HOST}:{PORT}", flush=True) + server.serve_forever() + + +if __name__ == "__main__": + main() diff --git a/.deploy/aslan-deploy/sync-and-deploy.sh b/.deploy/prod-deploy/sync-and-deploy.sh similarity index 84% rename from .deploy/aslan-deploy/sync-and-deploy.sh rename to .deploy/prod-deploy/sync-and-deploy.sh index c42786c1..60ec4129 100755 --- a/.deploy/aslan-deploy/sync-and-deploy.sh +++ b/.deploy/prod-deploy/sync-and-deploy.sh @@ -15,12 +15,12 @@ REPO_ROOT="$(cd "$SCRIPT_DIR/../.." && pwd)" usage() { cat <<'EOF' Usage: - .deploy/aslan-deploy/sync-and-deploy.sh [remote deploy args...] + .deploy/prod-deploy/sync-and-deploy.sh [remote deploy args...] Examples: - .deploy/aslan-deploy/sync-and-deploy.sh status - .deploy/aslan-deploy/sync-and-deploy.sh --mode preload other - .deploy/aslan-deploy/sync-and-deploy.sh --mode preload other external console + .deploy/prod-deploy/sync-and-deploy.sh status + .deploy/prod-deploy/sync-and-deploy.sh --mode preload other + .deploy/prod-deploy/sync-and-deploy.sh --mode preload other external console Environment: DEPLOY_HOST default 43.160.219.15 @@ -29,7 +29,7 @@ Environment: Git source on deploy host: ssh -i ~/.ssh/aslan-deploy-sg-ed25519 ubuntu@43.160.219.15 \ - 'USE_GIT_SOURCE=1 GIT_REF=aslan_test /opt/aslan-deploy/deploy-likei-services.sh --mode preload other' + 'USE_GIT_SOURCE=1 GIT_REF=aslan_prod /opt/aslan-deploy/deploy-likei-services.sh --mode preload other' EOF } @@ -58,6 +58,10 @@ rsync -az \ -e "$rsync_ssh" \ "$SCRIPT_DIR/deploy-likei-services.sh" "$DEPLOY_USER@$DEPLOY_HOST:$REMOTE_BASE/deploy-likei-services.sh" +rsync -az --delete \ + -e "$rsync_ssh" \ + "$SCRIPT_DIR/ops/" "$DEPLOY_USER@$DEPLOY_HOST:$REMOTE_BASE/ops/" + for cache_group in $M2_CACHE_GROUPS; do local_cache_path="$LOCAL_M2_ROOT/$cache_group" remote_cache_parent="$REMOTE_M2_ROOT/$(dirname "$cache_group")" diff --git a/.deploy/test-deploy/deploy-likei-services.sh b/.deploy/test-deploy/deploy-likei-services.sh new file mode 100755 index 00000000..f69abc0c --- /dev/null +++ b/.deploy/test-deploy/deploy-likei-services.sh @@ -0,0 +1,345 @@ +#!/usr/bin/env bash +set -euo pipefail + +BASE_DIR="${BASE_DIR:-/opt/aslan-test-deploy}" +SRC_DIR="${SRC_DIR:-$BASE_DIR/source/likei-services}" +KUBECONFIG="${KUBECONFIG:-$BASE_DIR/kubeconfig}" +LOCK_FILE="${LOCK_FILE:-$BASE_DIR/deploy.lock}" +NAMESPACE="${NAMESPACE:-test}" +MODE="${MODE:-preload}" +SKIP_BUILD="${SKIP_BUILD:-0}" +USE_GIT_SOURCE="${USE_GIT_SOURCE:-0}" +GIT_REMOTE_URL="${GIT_REMOTE_URL:-git@gitea.haiyihy.com:hy/aslan-server.git}" +GIT_REF="${GIT_REF:-aslan_test}" +MAVEN_GOALS="${MAVEN_GOALS:-clean package}" +MAVEN_PROFILE="${MAVEN_PROFILE:-test}" +MAVEN_EXTRA_ARGS="${MAVEN_EXTRA_ARGS:-}" +BUILD_TS="${BUILD_TS:-$(date +%Y%m%dv%H%M%S)}" +IMAGE_REGISTRY="${IMAGE_REGISTRY:-tuokemi-con-registry.ap-southeast-1.cr.aliyuncs.com}" +IMAGE_PROJECT="${IMAGE_PROJECT:-atyou-test}" +BASE_IMAGE="tuokemi-con-registry.ap-southeast-1.cr.aliyuncs.com/public_mirror_images/eclipse-temurin:17-jdk-jammy" +FALLBACK_BASE_IMAGE="${FALLBACK_BASE_IMAGE:-eclipse-temurin:17-jdk-jammy}" + +usage() { + cat <<'EOF' +Usage: + deploy-likei-services.sh [--mode preload|push|build-only|status] [--skip-build] [other|external|console ...] + +Examples: + /opt/aslan-test-deploy/deploy-likei-services.sh status + /opt/aslan-test-deploy/deploy-likei-services.sh --mode preload other external console + USE_GIT_SOURCE=1 GIT_REF=aslan_test /opt/aslan-test-deploy/deploy-likei-services.sh --mode preload other + MODE=push ALIYUN_USER=xxx ALIYUN_PASS=xxx /opt/aslan-test-deploy/deploy-likei-services.sh other + +Notes: + - preload mode builds the image on this server, imports it into each TKE node's containerd, then updates the deployment. + - push mode follows the original Jenkins flow and requires registry credentials in ALIYUN_USER/ALIYUN_PASS. + - USE_GIT_SOURCE=1 clones or resets source from GIT_REMOTE_URL/GIT_REF before building. +EOF +} + +log() { + printf '[%s] %s\n' "$(date '+%F %T')" "$*" +} + +need_cmd() { + command -v "$1" >/dev/null 2>&1 || { + echo "missing command: $1" >&2 + exit 2 + } +} + +acquire_deploy_lock() { + need_cmd flock + exec 9>"$LOCK_FILE" + # Only one deploy should mutate source, build images, preload nodes, and roll deployments at a time. + flock -n 9 || { + echo "another deploy is already running: $LOCK_FILE" >&2 + exit 5 + } +} + +validate_git_ref() { + if [[ ! "$GIT_REF" =~ ^[A-Za-z0-9._/-]{1,80}$ || "$GIT_REF" == -* || "$GIT_REF" == *..* ]]; then + echo "invalid GIT_REF: $GIT_REF" >&2 + exit 2 + fi +} + +ensure_base_image() { + if docker image inspect "$BASE_IMAGE" >/dev/null 2>&1; then + return + fi + if docker pull "$BASE_IMAGE"; then + return + fi + log "base image not available from primary registry, pulling $FALLBACK_BASE_IMAGE" + docker pull "$FALLBACK_BASE_IMAGE" + docker tag "$FALLBACK_BASE_IMAGE" "$BASE_IMAGE" +} + +update_source_from_git() { + if [[ "$USE_GIT_SOURCE" != "1" ]]; then + return + fi + + need_cmd git + validate_git_ref + mkdir -p "$(dirname "$SRC_DIR")" + if [[ ! -d "$SRC_DIR/.git" ]]; then + rm -rf "$SRC_DIR" + log "git clone $GIT_REMOTE_URL ($GIT_REF) -> $SRC_DIR" + git clone --branch "$GIT_REF" "$GIT_REMOTE_URL" "$SRC_DIR" + return + fi + + log "git reset source to origin/$GIT_REF from $GIT_REMOTE_URL" + git -C "$SRC_DIR" remote set-url origin "$GIT_REMOTE_URL" + git -C "$SRC_DIR" fetch origin "$GIT_REF" + git -C "$SRC_DIR" checkout -B "$GIT_REF" "origin/$GIT_REF" + git -C "$SRC_DIR" reset --hard "origin/$GIT_REF" +} + +service_module() { + case "$1" in + other) echo "rc-service/rc-service-other/other-start" ;; + external) echo "rc-service/rc-service-external/external-start" ;; + console) echo "rc-service/rc-service-console/console-start" ;; + *) echo "unsupported service: $1" >&2; exit 2 ;; + esac +} + +service_dir() { + case "$1" in + other) echo "rc-service/rc-service-other" ;; + external) echo "rc-service/rc-service-external" ;; + console) echo "rc-service/rc-service-console" ;; + *) echo "unsupported service: $1" >&2; exit 2 ;; + esac +} + +image_repo() { + case "$1" in + other) echo "$IMAGE_REGISTRY/$IMAGE_PROJECT/other" ;; + external) echo "$IMAGE_REGISTRY/$IMAGE_PROJECT/external" ;; + console) echo "$IMAGE_REGISTRY/$IMAGE_PROJECT/console" ;; + *) echo "unsupported service: $1" >&2; exit 2 ;; + esac +} + +build_service() { + local svc="$1" + local module + local dir + local repo + local tag + local image + module="$(service_module "$svc")" + dir="$(service_dir "$svc")" + repo="$(image_repo "$svc")" + tag="${svc}-${BUILD_TS}" + image="${repo}:${tag}" + + cd "$SRC_DIR" + if [[ "$SKIP_BUILD" != "1" ]]; then + log "maven package $svc ($module)" + # shellcheck disable=SC2086 + # MAVEN_GOALS=package keeps the existing target cache for code-only fast builds; use the default clean package for safer full builds. + mvn $MAVEN_GOALS -pl "$module" -am -P "$MAVEN_PROFILE" -Dmaven.test.skip=true $MAVEN_EXTRA_ARGS + fi + + ensure_base_image + log "docker build $image" + docker build \ + --build-arg "SERVICE_VERSION=${IMAGE_PROJECT}:${tag}" \ + -f "$dir/Dockerfile" \ + -t "$image" \ + "$dir" + docker tag "$image" "${repo}:latest" + + case "$MODE" in + build-only) + log "build-only completed for $svc: $image" + ;; + push) + push_image "$svc" "$image" "$repo" + rollout "$svc" "$image" + ;; + preload) + ensure_preload_pull_policy "$svc" + preload_image_to_nodes "$svc" "$image" + rollout "$svc" "$image" + ;; + *) + echo "unsupported MODE: $MODE" >&2 + exit 2 + ;; + esac +} + +push_image() { + local svc="$1" + local image="$2" + local repo="$3" + if [[ -n "${ALIYUN_USER:-}" && -n "${ALIYUN_PASS:-}" ]]; then + log "docker login registry for $svc" + printf '%s' "$ALIYUN_PASS" | docker login --username "$ALIYUN_USER" --password-stdin tuokemi-con-registry.ap-southeast-1.cr.aliyuncs.com + fi + log "docker push $image" + docker push "$image" + docker push "${repo}:latest" +} + +preload_image_to_nodes() { + local svc="$1" + local image="$2" + local safe_tag="${image//[^A-Za-z0-9_.-]/-}" + local tar_path="$BASE_DIR/artifacts/${safe_tag}.tar" + mkdir -p "$BASE_DIR/artifacts" + log "docker save $image -> $tar_path" + docker save "$image" -o "$tar_path" + + mapfile -t nodes < <(kubectl --kubeconfig "$KUBECONFIG" get nodes -o jsonpath='{range .items[*]}{.metadata.name}{"\n"}{end}') + if [[ "${#nodes[@]}" -eq 0 ]]; then + echo "no kubernetes nodes found" >&2 + exit 3 + fi + + for node in "${nodes[@]}"; do + preload_image_to_node "$svc" "$image" "$tar_path" "$node" + done +} + +ensure_preload_pull_policy() { + local svc="$1" + local policy + policy="$(kubectl --kubeconfig "$KUBECONFIG" -n "$NAMESPACE" get "deployment/$svc" \ + -o "jsonpath={.spec.template.spec.containers[?(@.name=='$svc')].imagePullPolicy}")" + # Preload mode imports images into node containerd; Always-pull pods would bypass that cache and fail. + if [[ "$policy" == "Always" ]]; then + echo "deployment/$svc uses imagePullPolicy=Always; preload mode requires IfNotPresent or unset" >&2 + exit 4 + fi +} + +preload_image_to_node() { + local svc="$1" + local image="$2" + local tar_path="$3" + local node="$4" + local pod="image-preload-${svc}-${node//[^a-zA-Z0-9-]/-}-$(date +%s)" + local manifest + manifest="$(mktemp)" + cat >"$manifest" </dev/null + rm -f "$manifest" + if ! kubectl --kubeconfig "$KUBECONFIG" -n "$NAMESPACE" wait --for=condition=Ready "pod/$pod" --timeout=180s >/dev/null; then + kubectl --kubeconfig "$KUBECONFIG" -n "$NAMESPACE" describe "pod/$pod" || true + kubectl --kubeconfig "$KUBECONFIG" -n "$NAMESPACE" delete pod "$pod" --ignore-not-found=true >/dev/null + return 1 + fi + log "copy image tar to $pod" + if ! kubectl --kubeconfig "$KUBECONFIG" -n "$NAMESPACE" cp "$tar_path" "$pod:/tmp/image.tar" -c preload >/dev/null; then + kubectl --kubeconfig "$KUBECONFIG" -n "$NAMESPACE" delete pod "$pod" --ignore-not-found=true >/dev/null + return 1 + fi + log "import $image into node $node" + if ! kubectl --kubeconfig "$KUBECONFIG" -n "$NAMESPACE" exec "$pod" -c preload -- \ + ctr --address /run/containerd/containerd.sock -n k8s.io images import /tmp/image.tar >/dev/null; then + kubectl --kubeconfig "$KUBECONFIG" -n "$NAMESPACE" delete pod "$pod" --ignore-not-found=true >/dev/null + return 1 + fi + kubectl --kubeconfig "$KUBECONFIG" -n "$NAMESPACE" delete pod "$pod" --ignore-not-found=true >/dev/null +} + +rollout() { + local svc="$1" + local image="$2" + log "set image deployment/$svc $svc=$image" + kubectl --kubeconfig "$KUBECONFIG" -n "$NAMESPACE" set image "deployment/$svc" "$svc=$image" + kubectl --kubeconfig "$KUBECONFIG" -n "$NAMESPACE" rollout status "deployment/$svc" --timeout=600s + kubectl --kubeconfig "$KUBECONFIG" -n "$NAMESPACE" get deploy "$svc" -o wide +} + +status() { + kubectl --kubeconfig "$KUBECONFIG" -n "$NAMESPACE" get deploy other external console -o wide + kubectl --kubeconfig "$KUBECONFIG" get nodes -o wide +} + +main() { + while [[ "$#" -gt 0 ]]; do + case "$1" in + --mode) + [[ -n "${2:-}" ]] || { echo "--mode requires an argument" >&2; exit 2; } + MODE="$2" + shift 2 + ;; + --skip-build) + SKIP_BUILD=1 + shift + ;; + -h|--help) + usage + exit 0 + ;; + *) + break + ;; + esac + done + + if [[ "${1:-}" == "status" || "$MODE" == "status" ]]; then + need_cmd kubectl + [[ -f "$KUBECONFIG" ]] || { echo "missing kubeconfig: $KUBECONFIG" >&2; exit 2; } + status + exit 0 + fi + + need_cmd java + need_cmd mvn + need_cmd docker + need_cmd kubectl + [[ -f "$KUBECONFIG" ]] || { echo "missing kubeconfig: $KUBECONFIG" >&2; exit 2; } + + acquire_deploy_lock + update_source_from_git + + local services=("$@") + if [[ "${#services[@]}" -eq 0 ]]; then + services=(other external console) + fi + + for svc in "${services[@]}"; do + build_service "$svc" + done +} + +main "$@" diff --git a/.deploy/test-deploy/ops/aslan-test-ops.service b/.deploy/test-deploy/ops/aslan-test-ops.service new file mode 100644 index 00000000..77ae9ad6 --- /dev/null +++ b/.deploy/test-deploy/ops/aslan-test-ops.service @@ -0,0 +1,21 @@ +[Unit] +Description=Aslan test deployment operations panel +After=network-online.target docker.service +Wants=network-online.target + +[Service] +Type=simple +User=ubuntu +Group=ubuntu +WorkingDirectory=/opt/aslan-test-deploy +Environment=PATH=/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin +Environment=HOME=/home/ubuntu +Environment=ASLAN_ALLOWED_BRANCHES=aslan_test +EnvironmentFile=/etc/aslan-test-ops.env +ExecStart=/usr/bin/python3 /opt/aslan-test-deploy/ops/aslan_ops.py +Restart=always +RestartSec=5 +NoNewPrivileges=true + +[Install] +WantedBy=multi-user.target diff --git a/.deploy/test-deploy/ops/aslan_ops.py b/.deploy/test-deploy/ops/aslan_ops.py new file mode 100644 index 00000000..a07ef7df --- /dev/null +++ b/.deploy/test-deploy/ops/aslan_ops.py @@ -0,0 +1,607 @@ +#!/usr/bin/env python3 +import base64 +import hashlib +import hmac +import html +import json +import os +import re +import secrets +import subprocess +import threading +import time +from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer +from pathlib import Path +from urllib.parse import parse_qs, urlparse + + +BASE_DIR = Path(os.environ.get("ASLAN_DEPLOY_BASE", "/opt/aslan-test-deploy")) +RUN_DIR = Path(os.environ.get("ASLAN_OPS_RUN_DIR", str(BASE_DIR / "ops" / "runs"))) +KUBECONFIG = Path(os.environ.get("KUBECONFIG", str(BASE_DIR / "kubeconfig"))) +DEPLOY_SCRIPT = Path(os.environ.get("ASLAN_DEPLOY_SCRIPT", str(BASE_DIR / "deploy-likei-services.sh"))) +SOURCE_DIR = Path(os.environ.get("ASLAN_SOURCE_DIR", str(BASE_DIR / "source" / "likei-services"))) +NAMESPACE = os.environ.get("ASLAN_NAMESPACE", "test") +GIT_REF = os.environ.get("ASLAN_GIT_REF", "aslan_test") +REMOTE_URL = os.environ.get("ASLAN_GIT_REMOTE_URL", "git@gitea.haiyihy.com:hy/aslan-server.git") +HOST = os.environ.get("ASLAN_OPS_HOST", "0.0.0.0") +PORT = int(os.environ.get("ASLAN_OPS_PORT", "18081")) +USERNAME = os.environ.get("ASLAN_OPS_USER", "admin") +PASSWORD = os.environ.get("ASLAN_OPS_PASSWORD", "") +SESSION_SECRET = os.environ.get("ASLAN_OPS_SESSION_SECRET", PASSWORD or secrets.token_hex(32)) +ALLOWED_BRANCHES = { + branch.strip() + for branch in os.environ.get("ASLAN_ALLOWED_BRANCHES", GIT_REF).split(",") + if branch.strip() +} + +SERVICES = { + "other": { + "label": "Other", + "port": "5800", + "health": "/actuator/health", + }, + "external": { + "label": "External", + "port": "5200", + "health": "/actuator/health", + }, + "console": { + "label": "Console", + "port": "5300", + "health": "/console/actuator/health", + }, +} + +BRANCH_RE = re.compile(r"^[A-Za-z0-9._/-]{1,80}$") + +job_lock = threading.Lock() +active_job = None +login_failures = {} + + +def now_ts(): + return time.strftime("%Y-%m-%d %H:%M:%S") + + +def json_response(handler, status, payload): + body = json.dumps(payload, ensure_ascii=False).encode("utf-8") + handler.send_response(status) + handler.send_header("Content-Type", "application/json; charset=utf-8") + handler.send_header("Cache-Control", "no-store") + handler.send_header("X-Frame-Options", "DENY") + handler.send_header("Content-Length", str(len(body))) + handler.end_headers() + handler.wfile.write(body) + + +def text_response(handler, status, body, content_type="text/plain; charset=utf-8"): + data = body.encode("utf-8") + handler.send_response(status) + handler.send_header("Content-Type", content_type) + handler.send_header("Cache-Control", "no-store") + handler.send_header("X-Frame-Options", "DENY") + handler.send_header("Content-Length", str(len(data))) + handler.end_headers() + handler.wfile.write(data) + + +def redirect_response(handler, location, cookies=None): + handler.send_response(302) + for cookie in cookies or []: + handler.send_header("Set-Cookie", cookie) + handler.send_header("Location", location) + handler.send_header("Content-Length", "0") + handler.end_headers() + + +def sign_session(value): + mac = hmac.new(SESSION_SECRET.encode("utf-8"), value.encode("utf-8"), hashlib.sha256).hexdigest() + return f"{value}.{mac}" + + +def verify_session(signed): + if not signed or "." not in signed: + return False + value, mac = signed.rsplit(".", 1) + expected = hmac.new(SESSION_SECRET.encode("utf-8"), value.encode("utf-8"), hashlib.sha256).hexdigest() + if not hmac.compare_digest(mac, expected): + return False + try: + username, issued = value.split(":", 1) + return username == USERNAME and time.time() - int(issued) < 86400 + except ValueError: + return False + + +def parse_cookies(header): + cookies = {} + for part in (header or "").split(";"): + if "=" not in part: + continue + key, value = part.strip().split("=", 1) + cookies[key] = value + return cookies + + +def too_many_login_failures(ip): + now = time.time() + window = [ts for ts in login_failures.get(ip, []) if now - ts < 300] + login_failures[ip] = window + return len(window) >= 5 + + +def record_login_failure(ip): + window = login_failures.get(ip, []) + window.append(time.time()) + login_failures[ip] = window[-10:] + + +def run_json(cmd, timeout=20): + result = subprocess.run(cmd, text=True, capture_output=True, timeout=timeout) + if result.returncode != 0: + raise RuntimeError(result.stderr.strip() or result.stdout.strip() or f"command failed: {cmd[0]}") + return json.loads(result.stdout) + + +def run_text(cmd, timeout=10): + result = subprocess.run(cmd, text=True, capture_output=True, timeout=timeout) + if result.returncode != 0: + return "" + return result.stdout.strip() + + +def deployment_status(): + names = list(SERVICES) + payload = run_json([ + "kubectl", + "--kubeconfig", + str(KUBECONFIG), + "-n", + NAMESPACE, + "get", + "deploy", + *names, + "-o", + "json", + ]) + items = [] + for item in payload.get("items", []): + name = item["metadata"]["name"] + spec = item.get("spec", {}) + status = item.get("status", {}) + containers = item.get("spec", {}).get("template", {}).get("spec", {}).get("containers", []) + image = next((c.get("image", "") for c in containers if c.get("name") == name), containers[0].get("image", "") if containers else "") + conditions = {c.get("type"): c.get("status") for c in status.get("conditions", [])} + items.append({ + "name": name, + "label": SERVICES.get(name, {}).get("label", name), + "ready": f"{status.get('readyReplicas', 0)}/{spec.get('replicas', 0)}", + "updated": status.get("updatedReplicas", 0), + "available": status.get("availableReplicas", 0), + "image": image, + "port": SERVICES.get(name, {}).get("port", ""), + "health": SERVICES.get(name, {}).get("health", ""), + "healthy": conditions.get("Available") == "True", + }) + order = {name: idx for idx, name in enumerate(names)} + return sorted(items, key=lambda x: order.get(x["name"], 99)) + + +def pod_status(): + selector = "app in (other,external,console)" + payload = run_json([ + "kubectl", + "--kubeconfig", + str(KUBECONFIG), + "-n", + NAMESPACE, + "get", + "pods", + "-l", + selector, + "-o", + "json", + ]) + pods = [] + for item in payload.get("items", []): + status = item.get("status", {}) + containers = status.get("containerStatuses", []) + ready_count = sum(1 for c in containers if c.get("ready")) + pods.append({ + "name": item["metadata"]["name"], + "app": item["metadata"].get("labels", {}).get("app", ""), + "ready": f"{ready_count}/{len(containers)}", + "phase": status.get("phase", ""), + "restarts": sum(c.get("restartCount", 0) for c in containers), + "node": item.get("spec", {}).get("nodeName", ""), + "ip": status.get("podIP", ""), + "age": item["metadata"].get("creationTimestamp", ""), + }) + return pods + + +def git_status(): + head = run_text(["git", "-C", str(SOURCE_DIR), "rev-parse", "HEAD"]) + branch = run_text(["git", "-C", str(SOURCE_DIR), "branch", "--show-current"]) + short = run_text(["git", "-C", str(SOURCE_DIR), "status", "--short", "--branch"]) + return {"head": head, "branch": branch, "status": short} + + +def list_jobs(): + RUN_DIR.mkdir(parents=True, exist_ok=True) + jobs = [] + for meta_path in sorted(RUN_DIR.glob("*.json"), key=lambda p: p.stat().st_mtime, reverse=True)[:20]: + try: + jobs.append(json.loads(meta_path.read_text())) + except json.JSONDecodeError: + continue + return jobs + + +def save_job(job): + RUN_DIR.mkdir(parents=True, exist_ok=True) + (RUN_DIR / f"{job['id']}.json").write_text(json.dumps(job, ensure_ascii=False, indent=2)) + + +def stream_process(job, cmd, env): + log_path = RUN_DIR / f"{job['id']}.log" + try: + with log_path.open("a", buffering=1) as log: + log.write(f"[{now_ts()}] start {' '.join(cmd)}\n") + process = subprocess.Popen( + cmd, + cwd=str(BASE_DIR), + env=env, + stdout=subprocess.PIPE, + stderr=subprocess.STDOUT, + text=True, + bufsize=1, + ) + job["pid"] = process.pid + job["status"] = "running" + save_job(job) + for line in process.stdout: + log.write(line) + rc = process.wait() + job["status"] = "success" if rc == 0 else "failed" + job["returnCode"] = rc + job["finishedAt"] = now_ts() + save_job(job) + log.write(f"[{now_ts()}] finished rc={rc}\n") + except Exception as exc: + job["status"] = "failed" + job["returnCode"] = -1 + job["finishedAt"] = now_ts() + job["error"] = str(exc) + save_job(job) + with log_path.open("a", buffering=1) as log: + log.write(f"[{now_ts()}] runner error: {exc}\n") + finally: + global active_job + with job_lock: + active_job = None + + +def start_deploy(services, branch, fast): + global active_job + invalid = [svc for svc in services if svc not in SERVICES] + if invalid: + raise ValueError(f"unsupported services: {', '.join(invalid)}") + if not services: + raise ValueError("select at least one service") + if not BRANCH_RE.match(branch): + raise ValueError("invalid branch name") + if branch not in ALLOWED_BRANCHES: + raise ValueError(f"branch is not allowed: {branch}") + if not DEPLOY_SCRIPT.exists(): + raise ValueError(f"missing deploy script: {DEPLOY_SCRIPT}") + + with job_lock: + if active_job: + raise RuntimeError(f"deploy already running: {active_job['id']}") + job_id = time.strftime("%Y%m%d%H%M%S") + "-" + secrets.token_hex(3) + job = { + "id": job_id, + "services": services, + "branch": branch, + "fast": fast, + "status": "queued", + "createdAt": now_ts(), + "finishedAt": "", + "returnCode": None, + } + active_job = job + save_job(job) + + env = os.environ.copy() + env.update({ + "USE_GIT_SOURCE": "1", + "GIT_REF": branch, + "GIT_REMOTE_URL": REMOTE_URL, + "MODE": "preload", + "KUBECONFIG": str(KUBECONFIG), + "NAMESPACE": NAMESPACE, + "ASLAN_NAMESPACE": NAMESPACE, + "HOME": str(Path.home()), + }) + if fast: + # Fast mode avoids Maven clean so unchanged modules and already downloaded dependencies can reuse local target output. + env["MAVEN_GOALS"] = "package" + cmd = [str(DEPLOY_SCRIPT), "--mode", "preload", *services] + thread = threading.Thread(target=stream_process, args=(job, cmd, env), daemon=True) + thread.start() + return job + + +def login_page(error=""): + message = f"
{html.escape(error)}
" if error else "" + return f""" + + + + + Aslan Ops + + +

Aslan Ops

{message}
+""" + + +INDEX_HTML = """ + + + + + Aslan Ops + + + +
+

Aslan Ops

+ +
+
+
+

微服务

+
+
+
+

部署

固定 preload 到 TKE 节点,不走 registry push
+
+
+ + + +
+
+
+
+

部署日志

无运行任务
+

+      
+
+

Pod

+
名称状态节点
+
+
+
+

最近任务

+
ID服务分支状态时间
+
+
+ + +""" + + +class Handler(BaseHTTPRequestHandler): + server_version = "AslanOps/1.0" + + def authenticated(self): + cookies = parse_cookies(self.headers.get("Cookie")) + return verify_session(cookies.get("aslan_ops")) + + def require_auth(self): + if self.authenticated(): + return True + redirect_response(self, "/login") + return False + + def read_body(self, max_bytes=65536): + size = int(self.headers.get("Content-Length", "0")) + if size > max_bytes: + raise ValueError("request body too large") + return self.rfile.read(size).decode("utf-8") + + def do_GET(self): + parsed = urlparse(self.path) + if parsed.path == "/login": + return text_response(self, 200, login_page(), "text/html; charset=utf-8") + if parsed.path == "/logout": + redirect_response(self, "/login", ["aslan_ops=; Max-Age=0; HttpOnly; SameSite=Lax; Path=/"]) + return + if not self.require_auth(): + return + if parsed.path == "/": + return text_response(self, 200, INDEX_HTML, "text/html; charset=utf-8") + if parsed.path == "/api/status": + try: + return json_response(self, 200, { + "deployments": deployment_status(), + "pods": pod_status(), + "git": git_status(), + "jobs": list_jobs(), + }) + except Exception as exc: + return json_response(self, 500, {"error": str(exc)}) + match = re.match(r"^/api/jobs/([A-Za-z0-9-]+)/log$", parsed.path) + if match: + job_id = match.group(1) + meta_path = RUN_DIR / f"{job_id}.json" + log_path = RUN_DIR / f"{job_id}.log" + if not meta_path.exists(): + return json_response(self, 404, {"error": "job not found"}) + job = json.loads(meta_path.read_text()) + log = log_path.read_text(errors="replace")[-120000:] if log_path.exists() else "" + return json_response(self, 200, {"job": job, "log": log}) + return text_response(self, 404, "not found") + + def do_POST(self): + parsed = urlparse(self.path) + if parsed.path == "/login": + if too_many_login_failures(self.client_address[0]): + return text_response(self, 429, login_page("登录失败次数过多,稍后再试"), "text/html; charset=utf-8") + try: + form = parse_qs(self.read_body(max_bytes=4096)) + except ValueError as exc: + return text_response(self, 413, str(exc)) + username = form.get("username", [""])[0] + password = form.get("password", [""])[0] + if hmac.compare_digest(username, USERNAME) and hmac.compare_digest(password, PASSWORD): + login_failures.pop(self.client_address[0], None) + session = sign_session(f"{USERNAME}:{int(time.time())}") + redirect_response(self, "/", [f"aslan_ops={session}; HttpOnly; SameSite=Lax; Path=/"]) + return + record_login_failure(self.client_address[0]) + return text_response(self, 401, login_page("账号或密码错误"), "text/html; charset=utf-8") + if not self.require_auth(): + return + if parsed.path == "/api/deploy": + try: + if self.headers.get("X-Aslan-Ops") != "1": + return json_response(self, 403, {"error": "missing deploy header"}) + payload = json.loads(self.read_body(max_bytes=4096) or "{}") + services = payload.get("services") or [] + branch = payload.get("branch") or GIT_REF + fast = bool(payload.get("fast", True)) + job = start_deploy(services, branch, fast) + return json_response(self, 202, {"job": job}) + except Exception as exc: + return json_response(self, 400, {"error": str(exc)}) + return text_response(self, 404, "not found") + + def log_message(self, fmt, *args): + print(f"[{now_ts()}] {self.address_string()} {fmt % args}", flush=True) + + +def main(): + if not PASSWORD: + raise SystemExit("ASLAN_OPS_PASSWORD is required") + RUN_DIR.mkdir(parents=True, exist_ok=True) + server = ThreadingHTTPServer((HOST, PORT), Handler) + print(f"aslan ops listening on {HOST}:{PORT}", flush=True) + server.serve_forever() + + +if __name__ == "__main__": + main() diff --git a/.deploy/test-deploy/sync-and-deploy.sh b/.deploy/test-deploy/sync-and-deploy.sh new file mode 100755 index 00000000..59357ee4 --- /dev/null +++ b/.deploy/test-deploy/sync-and-deploy.sh @@ -0,0 +1,84 @@ +#!/usr/bin/env bash +set -euo pipefail + +DEPLOY_HOST="${DEPLOY_HOST:-43.160.219.15}" +DEPLOY_USER="${DEPLOY_USER:-ubuntu}" +SSH_KEY="${SSH_KEY:-$HOME/.ssh/aslan-deploy-sg-ed25519}" +REMOTE_BASE="${REMOTE_BASE:-/opt/aslan-test-deploy}" +REMOTE_SRC="$REMOTE_BASE/source/likei-services" +LOCAL_M2_ROOT="${LOCAL_M2_ROOT:-$HOME/.m2/repository}" +REMOTE_M2_ROOT="${REMOTE_M2_ROOT:-/home/$DEPLOY_USER/.m2/repository}" +M2_CACHE_GROUPS="${M2_CACHE_GROUPS:-com/red/circle com/github/sud}" +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +REPO_ROOT="$(cd "$SCRIPT_DIR/../.." && pwd)" + +usage() { + cat <<'EOF' +Usage: + .deploy/test-deploy/sync-and-deploy.sh [remote deploy args...] + +Examples: + .deploy/test-deploy/sync-and-deploy.sh status + .deploy/test-deploy/sync-and-deploy.sh --mode preload other + .deploy/test-deploy/sync-and-deploy.sh --mode preload other external console + +Environment: + DEPLOY_HOST default 43.160.219.15 + DEPLOY_USER default ubuntu + SSH_KEY default ~/.ssh/aslan-deploy-sg-ed25519 + +Git source on deploy host: + ssh -i ~/.ssh/aslan-deploy-sg-ed25519 ubuntu@43.160.219.15 \ + 'USE_GIT_SOURCE=1 GIT_REF=aslan_test /opt/aslan-test-deploy/deploy-likei-services.sh --mode preload other' +EOF +} + +if [[ "${1:-}" == "-h" || "${1:-}" == "--help" ]]; then + usage + exit 0 +fi + +ssh_base=(ssh -i "$SSH_KEY" -o StrictHostKeyChecking=accept-new "$DEPLOY_USER@$DEPLOY_HOST") +rsync_ssh="ssh -i $SSH_KEY -o StrictHostKeyChecking=accept-new" + +"${ssh_base[@]}" "mkdir -p '$REMOTE_SRC' '$REMOTE_BASE'" + +rsync -az --delete \ + --exclude '.git/' \ + --exclude 'target/' \ + --exclude '**/target/' \ + --exclude '.idea/' \ + --exclude '.DS_Store' \ + --exclude 'build.md' \ + --exclude 'node_modules/' \ + -e "$rsync_ssh" \ + "$REPO_ROOT/" "$DEPLOY_USER@$DEPLOY_HOST:$REMOTE_SRC/" + +rsync -az \ + -e "$rsync_ssh" \ + "$SCRIPT_DIR/deploy-likei-services.sh" "$DEPLOY_USER@$DEPLOY_HOST:$REMOTE_BASE/deploy-likei-services.sh" + +rsync -az --delete \ + -e "$rsync_ssh" \ + "$SCRIPT_DIR/ops/" "$DEPLOY_USER@$DEPLOY_HOST:$REMOTE_BASE/ops/" + +for cache_group in $M2_CACHE_GROUPS; do + local_cache_path="$LOCAL_M2_ROOT/$cache_group" + remote_cache_parent="$REMOTE_M2_ROOT/$(dirname "$cache_group")" + if [[ ! -d "$local_cache_path" ]]; then + continue + fi + "${ssh_base[@]}" "mkdir -p '$remote_cache_parent'" + rsync -az --delete \ + --exclude '*.lastUpdated' \ + -e "$rsync_ssh" \ + "$local_cache_path" "$DEPLOY_USER@$DEPLOY_HOST:$remote_cache_parent/" +done + +printf -v remote_script_q '%q' "$REMOTE_BASE/deploy-likei-services.sh" +remote_cmd="$remote_script_q" +if [[ "$#" -gt 0 ]]; then + printf -v remote_args_q ' %q' "$@" + remote_cmd+="$remote_args_q" +fi +"${ssh_base[@]}" "chmod +x $remote_script_q && $remote_cmd"