From 8722d6cf7df951fd8533e802c2436391bb590b31 Mon Sep 17 00:00:00 2001 From: zhx Date: Sat, 9 May 2026 14:28:09 +0800 Subject: [PATCH] Add testbox auto deploy polling --- testbox/.env.example | 14 +- testbox/README.md | 47 ++- testbox/scripts/auto-pull-restart.sh | 280 ++++++++++++++++++ testbox/scripts/deploy-from-git-remote.sh | 3 +- .../scripts/install-auto-deploy-service.sh | 31 ++ .../systemd/chatapp-test-auto-deploy.service | 15 + 6 files changed, 384 insertions(+), 6 deletions(-) create mode 100755 testbox/scripts/auto-pull-restart.sh create mode 100755 testbox/scripts/install-auto-deploy-service.sh create mode 100644 testbox/systemd/chatapp-test-auto-deploy.service diff --git a/testbox/.env.example b/testbox/.env.example index a4ec17a..431805b 100644 --- a/testbox/.env.example +++ b/testbox/.env.example @@ -240,15 +240,23 @@ CHATAPP_WEEK_STAR_ROCKETMQ_ACCESS_SECRET=test TESTBOX_MONITOR_REPO_URL=git@gitea.haiyihy.com:hy/hy-app-monitor.git TESTBOX_MONITOR_BRANCH=main JAVA_REPO_URL=git@gitea.haiyihy.com:hy/chatapp3-java.git -JAVA_BRANCH=main +JAVA_BRANCH=test GOLANG_REPO_URL=git@gitea.haiyihy.com:hy/chatapp3-golang.git -GOLANG_BRANCH=main +GOLANG_BRANCH=test CRON_REPO_URL=git@gitea.haiyihy.com:hy/chatapp-cron.git CRON_BRANCH=main ADMIN_REPO_URL=git@gitea.haiyihy.com:hy/chatapp3-admin.git -ADMIN_BRANCH=main +ADMIN_BRANCH=test TESTBOX_SRC_ROOT=/opt/chatapp-test/sources JAVA_SOURCE_DIR=/opt/chatapp-test/sources/chatapp3-java GOLANG_SOURCE_DIR=/opt/chatapp-test/sources/chatapp3-golang CRON_SOURCE_DIR=/opt/chatapp-test/sources/chatapp-cron ADMIN_SOURCE_DIR=/opt/chatapp-test/sources/chatapp3-admin + +# Testbox auto deploy. The systemd service polls these branches every 30s and +# rebuilds/restarts only changed admin/golang/java services. +TESTBOX_AUTODEPLOY_ENABLED=1 +TESTBOX_AUTODEPLOY_INTERVAL_SECONDS=30 +TESTBOX_AUTODEPLOY_JAVA_BRANCH=test +TESTBOX_AUTODEPLOY_GOLANG_BRANCH=test +TESTBOX_AUTODEPLOY_ADMIN_BRANCH=test diff --git a/testbox/README.md b/testbox/README.md index f69674f..6ced94e 100644 --- a/testbox/README.md +++ b/testbox/README.md @@ -58,6 +58,49 @@ ssh root@43.165.195.39 'bash /opt/chatapp-test/scripts/deploy-from-git-remote.sh The runtime `.env` stays on the server at `/opt/chatapp-test/.env`; git deploys do not overwrite it. +## 自动拉取 test 分支 + +测试机安装后会启用 `chatapp-test-auto-deploy.service`。它每 30 秒读取一次 +`admin`、`chatapp3-java`、`chatapp3-golang` 的 `test` 分支，如果远端 commit +有变化，就只构建并重启变化对应的服务。 + +默认配置在 `/opt/chatapp-test/.env`： + +```text +TESTBOX_AUTODEPLOY_ENABLED=1 +TESTBOX_AUTODEPLOY_INTERVAL_SECONDS=30 +TESTBOX_AUTODEPLOY_JAVA_BRANCH=test +TESTBOX_AUTODEPLOY_GOLANG_BRANCH=test +TESTBOX_AUTODEPLOY_ADMIN_BRANCH=test +``` + +Java 会按变更路径细分服务： + +```text +rc-auth -> auth +rc-gateway -> gateway +rc-service/rc-service-external -> external +rc-service/rc-service-console -> console +rc-service/rc-service-other -> other +rc-service/rc-service-live -> live +rc-service/rc-service-wallet -> wallet +rc-service/rc-service-order -> order +nacos_config/ -> 对应服务 +nacos_config/common、公共模块、父 pom、未知路径 -> 全部 Java 服务 +``` + +查看自动部署日志： + +```bash +ssh root@43.165.195.39 'tail -f /opt/chatapp-test/logs/auto-deploy.log' +``` + +手动跑一轮： + +```bash +ssh root@43.165.195.39 'bash /opt/chatapp-test/scripts/auto-pull-restart.sh --once' +``` + ## 新同学日常部署 场景：你改了某个 Java 服务或 `chatapp3-admin`，想在测试环境看效果。 @@ -73,7 +116,7 @@ cd /path/to/chatapp3-java git status git add git commit -m "your change message" -git push origin main +git push origin test ``` Admin 前端： @@ -83,7 +126,7 @@ cd /path/to/chatapp3-admin git status git add git commit -m "your change message" -git push origin main +git push origin test ``` 如果同时改了部署脚本、Caddy、测试环境配置模板，则还需要推 `hy-app-monitor`： diff --git a/testbox/scripts/auto-pull-restart.sh b/testbox/scripts/auto-pull-restart.sh new file mode 100755 index 0000000..319ebc8 --- /dev/null +++ b/testbox/scripts/auto-pull-restart.sh @@ -0,0 +1,280 @@ +#!/usr/bin/env bash +set -euo pipefail + +ROOT_DIR="${ROOT_DIR:-$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)}" +cd "$ROOT_DIR" + +LOG_FILE="${TESTBOX_AUTODEPLOY_LOG_FILE:-$ROOT_DIR/logs/auto-deploy.log}" +LOCK_FILE="${TESTBOX_DEPLOY_LOCK_FILE:-$ROOT_DIR/run/deploy.lock}" + +mkdir -p "$ROOT_DIR/logs" "$ROOT_DIR/run" + +env_value() { + python3 "$ROOT_DIR/scripts/env-value.py" "$ROOT_DIR/.env" "$1" "${2:-}" +} + +log() { + printf '[%s] %s\n' "$(date '+%Y-%m-%d %H:%M:%S')" "$*" | tee -a "$LOG_FILE" +} + +compose() { + docker compose --env-file .env -f docker-compose.infra.yml -f docker-compose.apps.yml "$@" +} + +repo_changed() { + local dir="$1" + local url="$2" + local branch="$3" + local label="$4" + local before="" + local after="" + + mkdir -p "$(dirname "$dir")" + + if [ -e "$dir" ] && [ ! -d "$dir/.git" ]; then + mv "$dir" "$dir.pre-auto-git-$(date +%Y%m%d%H%M%S)" + fi + + if [ ! -d "$dir/.git" ]; then + log "$label clone branch=$branch" + git clone --branch "$branch" "$url" "$dir" + git -C "$dir" rev-parse HEAD >"$ROOT_DIR/run/$label.after" + return 0 + fi + + before="$(git -C "$dir" rev-parse HEAD 2>/dev/null || true)" + if ! git -C "$dir" fetch --prune origin "$branch" >>"$LOG_FILE" 2>&1; then + log "$label fetch failed branch=$branch" + return 2 + fi + git -C "$dir" checkout -B "$branch" "origin/$branch" >>"$LOG_FILE" 2>&1 + git -C "$dir" reset --hard "origin/$branch" >>"$LOG_FILE" 2>&1 + after="$(git -C "$dir" rev-parse HEAD)" + + printf '%s\n' "$before" >"$ROOT_DIR/run/$label.before" + printf '%s\n' "$after" >"$ROOT_DIR/run/$label.after" + + if [ "$before" != "$after" ]; then + log "$label changed ${before:-none} -> $after" + return 0 + fi + return 10 +} + +unique_words() { + tr ' ' '\n' | awk 'NF && !seen[$0]++' | tr '\n' ' ' | sed 's/[[:space:]]*$//' +} + +all_java_services() { + printf '%s\n' "auth gateway external console other live wallet order" +} + +java_module_for_service() { + case "$1" in + auth) echo "rc-auth" ;; + gateway) echo "rc-gateway" ;; + external) echo "rc-service/rc-service-external/external-start" ;; + console) echo "rc-service/rc-service-console/console-start" ;; + other) echo "rc-service/rc-service-other/other-start" ;; + live) echo "rc-service/rc-service-live/live-start" ;; + wallet) echo "rc-service/rc-service-wallet/wallet-start" ;; + order) echo "rc-service/rc-service-order/order-start" ;; + *) return 1 ;; + esac +} + +service_for_java_path() { + case "$1" in + rc-auth/*|nacos_config/rc-auth/*) echo "auth" ;; + rc-gateway/*|nacos_config/rc-gateway/*) echo "gateway" ;; + rc-service/rc-service-external/*|nacos_config/rc-service-external/*) echo "external" ;; + rc-service/rc-service-console/*|nacos_config/rc-service-console/*) echo "console" ;; + rc-service/rc-service-other/*|nacos_config/rc-service-other/*) echo "other" ;; + rc-service/rc-service-live/*|nacos_config/rc-service-live/*) echo "live" ;; + rc-service/rc-service-wallet/*|nacos_config/rc-service-wallet/*) echo "wallet" ;; + rc-service/rc-service-order/*|nacos_config/rc-service-order/*) echo "order" ;; + nacos_config/common/*|pom.xml|*/pom.xml|rc-common*/*|rc-service/rc-inner-api/*|ci/*) + all_java_services + ;; + sql/*) + ;; + *) + all_java_services + ;; + esac +} + +java_changed_services() { + local before="$1" + local after="$2" + local java_dir="$3" + local changed_paths + local services="" + local path + + if [ -z "$before" ]; then + all_java_services + return 0 + fi + + changed_paths="$(git -C "$java_dir" diff --name-only "$before" "$after" || true)" + while IFS= read -r path; do + [ -n "$path" ] || continue + services="$services $(service_for_java_path "$path" | tr '\n' ' ')" + done <<< "$changed_paths" + + printf '%s\n' "$services" | unique_words +} + +build_and_restart_java() { + local java_dir="$1" + shift + local services=("$@") + local modules="" + local compose_services="" + local service + + [ "${#services[@]}" -gt 0 ] || return 0 + + for service in "${services[@]}"; do + modules="${modules:+$modules,}$(java_module_for_service "$service")" + compose_services="$compose_services $service" + done + + log "java build services=${services[*]} modules=$modules" + cd "$java_dir" + if [ -x ./ci/install-private-maven.sh ]; then + ./ci/install-private-maven.sh >>"$LOG_FILE" 2>&1 + fi + mvn -Ptest -T 1C -pl "$modules" -am -Dmaven.test.skip=true package >>"$LOG_FILE" 2>&1 + + cd "$ROOT_DIR" + python3 scripts/import-nacos-configs.py >>"$LOG_FILE" 2>&1 + TESTBOX_APPLY_JAVA_SQL=1 scripts/apply-mysql-baseline.sh >>"$LOG_FILE" 2>&1 + compose build "${services[@]}" >>"$LOG_FILE" 2>&1 + for service in "${services[@]}"; do + log "java restart service=$service" + compose up -d --no-deps --force-recreate "$service" >>"$LOG_FILE" 2>&1 + done + compose ps "${services[@]}" | tee -a "$LOG_FILE" +} + +build_and_restart_golang() { + log "golang build/restart" + scripts/apply-mysql-baseline.sh >>"$LOG_FILE" 2>&1 + compose build golang >>"$LOG_FILE" 2>&1 + compose up -d --no-deps --force-recreate golang >>"$LOG_FILE" 2>&1 + compose ps golang | tee -a "$LOG_FILE" +} + +build_and_restart_admin() { + local admin_dir="$1" + local admin_api_base_url + local admin_oss_url + + admin_api_base_url="$(env_value TESTBOX_ADMIN_API_BASE_URL "https://yumi-test-admin.haiyihy.com/console")" + admin_oss_url="$(env_value TESTBOX_ADMIN_OSS_URL "")" + + log "admin build/restart" + docker build \ + -f "$ROOT_DIR/docker/chatapp3-admin.Dockerfile" \ + --build-arg "ADMIN_API_BASE_URL=$admin_api_base_url" \ + --build-arg "ADMIN_OSS_URL=$admin_oss_url" \ + -t chatapp-test/admin:testbox \ + "$admin_dir" >>"$LOG_FILE" 2>&1 + compose up -d --no-deps --force-recreate admin >>"$LOG_FILE" 2>&1 + compose ps admin | tee -a "$LOG_FILE" +} + +run_once() { + local java_repo_url golang_repo_url admin_repo_url + local java_branch golang_branch admin_branch + local java_dir golang_dir admin_dir + local rc + local java_before java_after + local services_line + + exec 9>"$LOCK_FILE" + if ! flock -n 9; then + log "deploy lock busy; skip this tick" + return 0 + fi + + java_repo_url="$(env_value JAVA_REPO_URL git@gitea.haiyihy.com:hy/chatapp3-java.git)" + golang_repo_url="$(env_value GOLANG_REPO_URL git@gitea.haiyihy.com:hy/chatapp3-golang.git)" + admin_repo_url="$(env_value ADMIN_REPO_URL git@gitea.haiyihy.com:hy/chatapp3-admin.git)" + java_branch="$(env_value TESTBOX_AUTODEPLOY_JAVA_BRANCH test)" + golang_branch="$(env_value TESTBOX_AUTODEPLOY_GOLANG_BRANCH test)" + admin_branch="$(env_value TESTBOX_AUTODEPLOY_ADMIN_BRANCH test)" + java_dir="$(env_value JAVA_SOURCE_DIR "$ROOT_DIR/sources/chatapp3-java")" + golang_dir="$(env_value GOLANG_SOURCE_DIR "$ROOT_DIR/sources/chatapp3-golang")" + admin_dir="$(env_value ADMIN_SOURCE_DIR "$ROOT_DIR/sources/chatapp3-admin")" + + rc=0 + if repo_changed "$java_dir" "$java_repo_url" "$java_branch" java; then + java_before="$(cat "$ROOT_DIR/run/java.before" 2>/dev/null || true)" + java_after="$(cat "$ROOT_DIR/run/java.after" 2>/dev/null || true)" + services_line="$(java_changed_services "$java_before" "$java_after" "$java_dir")" + if [ -n "$services_line" ]; then + # shellcheck disable=SC2206 + local java_services=($services_line) + build_and_restart_java "$java_dir" "${java_services[@]}" + else + log "java changed without service rebuild target; applying SQL/Nacos only" + python3 scripts/import-nacos-configs.py >>"$LOG_FILE" 2>&1 + TESTBOX_APPLY_JAVA_SQL=1 scripts/apply-mysql-baseline.sh >>"$LOG_FILE" 2>&1 + fi + else + rc=$? + [ "$rc" -eq 10 ] || log "java update skipped due to error rc=$rc" + fi + + rc=0 + if repo_changed "$golang_dir" "$golang_repo_url" "$golang_branch" golang; then + build_and_restart_golang + else + rc=$? + [ "$rc" -eq 10 ] || log "golang update skipped due to error rc=$rc" + fi + + rc=0 + if repo_changed "$admin_dir" "$admin_repo_url" "$admin_branch" admin; then + build_and_restart_admin "$admin_dir" + else + rc=$? + [ "$rc" -eq 10 ] || log "admin update skipped due to error rc=$rc" + fi +} + +usage() { + cat <<'EOF' +Usage: + auto-pull-restart.sh # loop forever + auto-pull-restart.sh --once # run one poll cycle + +Reads TESTBOX_AUTODEPLOY_* from /opt/chatapp-test/.env. +EOF +} + +case "${1:-}" in + --once) + run_once + ;; + -h|--help) + usage + ;; + "") + interval="$(env_value TESTBOX_AUTODEPLOY_INTERVAL_SECONDS 30)" + log "auto deploy loop started interval=${interval}s" + while true; do + if ! run_once; then + log "auto deploy tick failed" + fi + sleep "$interval" + done + ;; + *) + usage >&2 + exit 2 + ;; +esac diff --git a/testbox/scripts/deploy-from-git-remote.sh b/testbox/scripts/deploy-from-git-remote.sh index 5af2109..7f25b89 100755 --- a/testbox/scripts/deploy-from-git-remote.sh +++ b/testbox/scripts/deploy-from-git-remote.sh @@ -97,4 +97,5 @@ sync_repo "$CRON_SOURCE_DIR" "$CRON_REPO_URL" "$CRON_BRANCH" sync_repo "$ADMIN_SOURCE_DIR" "$ADMIN_REPO_URL" "$ADMIN_BRANCH" cd "$REMOTE_DIR" -bash scripts/deploy-all.sh +flock "$REMOTE_DIR/run/deploy.lock" bash scripts/deploy-all.sh +bash scripts/install-auto-deploy-service.sh diff --git a/testbox/scripts/install-auto-deploy-service.sh b/testbox/scripts/install-auto-deploy-service.sh new file mode 100755 index 0000000..f25eac4 --- /dev/null +++ b/testbox/scripts/install-auto-deploy-service.sh @@ -0,0 +1,31 @@ +#!/usr/bin/env bash +set -euo pipefail + +ROOT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" +cd "$ROOT_DIR" + +if [ "$(id -u)" -ne 0 ]; then + echo "install-auto-deploy-service.sh must run as root" >&2 + exit 1 +fi + +enabled="$(python3 "$ROOT_DIR/scripts/env-value.py" "$ROOT_DIR/.env" TESTBOX_AUTODEPLOY_ENABLED 1)" +unit_src="$ROOT_DIR/systemd/chatapp-test-auto-deploy.service" +unit_dst="/etc/systemd/system/chatapp-test-auto-deploy.service" + +if [ ! -f "$unit_src" ]; then + echo "missing unit file: $unit_src" >&2 + exit 1 +fi + +install -m 0644 "$unit_src" "$unit_dst" +systemctl daemon-reload + +if [ "$enabled" = "1" ] || [ "$enabled" = "true" ] || [ "$enabled" = "yes" ]; then + systemctl enable --now chatapp-test-auto-deploy.service + systemctl restart chatapp-test-auto-deploy.service +else + systemctl disable --now chatapp-test-auto-deploy.service >/dev/null 2>&1 || true +fi + +systemctl --no-pager --full status chatapp-test-auto-deploy.service || true diff --git a/testbox/systemd/chatapp-test-auto-deploy.service b/testbox/systemd/chatapp-test-auto-deploy.service new file mode 100644 index 0000000..08fb90a --- /dev/null +++ b/testbox/systemd/chatapp-test-auto-deploy.service @@ -0,0 +1,15 @@ +[Unit] +Description=Chatapp testbox git auto deploy loop +After=network-online.target docker.service +Wants=network-online.target +Requires=docker.service + +[Service] +Type=simple +WorkingDirectory=/opt/chatapp-test +ExecStart=/bin/bash /opt/chatapp-test/scripts/auto-pull-restart.sh +Restart=always +RestartSec=5 + +[Install] +WantedBy=multi-user.target