diff --git a/backend/internal/http/router.go b/backend/internal/http/router.go index 481d422..ef35de2 100644 --- a/backend/internal/http/router.go +++ b/backend/internal/http/router.go @@ -111,6 +111,7 @@ func NewRouter(cfg config.Config, manager *service.Manager) *gin.Engine { api.POST("/releases", handler.saveRelease) api.GET("/builds", handler.listBuilds) + api.GET("/builds/:id", handler.getBuild) api.POST("/builds", handler.createBuild) api.GET("/release-runs", handler.listReleaseRuns) @@ -287,6 +288,15 @@ func (h *routerHandler) listBuilds(c *gin.Context) { c.JSON(http.StatusOK, data) } +func (h *routerHandler) getBuild(c *gin.Context) { + data, err := h.manager.GetBuild(c.Request.Context(), uintFromParam(c.Param("id"))) + if err != nil { + c.JSON(http.StatusNotFound, gin.H{"error": err.Error()}) + return + } + c.JSON(http.StatusOK, data) +} + func (h *routerHandler) createBuild(c *gin.Context) { var payload buildPayload if err := c.ShouldBindJSON(&payload); err != nil { diff --git a/backend/internal/model/models.go b/backend/internal/model/models.go index 4328b3e..ef22ef2 100644 --- a/backend/internal/model/models.go +++ b/backend/internal/model/models.go @@ -80,6 +80,7 @@ type BuildJob struct { COSKey string `gorm:"size:255" json:"cos_key"` ArtifactURL string `gorm:"size:255" json:"artifact_url"` SHA256 string `gorm:"size:128" json:"sha256"` + LogOutput string `gorm:"type:text" json:"log_output"` LogExcerpt string `gorm:"type:text" json:"log_excerpt"` ErrorMessage string `gorm:"type:text" json:"error_message"` StartedAt *time.Time `json:"started_at"` diff --git a/backend/internal/orchestrator/executor.go b/backend/internal/orchestrator/executor.go index 8d93814..e3fa0da 100644 --- a/backend/internal/orchestrator/executor.go +++ b/backend/internal/orchestrator/executor.go @@ -1,10 +1,12 @@ package orchestrator import ( + "bufio" "bytes" "context" "encoding/json" "fmt" + "io" "os/exec" "strings" @@ -25,6 +27,7 @@ type BuildContext struct { ServiceName string ReleaseID string Branch string + OnOutput func(string) } type BuildResult struct { @@ -142,11 +145,36 @@ func (e *ScriptBuildExecutor) Build(ctx context.Context, buildCtx BuildContext) command.Dir = e.workDir } - var output bytes.Buffer - command.Stdout = &output - command.Stderr = &output + stdout, err := command.StdoutPipe() + if err != nil { + return result, "", err + } + command.Stderr = command.Stdout - err := command.Run() + var output bytes.Buffer + if err := command.Start(); err != nil { + return result, "", err + } + + reader := bufio.NewReader(stdout) + for { + chunk, readErr := reader.ReadString('\n') + if chunk != "" { + output.WriteString(chunk) + if buildCtx.OnOutput != nil { + buildCtx.OnOutput(chunk) + } + } + if readErr == nil { + continue + } + if readErr == io.EOF { + break + } + return result, output.String(), readErr + } + + err = command.Wait() raw := output.String() for _, line := range strings.Split(raw, "\n") { diff --git a/backend/internal/service/manager.go b/backend/internal/service/manager.go index 1e132d5..1f47c83 100644 --- a/backend/internal/service/manager.go +++ b/backend/internal/service/manager.go @@ -157,25 +157,71 @@ func (m *Manager) ProcessBuild(ctx context.Context, buildID uint) error { if err := m.db.WithContext(ctx).Model(&model.BuildJob{}). Where("id = ?", buildID). Updates(map[string]any{ - "status": model.BuildRunning, - "started_at": &startedAt, + "status": model.BuildRunning, + "started_at": &startedAt, + "log_output": "", + "log_excerpt": "", + "error_message": "", }).Error; err != nil { return err } + var liveOutput strings.Builder + lastPersistedAt := time.Now() + lastPersistedLen := 0 + persistLiveOutput := func(force bool) { + raw := liveOutput.String() + if !force { + if len(raw) == lastPersistedLen { + return + } + if time.Since(lastPersistedAt) < time.Second && len(raw)-lastPersistedLen < 1024 { + return + } + } + if err := m.db.Model(&model.BuildJob{}). + Where("id = ?", buildID). + Updates(map[string]any{ + "log_output": raw, + "log_excerpt": trimLog(raw), + }).Error; err != nil { + log.Printf("persist build log %d: %v", buildID, err) + return + } + lastPersistedAt = time.Now() + lastPersistedLen = len(raw) + } + result, output, err := m.buildExecutor.Build(ctx, orchestrator.BuildContext{ ServiceName: buildJob.ServiceName, ReleaseID: buildJob.ReleaseID, Branch: buildJob.Branch, + OnOutput: func(chunk string) { + if chunk == "" { + return + } + liveOutput.WriteString(chunk) + persistLiveOutput(false) + }, }) + rawOutput := output + if rawOutput == "" { + rawOutput = liveOutput.String() + } + if rawOutput != "" && liveOutput.Len() == 0 { + liveOutput.WriteString(rawOutput) + } + persistLiveOutput(true) + finishedAt := time.Now() if err != nil { return m.db.WithContext(ctx).Model(&model.BuildJob{}). Where("id = ?", buildID). Updates(map[string]any{ "status": model.BuildFailed, - "log_excerpt": trimLog(output), - "error_message": err.Error(), + "log_output": rawOutput, + "log_excerpt": trimLog(rawOutput), + "error_message": summarizeBuildError(err, rawOutput), "finished_at": &finishedAt, }).Error } @@ -193,7 +239,8 @@ func (m *Manager) ProcessBuild(ctx context.Context, buildID uint) error { "cos_key": result.COSKey, "artifact_url": result.ArtifactURL, "sha256": result.SHA256, - "log_excerpt": trimLog(output), + "log_output": rawOutput, + "log_excerpt": trimLog(rawOutput), "finished_at": &finishedAt, }).Error; err != nil { return err @@ -850,6 +897,20 @@ func trimLog(raw string) string { return raw[len(raw)-maxLen:] } +func summarizeBuildError(err error, output string) string { + lines := strings.Split(output, "\n") + for i := len(lines) - 1; i >= 0; i-- { + line := strings.TrimSpace(lines[i]) + if line == "" || strings.HasPrefix(line, "BUILD_RESULT_JSON=") { + continue + } + if line != err.Error() { + return line + } + } + return err.Error() +} + func deploymentLogExcerpt(deployment *model.Deployment) string { if deployment == nil { return "" diff --git a/frontend/src/api/client.ts b/frontend/src/api/client.ts index 5cdc3fe..71c3968 100644 --- a/frontend/src/api/client.ts +++ b/frontend/src/api/client.ts @@ -30,6 +30,7 @@ export const api = { listReleases: async (serviceName?: string) => (await client.get('/releases', { params: { service_name: serviceName } })).data, saveRelease: async (payload: Partial) => (await client.post('/releases', payload)).data, listBuilds: async () => (await client.get('/builds')).data, + getBuild: async (id: number) => (await client.get(`/builds/${id}`)).data, createBuild: async (payload: Record) => (await client.post('/builds', payload)).data, listReleaseRuns: async () => (await client.get('/release-runs')).data, getReleaseRun: async (id: number) => (await client.get(`/release-runs/${id}`)).data, diff --git a/frontend/src/pages/BuildsPage.tsx b/frontend/src/pages/BuildsPage.tsx index 00558ad..3918bfe 100644 --- a/frontend/src/pages/BuildsPage.tsx +++ b/frontend/src/pages/BuildsPage.tsx @@ -1,23 +1,26 @@ import { - Alert, - Button, - Card, - CardContent, - Dialog, - DialogActions, - DialogContent, - DialogTitle, - MenuItem, - Snackbar, - Stack, - Table, - TableBody, - TableCell, - TableContainer, - TableHead, - TableRow, - TextField, - Typography + Alert, + Box, + Button, + Card, + CardContent, + Chip, + CircularProgress, + Dialog, + DialogActions, + DialogContent, + DialogTitle, + MenuItem, + Snackbar, + Stack, + Table, + TableBody, + TableCell, + TableContainer, + TableHead, + TableRow, + TextField, + Typography } from '@mui/material' import { useEffect, useMemo, useState } from 'react' @@ -26,191 +29,353 @@ import StatusChip from '@/components/StatusChip' import type { BuildJob, CatalogService } from '@/types' type FormState = { - service_name: string - release_id: string - branch: string - operator: string + service_name: string + release_id: string + branch: string + operator: string } const defaultOperator = 'platform-admin' +const activeBuildStatuses = new Set(['queued', 'running']) + +function formatDateTime(value?: string) { + if (!value) return '-' + return new Date(value).toLocaleString() +} export default function BuildsPage() { - const [builds, setBuilds] = useState([]) - const [services, setServices] = useState([]) - const [open, setOpen] = useState(false) - const [message, setMessage] = useState('') - const [form, setForm] = useState({ - service_name: '', - release_id: '', - branch: 'main', - operator: defaultOperator + const [builds, setBuilds] = useState([]) + const [services, setServices] = useState([]) + const [open, setOpen] = useState(false) + const [detailOpen, setDetailOpen] = useState(false) + const [selectedBuildID, setSelectedBuildID] = useState(null) + const [buildDetail, setBuildDetail] = useState(null) + const [detailLoading, setDetailLoading] = useState(false) + const [message, setMessage] = useState('') + const [error, setError] = useState('') + const [form, setForm] = useState({ + service_name: '', + release_id: '', + branch: 'main', + operator: defaultOperator + }) + + const selectedService = useMemo( + () => services.find((service) => service.name === form.service_name), + [form.service_name, services] + ) + + const selectedBuild = useMemo(() => { + if (buildDetail && buildDetail.id === selectedBuildID) { + return buildDetail + } + return builds.find((build) => build.id === selectedBuildID) ?? null + }, [buildDetail, builds, selectedBuildID]) + + const resetForm = (catalog: CatalogService[]) => { + const first = catalog[0] + setForm({ + service_name: first?.name ?? '', + release_id: '', + branch: first?.default_branch ?? 'main', + operator: defaultOperator }) + } - const selectedService = useMemo( - () => services.find((service) => service.name === form.service_name), - [form.service_name, services] - ) + const upsertBuild = (build: BuildJob) => { + setBuilds((prev) => { + const exists = prev.some((item) => item.id === build.id) + if (!exists) { + return [build, ...prev] + } + return prev.map((item) => (item.id === build.id ? build : item)) + }) + } - const resetForm = (catalog: CatalogService[]) => { - const first = catalog[0] - setForm({ - service_name: first?.name ?? '', - release_id: '', - branch: first?.default_branch ?? 'main', - operator: defaultOperator - }) + const load = async () => { + const [buildRows, catalog] = await Promise.all([api.listBuilds(), api.listServices()]) + setBuilds(buildRows) + setServices(catalog) + if (!form.service_name && catalog.length > 0) { + resetForm(catalog) + } + } + + const loadBuildDetail = async (buildID: number, showLoader = false) => { + if (showLoader) { + setDetailLoading(true) + } + try { + const build = await api.getBuild(buildID) + setBuildDetail(build) + upsertBuild(build) + } catch (err) { + setError(err instanceof Error ? err.message : '加载构建日志失败') + } finally { + if (showLoader) { + setDetailLoading(false) + } + } + } + + useEffect(() => { + void load() + // eslint-disable-next-line react-hooks/exhaustive-deps + }, []) + + useEffect(() => { + if (!detailOpen || !selectedBuildID) return + + void loadBuildDetail(selectedBuildID, true) + + if (!selectedBuild || !activeBuildStatuses.has(selectedBuild.status)) { + return } - const load = async () => { - const [buildRows, catalog] = await Promise.all([api.listBuilds(), api.listServices()]) - setBuilds(buildRows) - setServices(catalog) - if (!form.service_name && catalog.length > 0) { - resetForm(catalog) - } + const timer = window.setInterval(() => { + void loadBuildDetail(selectedBuildID) + }, 2000) + + return () => window.clearInterval(timer) + }, [detailOpen, selectedBuildID, selectedBuild?.status]) + + const createBuild = async () => { + try { + const created = await api.createBuild(form) + setOpen(false) + resetForm(services) + setMessage('构建任务已创建,已打开实时构建日志') + setSelectedBuildID(created.id) + setBuildDetail(created) + setDetailOpen(true) + upsertBuild(created) + await load() + } catch (err) { + setError(err instanceof Error ? err.message : '创建构建任务失败') } + } - useEffect(() => { - void load() - // eslint-disable-next-line react-hooks/exhaustive-deps - }, []) + const openBuildDetail = (buildID: number) => { + setSelectedBuildID(buildID) + setBuildDetail(null) + setDetailOpen(true) + } - const createBuild = async () => { - await api.createBuild(form) - setOpen(false) - resetForm(services) - setMessage('构建任务已创建,后台会自动拉取仓库、构建并上传 COS') - await load() - } + return ( + + + + + + + + + - return ( - - - - - - - - - + + + + 构建记录 + + + + + + ID + 服务 + 版本 + 分支 + 状态 + 提交 + 构建机 + 错误摘要 + 动作 + + + + {builds.map((row) => ( + + {row.id} + {row.service_name} + {row.release_id} + {row.branch} + + + + {row.commit_sha || '-'} + {row.build_host || '-'} + + + {row.error_message || '-'} + + + + + + + ))} + +
+
+
+
- - - - 构建记录 - - - - - - ID - 服务 - 版本 - 分支 - 状态 - 提交 - 构建机 - COS Key - - - - {builds.map((row) => ( - - {row.id} - {row.service_name} - {row.release_id} - {row.branch} - - - - {row.commit_sha || '-'} - {row.build_host || '-'} - {row.cos_key || '-'} - - ))} - -
-
-
-
+ setOpen(false)} fullWidth maxWidth="sm"> + 新建构建任务 + + + { + const serviceName = event.target.value + const serviceMeta = services.find((item) => item.name === serviceName) + setForm((prev) => ({ + ...prev, + service_name: serviceName, + branch: serviceMeta?.default_branch ?? prev.branch + })) + }} + > + {services.map((service) => ( + + {service.name} + + ))} + + setForm((prev) => ({ ...prev, release_id: event.target.value }))} + /> + setForm((prev) => ({ ...prev, branch: event.target.value }))} + /> + setForm((prev) => ({ ...prev, operator: event.target.value }))} + /> + + + 平台会先在构建机本地源码目录中自动查找匹配仓库;找到后执行 `git fetch / checkout / pull`。如果本地没有对应仓库,再自动克隆到构建工作区。 + + + + + + + + - setOpen(false)} fullWidth maxWidth="sm"> - 新建构建任务 - - - { - const serviceName = event.target.value - const serviceMeta = services.find((item) => item.name === serviceName) - setForm((prev) => ({ - ...prev, - service_name: serviceName, - branch: serviceMeta?.default_branch ?? prev.branch - })) - }} - > - {services.map((service) => ( - - {service.name} - - ))} - - setForm((prev) => ({ ...prev, release_id: event.target.value }))} - /> - setForm((prev) => ({ ...prev, branch: event.target.value }))} - /> - setForm((prev) => ({ ...prev, operator: event.target.value }))} - /> - - - 平台会先在构建机本地源码目录中自动查找匹配仓库;找到后执行 `git fetch / checkout / pull`。如果本地没有对应仓库,再自动克隆到构建工作区。 - - - - - - - - + setDetailOpen(false)} fullWidth maxWidth="lg"> + 构建日志 + + {detailLoading && !selectedBuild ? ( + + + + ) : selectedBuild ? ( + + + + {`${selectedBuild.service_name} · ${selectedBuild.release_id}`} + + + + + + + + + + + + + - setMessage('')}> - - {message} + {activeBuildStatuses.has(selectedBuild.status) ? ( + + 当前任务仍在执行,日志会每 2 秒自动刷新。 - - - ) + ) : null} + + {selectedBuild.error_message ? ( + + {selectedBuild.error_message} + + ) : null} + + + {selectedBuild.log_output || selectedBuild.log_excerpt || '暂无日志输出'} + + + + COS Key: {selectedBuild.cos_key || '-'} + +
+ ) : ( + + 未找到该构建任务。 + + )} + + + + + + + setMessage('')}> + + {message} + + + + setError('')}> + + {error} + + +
+ ) } diff --git a/frontend/src/types/index.ts b/frontend/src/types/index.ts index e9324a9..1237533 100644 --- a/frontend/src/types/index.ts +++ b/frontend/src/types/index.ts @@ -51,6 +51,7 @@ export interface BuildJob { cos_key: string artifact_url: string sha256: string + log_output: string log_excerpt: string error_message: string created_at?: string diff --git a/ops-scripts/build_operator.py b/ops-scripts/build_operator.py index 63144b4..92e3d56 100644 --- a/ops-scripts/build_operator.py +++ b/ops-scripts/build_operator.py @@ -47,6 +47,18 @@ def normalize_remote(remote: str) -> str: return value.strip("/").lower() +def git_host(remote: str) -> str: + value = remote.strip() + if not value: + return "" + if value.startswith("git@") and ":" in value: + return value.split("@", 1)[1].split(":", 1)[0].strip().lower() + parsed = urlparse(value) + if parsed.netloc: + return parsed.netloc.strip().lower() + return "" + + def run_command( argv: list[str], *, @@ -55,19 +67,29 @@ def run_command( allow_failure: bool = False, ) -> subprocess.CompletedProcess[str]: log(f"run: {' '.join(argv)}") - proc = subprocess.run( + proc = subprocess.Popen( argv, cwd=str(cwd) if cwd else None, env=env, text=True, stdout=subprocess.PIPE, stderr=subprocess.STDOUT, + bufsize=1, ) - if proc.stdout: - print(proc.stdout, end="") - if proc.returncode != 0 and not allow_failure: - raise RuntimeError(f"command failed ({proc.returncode}): {' '.join(argv)}") - return proc + + output_parts: list[str] = [] + assert proc.stdout is not None + for line in proc.stdout: + output_parts.append(line) + print(line, end="", flush=True) + + returncode = proc.wait() + stdout = "".join(output_parts) + completed = subprocess.CompletedProcess(argv, returncode, stdout) + + if returncode != 0 and not allow_failure: + raise RuntimeError(f"command failed ({returncode}): {' '.join(argv)}") + return completed class BuildOperator: @@ -158,6 +180,10 @@ class BuildOperator: env["CGO_ENABLED"] = "0" env["GOOS"] = "linux" env["GOARCH"] = "amd64" + private_host = git_host(self.repo_url) or git_host(str(self.global_build_cfg.get("gitea_clone_prefix", ""))) + if private_host: + env.setdefault("GOPRIVATE", private_host) + env.setdefault("GONOSUMDB", private_host) if self.git_ssh_command: env["GIT_SSH_COMMAND"] = self.git_ssh_command return env