package service import ( "context" "fmt" "log/slog" "strings" "sync" "time" "hyapp/pkg/appcode" "hyapp/pkg/logx" "hyapp/services/room-service/internal/room/outbox" ) const ( // outboxRetryExponentialBackoff 是房间事件补偿的默认策略,避免外部故障时每秒刷同一条错误。 outboxRetryExponentialBackoff = "exponential_backoff" // outboxWorkerName 固定写入日志,便于后续按 worker 聚合检索。 outboxWorkerName = "room_outbox" // robotOutboxWorkerName 固定标识机器人展示事件通道,避免和主 room_outbox 监控混淆。 robotOutboxWorkerName = "room_robot_outbox" ) // OutboxWorkerOptions 控制 room_outbox pending 事件的持续补偿循环。 type OutboxWorkerOptions struct { // PollInterval 是两轮 pending 扫描之间的间隔;非正数会被归一到安全默认值。 PollInterval time.Duration // BatchSize 是单轮最多处理条数;非正数会被归一,避免全表扫描。 BatchSize int // Concurrency 是本进程内独立抢占 pending outbox 的 worker 数;每条 worker 都依赖 MySQL SKIP LOCKED 隔离批次。 Concurrency int // PublishTimeout 限制单条外部投递耗时,shutdown 时当前事件最多阻塞该时长。 PublishTimeout time.Duration // RetryStrategy 当前只支持 exponential_backoff。 RetryStrategy string // MaxRetryCount 是转入 failed 前允许的最大失败次数。 MaxRetryCount int // InitialBackoff 是首次失败后的等待时间。 InitialBackoff time.Duration // MaxBackoff 是指数退避上限。 MaxBackoff time.Duration // WorkerID 写入 outbox worker_id,空值时使用节点 ID;并发启动时会自动带上序号。 WorkerID string // WorkerName 写入日志 worker 字段,空值时使用主通道或机器人通道的稳定名称。 WorkerName string // StaleDiscardAfter 只用于 robot outbox;超过该窗口的展示事件直接删除,不再补发 MQ/IM。 StaleDiscardAfter time.Duration // CleanupInterval 控制 robot outbox 后台清理频率;非正数表示不启动清理。 CleanupInterval time.Duration // CleanupActiveAfter 控制 pending/retryable/delivering 机器人展示事件的最大保留时长。 CleanupActiveAfter time.Duration // CleanupTerminalAfter 控制 delivered/failed 机器人展示事件的最大保留时长。 CleanupTerminalAfter time.Duration // CleanupBatchSize 控制单轮清理最多删除多少行,避免长事务拖慢主库。 CleanupBatchSize int } func defaultOutboxWorkerOptions() OutboxWorkerOptions { // 默认值优先避免 busy loop、无限重试和日志污染。 return OutboxWorkerOptions{ PollInterval: time.Second, BatchSize: 100, Concurrency: 1, PublishTimeout: 3 * time.Second, RetryStrategy: outboxRetryExponentialBackoff, MaxRetryCount: 10, InitialBackoff: 5 * time.Second, MaxBackoff: 5 * time.Minute, } } func normalizeOutboxWorkerOptions(options OutboxWorkerOptions) OutboxWorkerOptions { // worker 层再次归一化,防止测试或手动构造绕过 config.Normalize。 defaults := defaultOutboxWorkerOptions() if options.PollInterval <= 0 { // 非正间隔不能直接传给 ticker,否则会 panic 或形成忙循环。 options.PollInterval = defaults.PollInterval } if options.BatchSize <= 0 { // 非正批量按安全默认值处理,避免 repository 扫描不受控。 options.BatchSize = defaults.BatchSize } if options.Concurrency <= 0 { // 并发数为 0 等价于历史单 worker,避免老测试或手工构造配置意外关闭补偿。 options.Concurrency = defaults.Concurrency } if options.PublishTimeout <= 0 { // 单条外部投递必须有 deadline,shutdown 才能等待有界。 options.PublishTimeout = defaults.PublishTimeout } if options.MaxRetryCount <= 0 { options.MaxRetryCount = defaults.MaxRetryCount } if options.InitialBackoff <= 0 { options.InitialBackoff = defaults.InitialBackoff } if options.MaxBackoff <= 0 { options.MaxBackoff = defaults.MaxBackoff } if options.MaxBackoff < options.InitialBackoff { options.MaxBackoff = options.InitialBackoff } if options.CleanupBatchSize <= 0 && (options.CleanupActiveAfter > 0 || options.CleanupTerminalAfter > 0) { // robot outbox 清理开启时必须有批量上限;默认 5 万和线上一次性清理批量保持一致。 options.CleanupBatchSize = 50000 } options.RetryStrategy = strings.TrimSpace(options.RetryStrategy) if options.RetryStrategy == "" { // 空策略按指数退避处理,保持和配置默认值一致。 options.RetryStrategy = defaults.RetryStrategy } return options } // RunOutboxWorker 持续扫描 pending outbox;它不执行房间命令,只投递已持久化事件。 func (s *Service) RunOutboxWorker(ctx context.Context, options OutboxWorkerOptions) { s.runOutboxWorker(ctx, options, outboxWorkerName, s.ProcessPendingOutbox) } // RunRobotOutboxWorker 持续扫描机器人房间展示 outbox;它和主 room_outbox 完全分开抢占。 func (s *Service) RunRobotOutboxWorker(ctx context.Context, options OutboxWorkerOptions) { s.runOutboxWorker(ctx, options, robotOutboxWorkerName, s.ProcessPendingRobotOutbox) } func (s *Service) runOutboxWorker(ctx context.Context, options OutboxWorkerOptions, workerName string, process func(context.Context, OutboxWorkerOptions) error) { options = normalizeOutboxWorkerOptions(options) if options.RetryStrategy != outboxRetryExponentialBackoff { // 配置层会拒绝未知策略;这里兜底避免测试或手写构造静默改变语义。 logx.Error(ctx, "worker_stopped", fmt.Errorf("unsupported retry_strategy"), slog.String("worker", workerName), slog.String("node_id", s.nodeID)) return } if options.Concurrency == 1 { // 单 worker 保持历史 worker_id 和日志名称,避免线上排障查询字段无意义变化。 options.WorkerID = firstNonEmpty(strings.TrimSpace(options.WorkerID), s.nodeID) options.WorkerName = firstNonEmpty(strings.TrimSpace(options.WorkerName), workerName) s.runOutboxWorkerLoop(ctx, options, process) return } var wg sync.WaitGroup for index := 1; index <= options.Concurrency; index++ { workerOptions := options // 多 worker 共用同一个进程上下文,但必须写不同 worker_id,方便从 delivering 记录定位卡住的协程。 workerOptions.WorkerName = fmt.Sprintf("%s-%02d", workerName, index) workerOptions.WorkerID = fmt.Sprintf("%s/%s", s.nodeID, workerOptions.WorkerName) wg.Add(1) go func() { defer wg.Done() s.runOutboxWorkerLoop(ctx, workerOptions, process) }() } wg.Wait() } func (s *Service) runOutboxWorkerLoop(ctx context.Context, options OutboxWorkerOptions, process func(context.Context, OutboxWorkerOptions) error) { workerName := firstNonEmpty(strings.TrimSpace(options.WorkerName), outboxWorkerName) // 启动后先执行一轮,避免服务刚恢复时还要等一个 poll interval 才补偿历史事件。 if err := process(ctx, options); err != nil && ctx.Err() == nil { logx.Error(ctx, "worker_batch_failed", err, slog.String("worker", workerName), slog.String("node_id", s.nodeID), slog.Int("batch_size", options.BatchSize)) } ticker := time.NewTicker(options.PollInterval) defer ticker.Stop() for { select { case <-ctx.Done(): return case <-ticker.C: if err := process(ctx, options); err != nil && ctx.Err() == nil { logx.Error(ctx, "worker_batch_failed", err, slog.String("worker", workerName), slog.String("node_id", s.nodeID), slog.Int("batch_size", options.BatchSize)) } } } } // ProcessPendingOutbox 把待投递事件补偿性地推送给异步消费者。 func (s *Service) ProcessPendingOutbox(ctx context.Context, options OutboxWorkerOptions) error { workerName := firstNonEmpty(strings.TrimSpace(options.WorkerName), outboxWorkerName) return s.processPendingOutbox(ctx, options, workerName, false) } // ProcessPendingRobotOutbox 把机器人展示事件补偿性地推送给独立机器人异步消费者。 func (s *Service) ProcessPendingRobotOutbox(ctx context.Context, options OutboxWorkerOptions) error { workerName := firstNonEmpty(strings.TrimSpace(options.WorkerName), robotOutboxWorkerName) if err := s.maybeCleanupRobotOutbox(ctx, normalizeOutboxWorkerOptions(options)); err != nil { return err } return s.processPendingOutbox(ctx, options, workerName, true) } func (s *Service) processPendingOutbox(ctx context.Context, options OutboxWorkerOptions, workerName string, robotLane bool) error { options = normalizeOutboxWorkerOptions(options) if options.RetryStrategy != outboxRetryExponentialBackoff { return fmt.Errorf("unsupported outbox retry_strategy %q", options.RetryStrategy) } // 补偿 worker 先抢占 pending/retryable 事件,再投递,避免多机同时处理同一批。 records, err := s.claimPendingOutbox(ctx, robotLane, options) if err != nil { return err } for _, record := range records { select { case <-ctx.Done(): // 退出时不再领取新事件;已抢占但未处理完的事件会在 lock_until_ms 到期后重新可见。 return ctx.Err() default: } if robotLane && staleRobotOutboxRecord(record, options.StaleDiscardAfter) { // 机器人展示事件超过 TTL 后再补发会制造“迟到礼物”体验,直接删除并释放 MQ/IM 资源。 deleteCtx, deleteCancel := context.WithTimeout(appcode.WithContext(context.Background(), record.AppCode), options.PublishTimeout) deleteErr := s.repository.DeleteRobotOutbox(deleteCtx, record.EventID) deleteCancel() if deleteErr != nil { return deleteErr } logx.Warn(ctx, "robot_outbox_stale_discarded", slog.String("worker", workerName), slog.String("node_id", s.nodeID), slog.String("event_id", record.EventID), slog.String("event_type", record.EventType), slog.String("room_id", record.RoomID), slog.Int64("created_at_ms", record.CreatedAtMS), slog.Int64("stale_discard_after_ms", options.StaleDiscardAfter.Milliseconds()), ) continue } if record.RetryCount >= options.MaxRetryCount { // 已达到最大失败次数的历史事件直接转死信,防止每轮 worker 继续打外部系统。 markCtx, markCancel := context.WithTimeout(appcode.WithContext(context.Background(), record.AppCode), options.PublishTimeout) markErr := s.markOutboxDead(markCtx, robotLane, record.EventID, deadLetterReason(record)) markCancel() if markErr != nil { return markErr } logx.Error(ctx, "outbox_dead_letter", nil, slog.String("worker", workerName), slog.String("node_id", s.nodeID), slog.String("event_id", record.EventID), slog.String("event_type", record.EventType), slog.String("room_id", record.RoomID), slog.Int("retry_count", record.RetryCount), slog.String("last_error", trimOutboxError(record.LastError)), ) continue } publishCtx, cancel := context.WithTimeout(appcode.WithContext(context.Background(), record.AppCode), options.PublishTimeout) // 发布上下文不继承 worker ctx,避免 shutdown 取消导致已开始的单条外部投递无界处于未知状态。 err := s.publishOutboxRecord(publishCtx, record, robotLane) cancel() if err != nil { // 投递失败转为 retryable,并写入指数退避时间,等待下一轮 worker 到期后再抢占。 nextRetryAtMS := time.Now().UTC().Add(outboxBackoff(record.RetryCount+1, options)).UnixMilli() markCtx, markCancel := context.WithTimeout(appcode.WithContext(context.Background(), record.AppCode), options.PublishTimeout) markErr := s.markOutboxRetryable(markCtx, robotLane, record.EventID, trimOutboxError(err.Error()), nextRetryAtMS) markCancel() if markErr != nil { return markErr } logx.Warn(ctx, "outbox_publish_retryable", slog.String("worker", workerName), slog.String("node_id", s.nodeID), slog.String("event_id", record.EventID), slog.String("event_type", record.EventType), slog.String("room_id", record.RoomID), slog.Int("retry_count", record.RetryCount+1), slog.Int64("next_retry_at_ms", nextRetryAtMS), slog.String("error", trimOutboxError(err.Error())), ) continue } // 只有外部发布成功后才能标记 delivered。 markCtx, markCancel := context.WithTimeout(appcode.WithContext(context.Background(), record.AppCode), options.PublishTimeout) markErr := s.markOutboxDelivered(markCtx, robotLane, record.EventID) markCancel() if markErr != nil { return markErr } logx.Info(ctx, "outbox_delivered", slog.String("worker", workerName), slog.String("node_id", s.nodeID), slog.String("event_id", record.EventID), slog.String("event_type", record.EventType), slog.String("room_id", record.RoomID), slog.Int("retry_count", record.RetryCount), ) } return nil } func (s *Service) maybeCleanupRobotOutbox(ctx context.Context, options OutboxWorkerOptions) error { if s == nil || s.repository == nil || options.CleanupInterval <= 0 || (options.CleanupActiveAfter <= 0 && options.CleanupTerminalAfter <= 0) { return nil } now := time.Now().UTC() s.robotOutboxCleanupMu.Lock() if !s.robotOutboxLastCleanup.IsZero() && now.Sub(s.robotOutboxLastCleanup) < options.CleanupInterval { s.robotOutboxCleanupMu.Unlock() return nil } // 先记录本次尝试时间,避免多 worker 或异常重试在同一秒内反复打 DELETE。 s.robotOutboxLastCleanup = now s.robotOutboxCleanupMu.Unlock() var activeBeforeMS int64 if options.CleanupActiveAfter > 0 { activeBeforeMS = now.Add(-options.CleanupActiveAfter).UnixMilli() } var terminalBeforeMS int64 if options.CleanupTerminalAfter > 0 { terminalBeforeMS = now.Add(-options.CleanupTerminalAfter).UnixMilli() } cleanupCtx, cancel := context.WithTimeout(appcode.WithContext(context.Background(), appcode.FromContext(ctx)), options.PublishTimeout) activeDeleted, terminalDeleted, err := s.repository.CleanupRobotOutbox(cleanupCtx, activeBeforeMS, terminalBeforeMS, options.CleanupBatchSize) cancel() if err != nil { return err } if activeDeleted > 0 || terminalDeleted > 0 { logx.Info(ctx, "robot_outbox_cleanup_deleted", slog.String("worker", robotOutboxWorkerName), slog.String("node_id", s.nodeID), slog.Int64("active_deleted", activeDeleted), slog.Int64("terminal_deleted", terminalDeleted), slog.Int("batch_size", options.CleanupBatchSize), ) } return nil } func staleRobotOutboxRecord(record outbox.Record, ttl time.Duration) bool { if ttl <= 0 || record.CreatedAtMS <= 0 { return false } // 使用当前 UTC 毫秒判断 TTL,和 outbox created_at_ms 的持久化口径保持一致。 return time.Now().UTC().UnixMilli()-record.CreatedAtMS >= ttl.Milliseconds() } func (s *Service) claimPendingOutbox(ctx context.Context, robotLane bool, options OutboxWorkerOptions) ([]outbox.Record, error) { lockUntilMS := time.Now().UTC().Add(options.PublishTimeout).UnixMilli() workerID := firstNonEmpty(strings.TrimSpace(options.WorkerID), s.nodeID) if robotLane { return s.repository.ClaimPendingRobotOutbox(ctx, workerID, options.BatchSize, lockUntilMS) } return s.repository.ClaimPendingOutbox(ctx, workerID, options.BatchSize, lockUntilMS) } func (s *Service) markOutboxDelivered(ctx context.Context, robotLane bool, eventID string) error { if robotLane { return s.repository.MarkRobotOutboxDelivered(ctx, eventID) } return s.repository.MarkOutboxDelivered(ctx, eventID) } func (s *Service) markOutboxRetryable(ctx context.Context, robotLane bool, eventID string, lastErr string, nextRetryAtMS int64) error { if robotLane { return s.repository.MarkRobotOutboxRetryable(ctx, eventID, lastErr, nextRetryAtMS) } return s.repository.MarkOutboxRetryable(ctx, eventID, lastErr, nextRetryAtMS) } func (s *Service) markOutboxDead(ctx context.Context, robotLane bool, eventID string, lastErr string) error { if robotLane { return s.repository.MarkRobotOutboxDead(ctx, eventID, lastErr) } return s.repository.MarkOutboxDead(ctx, eventID, lastErr) } func (s *Service) publishOutboxRecord(ctx context.Context, record outbox.Record, robotLane bool) error { // Ignited 先安排延迟发射,再对外广播事件;如果 MQ 延迟调度不可用,本条 outbox 保持 retryable。 if err := s.scheduleRoomRocketLaunchFromEnvelope(ctx, record.Envelope); err != nil { return err } if robotLane { if s.robotOutboxPublisher == nil { return nil } return s.robotOutboxPublisher.PublishOutboxEvent(ctx, record.Envelope) } if s.outboxPublisher == nil { return nil } return s.outboxPublisher.PublishOutboxEvent(ctx, record.Envelope) } // outboxBackoff 根据失败次数计算指数退避,且永远不超过 MaxBackoff。 func outboxBackoff(retryCount int, options OutboxWorkerOptions) time.Duration { if retryCount <= 1 { return options.InitialBackoff } backoff := options.InitialBackoff for i := 1; i < retryCount; i++ { if backoff >= options.MaxBackoff/2 { // 提前截断,避免 duration 乘 2 溢出或超过配置上限。 return options.MaxBackoff } backoff *= 2 } if backoff > options.MaxBackoff { return options.MaxBackoff } return backoff } // deadLetterReason 生成写入 failed 状态的稳定错误原因。 func deadLetterReason(record outbox.Record) string { if strings.TrimSpace(record.LastError) == "" { return "outbox retry limit exceeded" } return trimOutboxError("outbox retry limit exceeded: " + record.LastError) } func trimOutboxError(value string) string { // last_error 会写入 MySQL 和日志,先去掉首尾空白避免不可见差异影响排障。 value = strings.TrimSpace(value) if len(value) <= 512 { return value } // V1 只保留错误前缀,避免外部响应过大撑爆日志或 TEXT 字段查询成本。 return value[:512] } func firstNonEmpty(values ...string) string { for _, value := range values { if strings.TrimSpace(value) != "" { return strings.TrimSpace(value) } } return "" }