package app import ( "context" "errors" "fmt" "log/slog" "strings" "time" "hyapp/pkg/appcode" "hyapp/pkg/logx" "hyapp/pkg/rocketmqx" "hyapp/pkg/walletmq" "hyapp/services/wallet-service/internal/config" mysqlstorage "hyapp/services/wallet-service/internal/storage/mysql" ) func (a *App) startWalletOutboxWorkers(ctx context.Context, workerName string, options config.OutboxWorkerConfig, producer *rocketmqx.Producer, topic string, includeEventTypes []string, excludeEventTypes []string) { if options.Concurrency <= 0 { options.Concurrency = 1 } for workerIndex := 1; workerIndex <= options.Concurrency; workerIndex++ { workerID := fmt.Sprintf("%s-%s-%02d", workerName, a.nodeID, workerIndex) a.workers.Add(1) go func() { defer a.workers.Done() a.runWalletOutboxWorker(ctx, workerID, options, producer, topic, includeEventTypes, excludeEventTypes) }() } } func (a *App) runWalletOutboxWorker(ctx context.Context, workerID string, options config.OutboxWorkerConfig, producer *rocketmqx.Producer, topic string, includeEventTypes []string, excludeEventTypes []string) { ticker := time.NewTicker(options.PollInterval) defer ticker.Stop() for { processed, err := a.processWalletOutboxBatch(ctx, workerID, options, producer, topic, includeEventTypes, excludeEventTypes) if err != nil && !errors.Is(err, context.Canceled) { logx.Error(ctx, "wallet_outbox_publish_failed", err, slog.String("worker_id", workerID)) } if processed >= options.BatchSize { continue } select { case <-ctx.Done(): return case <-ticker.C: } } } func (a *App) processWalletOutboxBatch(ctx context.Context, workerID string, options config.OutboxWorkerConfig, producer *rocketmqx.Producer, topic string, includeEventTypes []string, excludeEventTypes []string) (int, error) { if a.mysqlRepo == nil || producer == nil { return 0, nil } records, err := a.mysqlRepo.ClaimPendingWalletOutboxFiltered(ctx, workerID, options.BatchSize, time.Now().UTC().Add(options.PublishTimeout).UnixMilli(), mysqlstorage.WalletOutboxClaimFilter{ IncludeEventTypes: includeEventTypes, ExcludeEventTypes: excludeEventTypes, }) if err != nil { return 0, err } for _, record := range records { if record.RetryCount >= options.MaxRetryCount { markCtx, cancel := context.WithTimeout(appcode.WithContext(context.Background(), record.AppCode), options.PublishTimeout) err := a.mysqlRepo.MarkWalletOutboxDead(markCtx, record.EventID, deadWalletOutboxReason(record)) cancel() if err != nil { return 0, err } continue } publishCtx, cancel := context.WithTimeout(appcode.WithContext(context.Background(), record.AppCode), options.PublishTimeout) err := a.publishWalletOutboxRecord(publishCtx, producer, topic, record) cancel() if err != nil { nextRetryAtMS := time.Now().UTC().Add(walletOutboxBackoff(record.RetryCount+1, options)).UnixMilli() markCtx, markCancel := context.WithTimeout(appcode.WithContext(context.Background(), record.AppCode), options.PublishTimeout) markErr := a.mysqlRepo.MarkWalletOutboxRetryable(markCtx, record.EventID, err.Error(), nextRetryAtMS) markCancel() if markErr != nil { return 0, markErr } continue } markCtx, markCancel := context.WithTimeout(appcode.WithContext(context.Background(), record.AppCode), options.PublishTimeout) markErr := a.mysqlRepo.MarkWalletOutboxDelivered(markCtx, record.EventID) markCancel() if markErr != nil { return 0, markErr } } return len(records), nil } func (a *App) publishWalletOutboxRecord(ctx context.Context, producer *rocketmqx.Producer, topic string, record mysqlstorage.WalletOutboxRecord) error { body, err := walletmq.EncodeWalletOutboxMessage(walletmq.WalletOutboxMessage{ AppCode: record.AppCode, EventID: record.EventID, EventType: record.EventType, TransactionID: record.TransactionID, CommandID: record.CommandID, UserID: record.UserID, AssetType: record.AssetType, AvailableDelta: record.AvailableDelta, FrozenDelta: record.FrozenDelta, PayloadJSON: record.PayloadJSON, OccurredAtMS: record.CreatedAtMS, }) if err != nil { return err } return producer.SendSync(ctx, rocketmqx.Message{ Topic: topic, Tag: walletmq.TagWalletOutboxEvent, Keys: []string{record.EventID, record.TransactionID, record.CommandID}, Body: body, }) } func deadWalletOutboxReason(record mysqlstorage.WalletOutboxRecord) string { last := strings.TrimSpace(record.LastError) if last == "" { return fmt.Sprintf("wallet outbox exceeded retry limit: retry_count=%d", record.RetryCount) } return last } func walletOutboxBackoff(retryCount int, options config.OutboxWorkerConfig) time.Duration { if retryCount <= 0 { return options.InitialBackoff } backoff := options.InitialBackoff for i := 1; i < retryCount; i++ { backoff *= 2 if backoff >= options.MaxBackoff { return options.MaxBackoff } } return backoff } func (a *App) closeBackgroundWorkers() { if a.stopWorker != nil { a.stopWorker() } a.workers.Wait() }