diff --git a/pkg/apptracking/limits.go b/pkg/apptracking/limits.go new file mode 100644 index 00000000..40090f05 --- /dev/null +++ b/pkg/apptracking/limits.go @@ -0,0 +1,68 @@ +// Package apptracking defines the transport and persistence limits shared by the +// public App tracking endpoint and statistics-service. +package apptracking + +import ( + "bytes" + "encoding/json" + "strings" +) + +const ( + // MaxBatchSize keeps one request bounded before gateway fans it into the + // statistics write transaction. + MaxBatchSize = 50 + // MaxPropertiesBytes bounds the raw JSON value after surrounding whitespace + // is removed. The value is rejected instead of truncated because truncation + // can change JSON meaning or produce invalid JSON. + MaxPropertiesBytes = 4096 + + // Text limits are byte limits, not rune limits. They mirror the durable + // app_tracking_events columns and therefore apply identically at both service + // boundaries; an oversized client value must never be deferred to MySQL. + MaxAppCodeBytes = 32 + MaxEventIDBytes = 160 + MaxEventNameBytes = 96 + MaxEventTypeBytes = 64 + MaxScreenBytes = 128 + MaxTargetTypeBytes = 64 + MaxTargetIDBytes = 128 + MaxDeviceIDBytes = 128 + MaxSessionIDBytes = 160 + MaxPlatformBytes = 32 + MaxAppVersionBytes = 64 + MaxLanguageBytes = 32 + MaxTimezoneBytes = 64 + MaxErrorCodeBytes = 64 +) + +// ValidBatchSize rejects both empty reports and reports large enough to hold a +// statistics transaction open for an unbounded period. +func ValidBatchSize(size int) bool { + return size > 0 && size <= MaxBatchSize +} + +// NormalizeText removes transport-only surrounding whitespace and checks the +// encoded byte length. Oversized values return false and are never shortened: +// identifiers and error codes are facts whose suffix may be significant. +func NormalizeText(value string, maxBytes int) (string, bool) { + value = strings.TrimSpace(value) + if maxBytes > 0 && len(value) > maxBytes { + return "", false + } + return value, true +} + +// NormalizeProperties returns a detached, trimmed JSON value. Empty and JSON +// null both mean no properties; non-empty values must satisfy the same byte and +// syntax contract in gateway and statistics-service. +func NormalizeProperties(raw json.RawMessage) (json.RawMessage, bool) { + trimmed := bytes.TrimSpace(raw) + if len(trimmed) == 0 || bytes.Equal(trimmed, []byte("null")) { + return nil, true + } + if len(trimmed) > MaxPropertiesBytes || !json.Valid(trimmed) { + return nil, false + } + return json.RawMessage(bytes.Clone(trimmed)), true +} diff --git a/pkg/apptracking/limits_test.go b/pkg/apptracking/limits_test.go new file mode 100644 index 00000000..58c69207 --- /dev/null +++ b/pkg/apptracking/limits_test.go @@ -0,0 +1,36 @@ +package apptracking + +import ( + "encoding/json" + "strings" + "testing" +) + +func TestNormalizeTextUsesBytesAndNeverTruncates(t *testing.T) { + if got, ok := NormalizeText(" "+strings.Repeat("e", MaxErrorCodeBytes)+" ", MaxErrorCodeBytes); !ok || got != strings.Repeat("e", MaxErrorCodeBytes) { + t.Fatalf("text at byte limit must be trimmed but otherwise preserved: ok=%v got=%q", ok, got) + } + if got, ok := NormalizeText(strings.Repeat("e", 185), MaxErrorCodeBytes); ok || got != "" { + t.Fatalf("oversized error_code must be rejected, never truncated: ok=%v got=%q", ok, got) + } + // 一个汉字编码为三个 UTF-8 bytes；这里防止调用方误把字段限制实现成 rune 数。 + if got, ok := NormalizeText(strings.Repeat("界", 22), MaxErrorCodeBytes); ok || got != "" { + t.Fatalf("text limits must count UTF-8 bytes: ok=%v got=%q", ok, got) + } +} + +func TestNormalizePropertiesAndBatchContract(t *testing.T) { + if !ValidBatchSize(1) || !ValidBatchSize(MaxBatchSize) || ValidBatchSize(0) || ValidBatchSize(MaxBatchSize+1) { + t.Fatalf("batch bounds do not match the shared contract") + } + got, ok := NormalizeProperties(json.RawMessage(` {"screen":"home"} `)) + if !ok || string(got) != `{"screen":"home"}` { + t.Fatalf("valid properties mismatch: ok=%v got=%s", ok, got) + } + if got, ok := NormalizeProperties(json.RawMessage(`"` + strings.Repeat("x", MaxPropertiesBytes) + `"`)); ok || got != nil { + t.Fatalf("properties over the encoded byte limit must be rejected: ok=%v bytes=%d", ok, len(got)) + } + if got, ok := NormalizeProperties(json.RawMessage(`{"broken"`)); ok || got != nil { + t.Fatalf("invalid properties JSON must be rejected: ok=%v got=%s", ok, got) + } +} diff --git a/pkg/outboxpartition/partitions.go b/pkg/outboxpartition/partitions.go new file mode 100644 index 00000000..67d01c05 --- /dev/null +++ b/pkg/outboxpartition/partitions.go @@ -0,0 +1,163 @@ +// Package outboxpartition provides the shared tenant scheduling rules used by +// owner-service outbox workers. It does not read or publish business facts; +// repositories retain ownership of indexed claims and delivery state. +package outboxpartition + +import ( + "context" + "fmt" + "sort" + "strings" + "sync" + "sync/atomic" + "time" + + "hyapp/pkg/appcode" +) + +const DefaultDiscoveryRefreshInterval = 5 * time.Minute + +// Partitions resolves the App partitions visible to one outbox lane and rotates +// the first partition across claims. A configured allowlist is immutable for the +// process lifetime; an empty allowlist falls back to low-frequency DB discovery. +type Partitions struct { + configured []string + refreshInterval time.Duration + + mu sync.Mutex + cached []string + hasCache bool + nextRefresh time.Time + cursor atomic.Uint64 +} + +// New creates a partition scheduler. Operators can stage a backlog safely by +// configuring only the Apps currently allowed to publish; an empty list keeps +// automatic discovery as the compatibility default for existing deployments. +func New(configured []string, refreshInterval time.Duration) *Partitions { + if refreshInterval <= 0 { + refreshInterval = DefaultDiscoveryRefreshInterval + } + return &Partitions{ + configured: Normalize(configured, false), + refreshInterval: refreshInterval, + } +} + +// Resolve returns configured Apps without touching MySQL. In discovery mode it +// refreshes at most once per interval across all local worker goroutines. A +// failed refresh returns the last successful list alongside the error so the +// caller can log the stale condition without stopping already-known partitions. +func (p *Partitions) Resolve(ctx context.Context, discover func(context.Context) ([]string, error)) ([]string, error) { + if p == nil { + return nil, nil + } + if len(p.configured) > 0 { + return append([]string(nil), p.configured...), nil + } + if discover == nil { + return nil, nil + } + + p.mu.Lock() + defer p.mu.Unlock() + now := time.Now().UTC() + if p.hasCache && now.Before(p.nextRefresh) { + return append([]string(nil), p.cached...), nil + } + + discovered, err := discover(ctx) + if err != nil { + // Move the next attempt out by the normal interval: a transient database + // error must not turn every hot poll into a DISTINCT query stampede. + p.nextRefresh = now.Add(p.refreshInterval) + if p.hasCache { + return append([]string(nil), p.cached...), err + } + return nil, err + } + p.cached = Normalize(discovered, true) + p.hasCache = true + p.nextRefresh = now.Add(p.refreshInterval) + return append([]string(nil), p.cached...), nil +} + +// Order rotates the first App atomically while preserving the remaining cyclic +// order. Multiple worker goroutines therefore share one fair cursor instead of +// independently preferring the first configured tenant. +func (p *Partitions) Order(appCodes []string) []string { + if len(appCodes) == 0 { + return nil + } + if p == nil { + return append([]string(nil), appCodes...) + } + start := int((p.cursor.Add(1) - 1) % uint64(len(appCodes))) + ordered := make([]string, 0, len(appCodes)) + for offset := 0; offset < len(appCodes); offset++ { + ordered = append(ordered, appCodes[(start+offset)%len(appCodes)]) + } + return ordered +} + +// Normalize canonicalizes tenant keys, removes blanks and duplicates, and can +// sort DB-discovered keys for deterministic rotation. Explicit configuration +// preserves operator order so staged rollouts remain readable and predictable. +func Normalize(values []string, sorted bool) []string { + normalized := make([]string, 0, len(values)) + seen := make(map[string]struct{}, len(values)) + for _, value := range values { + if strings.TrimSpace(value) == "" { + continue + } + value = appcode.Normalize(value) + if _, exists := seen[value]; exists { + continue + } + seen[value] = struct{}{} + normalized = append(normalized, value) + } + if sorted { + sort.Strings(normalized) + } + return normalized +} + +// ClaimFair gives every App one bounded first-pass quota, then lets non-empty +// partitions use capacity left by empty queues. Each callback remains App-scoped +// and is expected to perform its own indexed SKIP LOCKED transaction. +func ClaimFair[T any](ctx context.Context, orderedApps []string, limit int, claim func(context.Context, string, int) ([]T, error)) ([]T, error) { + if limit <= 0 || len(orderedApps) == 0 || claim == nil { + return nil, nil + } + quota := (limit + len(orderedApps) - 1) / len(orderedApps) + claimed := make([]T, 0, limit) + for pass := 0; pass < 2 && len(claimed) < limit; pass++ { + for _, app := range orderedApps { + remaining := limit - len(claimed) + if remaining <= 0 { + break + } + appLimit := remaining + if pass == 0 && appLimit > quota { + appLimit = quota + } + records, err := claim(ctx, app, appLimit) + if err != nil { + return nil, err + } + if len(records) > appLimit { + // Silently truncating would strand already-claimed rows until lease + // expiry. Treat repository limit violations as explicit worker errors. + return nil, fmt.Errorf("outbox claim for app %q returned %d records above limit %d", app, len(records), appLimit) + } + claimed = append(claimed, records...) + } + if pass == 0 && len(claimed) == 0 { + // A globally empty first pass must stop here; a second identical pass + // only doubles indexed polling queries and cannot discover new work. + break + } + } + return claimed, nil +} diff --git a/pkg/outboxpartition/partitions_test.go b/pkg/outboxpartition/partitions_test.go new file mode 100644 index 00000000..fcff7720 --- /dev/null +++ b/pkg/outboxpartition/partitions_test.go @@ -0,0 +1,96 @@ +package outboxpartition + +import ( + "context" + "errors" + "reflect" + "testing" + "time" +) + +func TestConfiguredPartitionsNormalizeAndSkipDiscovery(t *testing.T) { + partitions := New([]string{" LALU ", "fami", "HUWAA", "fami", ""}, time.Hour) + discoverCalls := 0 + apps, err := partitions.Resolve(context.Background(), func(context.Context) ([]string, error) { + discoverCalls++ + return nil, errors.New("must not run") + }) + if err != nil { + t.Fatalf("resolve configured partitions: %v", err) + } + if want := []string{"lalu", "fami", "huwaa"}; !reflect.DeepEqual(apps, want) { + t.Fatalf("configured apps = %v, want %v", apps, want) + } + if discoverCalls != 0 { + t.Fatalf("configured allowlist unexpectedly queried discovery %d times", discoverCalls) + } +} + +func TestDiscoveredPartitionsAreCachedAndStaleListSurvivesRefreshFailure(t *testing.T) { + partitions := New(nil, time.Hour) + discoverCalls := 0 + discover := func(context.Context) ([]string, error) { + discoverCalls++ + if discoverCalls == 1 { + return []string{"huwaa", " LALU ", "fami"}, nil + } + return nil, errors.New("mysql unavailable") + } + first, err := partitions.Resolve(context.Background(), discover) + if err != nil || !reflect.DeepEqual(first, []string{"fami", "huwaa", "lalu"}) { + t.Fatalf("initial discovery apps=%v err=%v", first, err) + } + second, err := partitions.Resolve(context.Background(), discover) + if err != nil || !reflect.DeepEqual(second, first) || discoverCalls != 1 { + t.Fatalf("cached discovery apps=%v err=%v calls=%d", second, err, discoverCalls) + } + + // Force only the refresh boundary; production reaches this state after the + // five-minute interval without exposing a public cache mutation API. + partitions.mu.Lock() + partitions.nextRefresh = time.Time{} + partitions.mu.Unlock() + stale, err := partitions.Resolve(context.Background(), discover) + if err == nil || !reflect.DeepEqual(stale, first) { + t.Fatalf("failed refresh must retain stale apps: apps=%v err=%v", stale, err) + } +} + +func TestClaimFairRotatesLaluFamiHuwaaAndSkipsEmptyQueue(t *testing.T) { + partitions := New([]string{"lalu", "fami", "huwaa"}, time.Hour) + queues := map[string][]string{ + "lalu": {"l1", "l2", "l3"}, + "fami": nil, + "huwaa": {"h1", "h2", "h3"}, + } + claim := func(_ context.Context, app string, limit int) ([]string, error) { + queue := queues[app] + if len(queue) < limit { + limit = len(queue) + } + result := append([]string(nil), queue[:limit]...) + queues[app] = queue[limit:] + return result, nil + } + + apps, err := partitions.Resolve(context.Background(), nil) + if err != nil { + t.Fatalf("resolve configured apps: %v", err) + } + first, err := ClaimFair(context.Background(), partitions.Order(apps), 3, claim) + if err != nil || !reflect.DeepEqual(first, []string{"l1", "h1", "l2"}) { + t.Fatalf("first fair claim records=%v err=%v", first, err) + } + second, err := ClaimFair(context.Background(), partitions.Order(apps), 3, claim) + if err != nil || !reflect.DeepEqual(second, []string{"h2", "l3", "h3"}) { + t.Fatalf("rotated fair claim records=%v err=%v", second, err) + } + emptyCalls := 0 + empty, err := ClaimFair[string](context.Background(), partitions.Order(apps), 3, func(context.Context, string, int) ([]string, error) { + emptyCalls++ + return nil, nil + }) + if err != nil || len(empty) != 0 || emptyCalls != 3 { + t.Fatalf("empty queues records=%v err=%v calls=%d", empty, err, emptyCalls) + } +} diff --git a/pkg/xerr/grpc.go b/pkg/xerr/grpc.go index ce09f661..a526601b 100644 --- a/pkg/xerr/grpc.go +++ b/pkg/xerr/grpc.go @@ -1,6 +1,7 @@ package xerr import ( + "context" "errors" "google.golang.org/genproto/googleapis/rpc/errdetails" @@ -27,6 +28,16 @@ func ToGRPCError(err error) error { return err } + // context 终止描述的是调用链生命周期，而不是业务内部故障。仓储层会直接返回 + // context.Canceled/DeadlineExceeded；若先走兜底 Internal，主动取消的并发 sibling + // 会在服务端被误报成 INTERNAL_ERROR，并把一次正常的快速失败伪装成数据库事故。 + if errors.Is(err, context.Canceled) { + return status.Error(codes.Canceled, context.Canceled.Error()) + } + if errors.Is(err, context.DeadlineExceeded) { + return status.Error(codes.DeadlineExceeded, context.DeadlineExceeded.Error()) + } + domainErr, ok := As(err) if !ok { domainErr = &Error{Code: Internal, Message: "internal error"} diff --git a/pkg/xerr/grpc_test.go b/pkg/xerr/grpc_test.go index 8f48e6ec..59e4004e 100644 --- a/pkg/xerr/grpc_test.go +++ b/pkg/xerr/grpc_test.go @@ -1,6 +1,8 @@ package xerr import ( + "context" + "fmt" "go/ast" "go/parser" "go/token" @@ -29,6 +31,29 @@ func TestToGRPCErrorCarriesReason(t *testing.T) { } } +// TestToGRPCErrorPreservesContextTermination 防止并发聚合器主动取消 sibling RPC 时， +// database/sql 返回的 context 错误被兜底成 Internal，污染告警并掩盖真正的首个失败。 +func TestToGRPCErrorPreservesContextTermination(t *testing.T) { + tests := []struct { + name string + err error + code codes.Code + }{ + {name: "canceled", err: context.Canceled, code: codes.Canceled}, + {name: "wrapped canceled", err: fmt.Errorf("query interrupted: %w", context.Canceled), code: codes.Canceled}, + {name: "deadline exceeded", err: context.DeadlineExceeded, code: codes.DeadlineExceeded}, + {name: "wrapped deadline exceeded", err: fmt.Errorf("query interrupted: %w", context.DeadlineExceeded), code: codes.DeadlineExceeded}, + } + + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + if got := status.Code(ToGRPCError(test.err)); got != test.code { + t.Fatalf("grpc code mismatch: got %s want %s", got, test.code) + } + }) + } +} + func TestCatalogCoversDeclaredCodes(t *testing.T) { declared := declaredCodesFromSource(t) diff --git a/services/activity-service/cmd/audit-wheel-rewards/main.go b/services/activity-service/cmd/audit-wheel-rewards/main.go new file mode 100644 index 00000000..2b440cc9 --- /dev/null +++ b/services/activity-service/cmd/audit-wheel-rewards/main.go @@ -0,0 +1,552 @@ +// Command audit-wheel-rewards performs a read-only, item-by-item reconciliation +// between unsettled WheelRewardSettlement facts and wallet owner receipts. +package main + +import ( + "context" + "crypto/sha256" + "database/sql" + "encoding/hex" + "encoding/json" + "errors" + "flag" + "fmt" + "log" + "os" + "sort" + "strconv" + "strings" + "time" + "unicode" + + _ "github.com/go-sql-driver/mysql" + "hyapp/pkg/appcode" + "hyapp/services/activity-service/internal/config" + domain "hyapp/services/activity-service/internal/domain/wheel" +) + +type outboxRow struct { + AppCode string + OutboxID string + Payload []byte + Status string + RetryCount int + CreatedMS int64 +} + +type rewardPayload struct { + AppCode string `json:"app_code"` + DrawID string `json:"draw_id"` + DrawIDs []string `json:"draw_ids"` + CommandID string `json:"command_id"` + UserID int64 `json:"user_id"` + WheelID string `json:"wheel_id"` + SelectedTierID string `json:"selected_tier_id"` + RewardType string `json:"reward_type"` + RewardID string `json:"reward_id"` + RewardCount int64 `json:"reward_count"` + RewardCoins int64 `json:"reward_coins"` + MetadataJSON string `json:"metadata_json"` + Rewards []domain.DrawReward `json:"rewards"` + VisibleRegionID int64 `json:"visible_region_id"` +} + +type itemAudit struct { + CommandID string `json:"command_id"` + Type string `json:"type"` + State string `json:"state"` + ReceiptID string `json:"receipt_id,omitempty"` + Reason string `json:"reason,omitempty"` +} + +type rowAudit struct { + AppCode string `json:"app_code"` + OutboxID string `json:"outbox_id"` + Status string `json:"outbox_status"` + State string `json:"state"` + Items []itemAudit `json:"items"` +} + +type auditSummary struct { + Rows int `json:"rows"` + FullyPaid int `json:"fully_paid"` + FullyUnpaid int `json:"fully_unpaid"` + PartiallyPaid int `json:"partially_paid"` + Conflict int `json:"conflict"` + PaidItems int `json:"paid_items"` + UnpaidItems int `json:"unpaid_items"` + ConflictItems int `json:"conflict_items"` + AuditSHA256 string `json:"audit_sha256"` +} + +type appCodeFlags []string + +func (values *appCodeFlags) String() string { + return strings.Join(*values, ",") +} + +func (values *appCodeFlags) Set(value string) error { + value = strings.TrimSpace(value) + if value == "" { + return fmt.Errorf("app-code must not be empty") + } + *values = append(*values, value) + return nil +} + +func main() { + configPath := flag.String("config", "services/activity-service/configs/config.yaml", "path to activity-service config") + walletDatabase := flag.String("wallet-database", "hyapp_wallet", "wallet owner database on the same MySQL cluster") + limit := flag.Int("limit", 500, "maximum unsettled rows, 1..500") + details := flag.Bool("details", false, "emit one JSON reconciliation record per outbox row") + var requestedAppCodes appCodeFlags + flag.Var(&requestedAppCodes, "app-code", "required App partition to audit; repeat for multiple Apps, for example --app-code lalu --app-code fami") + flag.Parse() + + if *limit <= 0 || *limit > 500 { + log.Fatal("limit must be between 1 and 500") + } + walletDB := normalizedDatabaseName(*walletDatabase) + if walletDB == "" { + log.Fatal("wallet-database contains invalid characters") + } + appCodes, err := normalizeAuditAppCodes(requestedAppCodes) + if err != nil { + log.Fatal(err) + } + cfg, err := config.Load(*configPath) + if err != nil { + log.Fatal(err) + } + db, err := sql.Open("mysql", cfg.MySQLDSN) + if err != nil { + log.Fatal(err) + } + defer func() { _ = db.Close() }() + db.SetMaxOpenConns(2) + db.SetMaxIdleConns(1) + + ctx, cancel := context.WithTimeout(context.Background(), 2*time.Minute) + defer cancel() + if err := db.PingContext(ctx); err != nil { + log.Fatal(err) + } + rows, err := loadUnsettledRows(ctx, db, appCodes, *limit) + if err != nil { + log.Fatal(err) + } + audits := make([]rowAudit, 0, len(rows)) + for _, row := range rows { + audit, err := auditRow(ctx, db, walletDB, row) + if err != nil { + log.Fatalf("audit app_code=%s outbox_id=%s: %v", row.AppCode, row.OutboxID, err) + } + audits = append(audits, audit) + } + summary := summarize(audits) + if *details { + encoder := json.NewEncoder(os.Stdout) + for _, audit := range audits { + if err := encoder.Encode(audit); err != nil { + log.Fatal(err) + } + } + } + encoded, err := json.Marshal(summary) + if err != nil { + log.Fatal(err) + } + fmt.Println(string(encoded)) +} + +func loadUnsettledRows(ctx context.Context, db *sql.DB, requestedAppCodes []string, limit int) ([]outboxRow, error) { + if limit <= 0 || limit > 500 { + return nil, fmt.Errorf("limit must be between 1 and 500") + } + appCodes, err := normalizeAuditAppCodes(requestedAppCodes) + if err != nil { + return nil, err + } + // activity_outbox 的线上索引都以 app_code 开头。每个 App 拆成 retry/lock 两个等值分区查询， + // 既能强制使用对应索引，也避免 status OR 让优化器退化为全表扫描。每个分支只取 global remaining， + // 合并出该 App 最早的一页后再进入下一 App，因此返回量严格不超过 limit，数据库返回行数也被约束在约 2*limit 内。 + result := make([]outboxRow, 0, limit) + for _, appCodeValue := range appCodes { + remaining := limit - len(result) + if remaining <= 0 { + break + } + branches := []struct { + query string + args []any + }{ + { + query: unsettledRetryQuery(), + args: []any{ + appCodeValue, + domain.EventTypeWheelRewardSettlement, + domain.OutboxStatusPending, + domain.OutboxStatusRetryable, + remaining, + }, + }, + { + query: unsettledDeliveringQuery(), + args: []any{ + appCodeValue, + domain.EventTypeWheelRewardSettlement, + domain.OutboxStatusDelivering, + remaining, + }, + }, + } + appCandidates := make([]outboxRow, 0, remaining) + seen := make(map[string]struct{}, remaining) + for _, branch := range branches { + rows, err := loadUnsettledBranch(ctx, db, branch.query, branch.args...) + if err != nil { + return nil, fmt.Errorf("load app_code=%s: %w", appCodeValue, err) + } + for _, row := range rows { + key := row.AppCode + "\x00" + row.OutboxID + if _, exists := seen[key]; exists { + continue + } + seen[key] = struct{}{} + appCandidates = append(appCandidates, row) + } + } + sort.Slice(appCandidates, func(i, j int) bool { + if appCandidates[i].CreatedMS != appCandidates[j].CreatedMS { + return appCandidates[i].CreatedMS < appCandidates[j].CreatedMS + } + if appCandidates[i].OutboxID != appCandidates[j].OutboxID { + return appCandidates[i].OutboxID < appCandidates[j].OutboxID + } + return appCandidates[i].Status < appCandidates[j].Status + }) + if len(appCandidates) > remaining { + appCandidates = appCandidates[:remaining] + } + result = append(result, appCandidates...) + } + return result, nil +} + +func unsettledRetryQuery() string { + return ` + SELECT app_code, outbox_id, payload, status, retry_count, created_at_ms + FROM activity_outbox FORCE INDEX (idx_activity_outbox_event_retry) + WHERE app_code = ? AND event_type = ? AND status IN (?, ?) + ORDER BY created_at_ms, outbox_id + LIMIT ?` +} + +func unsettledDeliveringQuery() string { + return ` + SELECT app_code, outbox_id, payload, status, retry_count, created_at_ms + FROM activity_outbox FORCE INDEX (idx_activity_outbox_event_lock) + WHERE app_code = ? AND event_type = ? AND status = ? + ORDER BY created_at_ms, outbox_id + LIMIT ?` +} + +func loadUnsettledBranch(ctx context.Context, db *sql.DB, query string, args ...any) ([]outboxRow, error) { + rows, err := db.QueryContext(ctx, query, args...) + if err != nil { + return nil, err + } + defer rows.Close() + result := make([]outboxRow, 0) + for rows.Next() { + var row outboxRow + if err := rows.Scan(&row.AppCode, &row.OutboxID, &row.Payload, &row.Status, &row.RetryCount, &row.CreatedMS); err != nil { + return nil, err + } + result = append(result, row) + } + return result, rows.Err() +} + +func normalizeAuditAppCodes(values []string) ([]string, error) { + if len(values) == 0 { + return nil, fmt.Errorf("at least one --app-code is required") + } + const maxAuditAppCodes = 16 + seen := make(map[string]struct{}, len(values)) + result := make([]string, 0, len(values)) + for _, value := range values { + if strings.TrimSpace(value) == "" { + return nil, fmt.Errorf("app-code must not be empty") + } + value = appcode.Normalize(value) + if _, exists := seen[value]; exists { + continue + } + seen[value] = struct{}{} + result = append(result, value) + } + if len(result) == 0 { + return nil, fmt.Errorf("at least one --app-code is required") + } + if len(result) > maxAuditAppCodes { + return nil, fmt.Errorf("at most %d app-code values are allowed", maxAuditAppCodes) + } + sort.Strings(result) + return result, nil +} + +func auditRow(ctx context.Context, db *sql.DB, walletDB string, row outboxRow) (rowAudit, error) { + var payload rewardPayload + if err := json.Unmarshal(row.Payload, &payload); err != nil { + return rowAudit{}, fmt.Errorf("decode immutable payload: %w", err) + } + row.AppCode = appcode.Normalize(row.AppCode) + if payload.AppCode != "" && appcode.Normalize(payload.AppCode) != row.AppCode { + return rowAudit{}, fmt.Errorf("payload app_code does not match outbox partition") + } + payload.DrawID = strings.TrimSpace(payload.DrawID) + payload.WheelID = strings.TrimSpace(payload.WheelID) + if payload.DrawID == "" || payload.WheelID == "" || payload.UserID <= 0 { + return rowAudit{}, fmt.Errorf("payload identity is incomplete") + } + rewards := payload.Rewards + if len(rewards) == 0 { + rewards = []domain.DrawReward{{ + SelectedTierID: payload.SelectedTierID, + RewardType: payload.RewardType, + RewardID: payload.RewardID, + RewardCount: payload.RewardCount, + RewardCoins: payload.RewardCoins, + MetadataJSON: payload.MetadataJSON, + }} + } + if len(rewards) == 0 { + return rowAudit{}, fmt.Errorf("payload reward snapshot is empty") + } + + audit := rowAudit{AppCode: row.AppCode, OutboxID: row.OutboxID, Status: row.Status, Items: make([]itemAudit, 0, len(rewards))} + for index, reward := range rewards { + reward.RewardType = strings.ToLower(strings.TrimSpace(reward.RewardType)) + var item itemAudit + var err error + switch reward.RewardType { + case domain.RewardTypeCoin: + item, err = auditCoin(ctx, db, walletDB, row.AppCode, payload, reward, index, len(rewards)) + case domain.RewardTypeGift, domain.RewardTypeProp, domain.RewardTypeDress: + item, err = auditResource(ctx, db, walletDB, row.AppCode, payload, reward, index, len(rewards)) + default: + item = itemAudit{Type: reward.RewardType, State: "conflict", Reason: "unsupported reward type"} + } + if err != nil { + return rowAudit{}, err + } + audit.Items = append(audit.Items, item) + } + audit.State = aggregateRowState(audit.Items) + return audit, nil +} + +func auditCoin(ctx context.Context, db *sql.DB, walletDB string, app string, payload rewardPayload, reward domain.DrawReward, index int, total int) (itemAudit, error) { + commandID := rewardCommandID("wheel_reward", payload.DrawID, index, total) + item := itemAudit{CommandID: commandID, Type: domain.RewardTypeCoin, State: "unpaid"} + query := coinAuditQuery(walletDB) + var transactionID, bizType, statusValue, metadataJSON string + err := db.QueryRowContext(ctx, query, app, commandID).Scan(&transactionID, &bizType, &statusValue, &metadataJSON) + if err != nil { + // database/sql 和 driver/observer 包装都可能给 ErrNoRows 增加上下文；审计必须把这种稳定语义归为 unpaid，而不是中止整批核账。 + if errors.Is(err, sql.ErrNoRows) { + return item, nil + } + return itemAudit{}, err + } + metadata := map[string]any{} + if err := json.Unmarshal([]byte(metadataJSON), &metadata); err != nil { + item.State, item.Reason = "conflict", "wallet metadata is invalid" + return item, nil + } + // wallet_transactions 和 resource_grants 共用 succeeded 终态；completed 从未是当前账本状态，不能把真实已发金币误判为冲突。 + if bizType != "wheel_reward" || statusValue != "succeeded" || jsonInt64(metadata, "target_user_id") != payload.UserID || jsonInt64(metadata, "amount") != reward.RewardCoins || jsonString(metadata, "draw_id") != payload.DrawID || jsonString(metadata, "wheel_id") != payload.WheelID || jsonString(metadata, "selected_tier_id") != strings.TrimSpace(reward.SelectedTierID) || jsonString(metadata, "reason") != "wheel_reward" || jsonInt64(metadata, "visible_region_id") != payload.VisibleRegionID { + item.State, item.Reason = "conflict", "existing wallet transaction does not match immutable reward snapshot" + return item, nil + } + item.State = "paid" + item.ReceiptID = transactionID + return item, nil +} + +func coinAuditQuery(walletDB string) string { + return fmt.Sprintf(` + SELECT transaction_id, biz_type, status, COALESCE(CAST(metadata_json AS CHAR), '{}') + FROM %s.wallet_transactions + WHERE app_code = ? AND command_id = ?`, walletDB) +} + +func auditResource(ctx context.Context, db *sql.DB, walletDB string, app string, payload rewardPayload, reward domain.DrawReward, index int, total int) (itemAudit, error) { + commandID := rewardCommandID("wheel_resource", payload.DrawID, index, total) + item := itemAudit{CommandID: commandID, Type: reward.RewardType, State: "unpaid"} + resourceID := rewardResourceID(reward) + quantity := reward.RewardCount + if quantity <= 0 { + quantity = 1 + } + durationMS := metadataInt64(reward.MetadataJSON, "duration_ms", "durationMs", "validity_ms", "validityMs") + query := resourceAuditQuery(walletDB) + var grantID, statusValue, grantSource, subjectType, subjectID, reason string + var targetUserID, operatorUserID, actualQuantity, actualDurationMS, actualResourceID int64 + err := db.QueryRowContext(ctx, query, app, commandID).Scan(&grantID, &statusValue, &targetUserID, &grantSource, &subjectType, &subjectID, &reason, &operatorUserID, &actualQuantity, &actualDurationMS, &actualResourceID) + if err != nil { + if errors.Is(err, sql.ErrNoRows) { + return item, nil + } + return itemAudit{}, err + } + if resourceID <= 0 || statusValue != "succeeded" || targetUserID != payload.UserID || grantSource != "wheel_reward" || subjectType != "resource" || subjectID != strconv.FormatInt(resourceID, 10) || reason != "wheel_reward" || operatorUserID != payload.UserID || actualQuantity != quantity || actualDurationMS != durationMS || actualResourceID != resourceID { + item.State, item.Reason = "conflict", "existing resource grant does not match immutable reward snapshot" + return item, nil + } + item.State = "paid" + item.ReceiptID = grantID + return item, nil +} + +func resourceAuditQuery(walletDB string) string { + // 每个 SELECT 列只出现一次且与 Scan 顺序一一对应；显式逐列排版避免核账命令在生产才暴露漏逗号或重复列。 + return fmt.Sprintf(` + SELECT rg.grant_id, + rg.status, + rg.target_user_id, + rg.grant_source, + rg.grant_subject_type, + rg.grant_subject_id, + rg.reason, + rg.operator_user_id, + COALESCE(rgi.quantity, 0), + COALESCE(rgi.duration_ms, 0), + COALESCE(rgi.resource_id, 0) + FROM %s.resource_grants rg + LEFT JOIN %s.resource_grant_items rgi + ON rgi.app_code = rg.app_code AND rgi.grant_id = rg.grant_id + WHERE rg.app_code = ? AND rg.command_id = ? + ORDER BY rgi.grant_item_id + LIMIT 1`, walletDB, walletDB) +} + +func aggregateRowState(items []itemAudit) string { + paid, unpaid, conflict := 0, 0, 0 + for _, item := range items { + switch item.State { + case "paid": + paid++ + case "unpaid": + unpaid++ + default: + conflict++ + } + } + if conflict > 0 { + return "conflict" + } + if paid == len(items) { + return "fully_paid" + } + if unpaid == len(items) { + return "fully_unpaid" + } + return "partially_paid" +} + +func summarize(audits []rowAudit) auditSummary { + summary := auditSummary{Rows: len(audits)} + canonical := make([]string, 0, len(audits)) + for _, audit := range audits { + switch audit.State { + case "fully_paid": + summary.FullyPaid++ + case "fully_unpaid": + summary.FullyUnpaid++ + case "partially_paid": + summary.PartiallyPaid++ + default: + summary.Conflict++ + } + for _, item := range audit.Items { + switch item.State { + case "paid": + summary.PaidItems++ + case "unpaid": + summary.UnpaidItems++ + default: + summary.ConflictItems++ + } + } + encoded, _ := json.Marshal(audit) + canonical = append(canonical, string(encoded)) + } + sort.Strings(canonical) + sum := sha256.Sum256([]byte(strings.Join(canonical, "\n"))) + summary.AuditSHA256 = hex.EncodeToString(sum[:]) + return summary +} + +func rewardCommandID(prefix string, drawID string, index int, total int) string { + if total <= 1 { + return prefix + ":" + drawID + } + return fmt.Sprintf("%s:%s:%d", prefix, drawID, index+1) +} + +func rewardResourceID(reward domain.DrawReward) int64 { + if value := metadataInt64(reward.MetadataJSON, "resource_id", "resourceId"); value > 0 { + return value + } + value, _ := strconv.ParseInt(strings.TrimSpace(reward.RewardID), 10, 64) + return value +} + +func metadataInt64(raw string, keys ...string) int64 { + metadata := map[string]any{} + if err := json.Unmarshal([]byte(strings.TrimSpace(raw)), &metadata); err != nil { + return 0 + } + for _, key := range keys { + if value := jsonInt64(metadata, key); value > 0 { + return value + } + } + return 0 +} + +func jsonInt64(values map[string]any, key string) int64 { + switch value := values[key].(type) { + case float64: + return int64(value) + case json.Number: + parsed, _ := value.Int64() + return parsed + case string: + parsed, _ := strconv.ParseInt(strings.TrimSpace(value), 10, 64) + return parsed + default: + return 0 + } +} + +func jsonString(values map[string]any, key string) string { + value, _ := values[key].(string) + return strings.TrimSpace(value) +} + +func normalizedDatabaseName(value string) string { + value = strings.TrimSpace(value) + if value == "" { + return "" + } + for _, char := range value { + if char == '_' || unicode.IsDigit(char) || unicode.IsLetter(char) { + continue + } + return "" + } + return value +} diff --git a/services/activity-service/cmd/audit-wheel-rewards/main_test.go b/services/activity-service/cmd/audit-wheel-rewards/main_test.go new file mode 100644 index 00000000..a797baad --- /dev/null +++ b/services/activity-service/cmd/audit-wheel-rewards/main_test.go @@ -0,0 +1,306 @@ +package main + +import ( + "context" + "database/sql" + "encoding/json" + "fmt" + "regexp" + "testing" + + "github.com/DATA-DOG/go-sqlmock" + domain "hyapp/services/activity-service/internal/domain/wheel" +) + +func TestAuditRowCoinStates(t *testing.T) { + tests := []struct { + name string + prepare func(sqlmock.Sqlmock, rewardPayload) + wantState string + wantItem string + }{ + { + name: "fully paid", + prepare: func(mock sqlmock.Sqlmock, payload rewardPayload) { + metadata, _ := json.Marshal(map[string]any{ + "target_user_id": payload.UserID, + "amount": int64(200), + "draw_id": payload.DrawID, + "wheel_id": payload.WheelID, + "selected_tier_id": "coin-200", + "reason": "wheel_reward", + "visible_region_id": payload.VisibleRegionID, + }) + mock.ExpectQuery(regexp.QuoteMeta(coinAuditQuery("hyapp_wallet"))). + WithArgs("fami", "wheel_reward:draw-1"). + WillReturnRows(coinAuditRows().AddRow("wtx-1", "wheel_reward", "succeeded", string(metadata))) + }, + wantState: "fully_paid", + wantItem: "paid", + }, + { + name: "fully unpaid uses errors Is", + prepare: func(mock sqlmock.Sqlmock, _ rewardPayload) { + mock.ExpectQuery(regexp.QuoteMeta(coinAuditQuery("hyapp_wallet"))). + WithArgs("fami", "wheel_reward:draw-1"). + WillReturnError(fmt.Errorf("wrapped wallet miss: %w", sql.ErrNoRows)) + }, + wantState: "fully_unpaid", + wantItem: "unpaid", + }, + { + name: "conflict on immutable amount", + prepare: func(mock sqlmock.Sqlmock, payload rewardPayload) { + metadata, _ := json.Marshal(map[string]any{ + "target_user_id": payload.UserID, + "amount": int64(201), + "draw_id": payload.DrawID, + "wheel_id": payload.WheelID, + "selected_tier_id": "coin-200", + "reason": "wheel_reward", + "visible_region_id": payload.VisibleRegionID, + }) + mock.ExpectQuery(regexp.QuoteMeta(coinAuditQuery("hyapp_wallet"))). + WithArgs("fami", "wheel_reward:draw-1"). + WillReturnRows(coinAuditRows().AddRow("wtx-conflict", "wheel_reward", "succeeded", string(metadata))) + }, + wantState: "conflict", + wantItem: "conflict", + }, + { + name: "conflict on biz type", + prepare: func(mock sqlmock.Sqlmock, payload rewardPayload) { + metadata := matchingCoinMetadata(payload) + mock.ExpectQuery(regexp.QuoteMeta(coinAuditQuery("hyapp_wallet"))). + WithArgs("fami", "wheel_reward:draw-1"). + WillReturnRows(coinAuditRows().AddRow("wtx-wrong-biz", "task_reward", "succeeded", metadata)) + }, + wantState: "conflict", + wantItem: "conflict", + }, + { + name: "conflict on visible region", + prepare: func(mock sqlmock.Sqlmock, payload rewardPayload) { + metadata := matchingCoinMetadata(payload) + var values map[string]any + _ = json.Unmarshal([]byte(metadata), &values) + values["visible_region_id"] = int64(99) + mismatch, _ := json.Marshal(values) + mock.ExpectQuery(regexp.QuoteMeta(coinAuditQuery("hyapp_wallet"))). + WithArgs("fami", "wheel_reward:draw-1"). + WillReturnRows(coinAuditRows().AddRow("wtx-wrong-region", "wheel_reward", "succeeded", string(mismatch))) + }, + wantState: "conflict", + wantItem: "conflict", + }, + } + + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + db, mock, err := sqlmock.New() + if err != nil { + t.Fatalf("create sqlmock: %v", err) + } + defer db.Close() + payload := rewardPayload{ + AppCode: "fami", + DrawID: "draw-1", + UserID: 1001, + WheelID: "classic", + VisibleRegionID: 9, + Rewards: []domain.DrawReward{{SelectedTierID: "coin-200", RewardType: domain.RewardTypeCoin, RewardCoins: 200}}, + } + test.prepare(mock, payload) + audit, err := auditRow(context.Background(), db, "hyapp_wallet", auditOutboxRow(t, payload)) + if err != nil { + t.Fatalf("auditRow returned error: %v", err) + } + if audit.State != test.wantState || len(audit.Items) != 1 || audit.Items[0].State != test.wantItem { + t.Fatalf("unexpected audit: %+v", audit) + } + if err := mock.ExpectationsWereMet(); err != nil { + t.Fatalf("unmet SQL expectations: %v", err) + } + }) + } +} + +func TestAuditRowResourceStates(t *testing.T) { + tests := []struct { + name string + prepare func(sqlmock.Sqlmock) + wantState string + wantItem string + }{ + { + name: "fully paid", + prepare: func(mock sqlmock.Sqlmock) { + mock.ExpectQuery(regexp.QuoteMeta(resourceAuditQuery("hyapp_wallet"))). + WithArgs("fami", "wheel_resource:draw-1"). + WillReturnRows(resourceAuditRows().AddRow("rgr-1", "succeeded", int64(1001), "wheel_reward", "resource", "326", "wheel_reward", int64(1001), int64(2), int64(5000), int64(326))) + }, + wantState: "fully_paid", + wantItem: "paid", + }, + { + name: "fully unpaid uses errors Is", + prepare: func(mock sqlmock.Sqlmock) { + mock.ExpectQuery(regexp.QuoteMeta(resourceAuditQuery("hyapp_wallet"))). + WithArgs("fami", "wheel_resource:draw-1"). + WillReturnError(fmt.Errorf("wrapped resource miss: %w", sql.ErrNoRows)) + }, + wantState: "fully_unpaid", + wantItem: "unpaid", + }, + { + name: "conflict rejects legacy active status", + prepare: func(mock sqlmock.Sqlmock) { + mock.ExpectQuery(regexp.QuoteMeta(resourceAuditQuery("hyapp_wallet"))). + WithArgs("fami", "wheel_resource:draw-1"). + WillReturnRows(resourceAuditRows().AddRow("rgr-conflict", "active", int64(1001), "wheel_reward", "resource", "326", "wheel_reward", int64(1001), int64(2), int64(5000), int64(326))) + }, + wantState: "conflict", + wantItem: "conflict", + }, + } + + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + db, mock, err := sqlmock.New() + if err != nil { + t.Fatalf("create sqlmock: %v", err) + } + defer db.Close() + payload := rewardPayload{ + AppCode: "fami", + DrawID: "draw-1", + UserID: 1001, + WheelID: "classic", + Rewards: []domain.DrawReward{{ + SelectedTierID: "gift-326", + RewardType: domain.RewardTypeGift, + RewardID: "326", + RewardCount: 2, + MetadataJSON: `{"resource_id":326,"duration_ms":5000}`, + }}, + } + test.prepare(mock) + audit, err := auditRow(context.Background(), db, "hyapp_wallet", auditOutboxRow(t, payload)) + if err != nil { + t.Fatalf("auditRow returned error: %v", err) + } + if audit.State != test.wantState || len(audit.Items) != 1 || audit.Items[0].State != test.wantItem { + t.Fatalf("unexpected audit: %+v", audit) + } + if err := mock.ExpectationsWereMet(); err != nil { + t.Fatalf("unmet SQL expectations: %v", err) + } + }) + } +} + +func TestLoadUnsettledRowsUsesAppScopedIndexesAndGlobalLimit(t *testing.T) { + db, mock, err := sqlmock.New() + if err != nil { + t.Fatalf("create sqlmock: %v", err) + } + defer db.Close() + const limit = 3 + columns := []string{"app_code", "outbox_id", "payload", "status", "retry_count", "created_at_ms"} + + // normalizeAuditAppCodes sorts and deduplicates, so SQL order is deterministic regardless of flag order. + mock.ExpectQuery(regexp.QuoteMeta(unsettledRetryQuery())). + WithArgs("fami", domain.EventTypeWheelRewardSettlement, domain.OutboxStatusPending, domain.OutboxStatusRetryable, limit). + WillReturnRows(sqlmock.NewRows(columns). + AddRow("fami", "fami-pending", []byte(`{}`), domain.OutboxStatusPending, 0, int64(20))) + mock.ExpectQuery(regexp.QuoteMeta(unsettledDeliveringQuery())). + WithArgs("fami", domain.EventTypeWheelRewardSettlement, domain.OutboxStatusDelivering, limit). + WillReturnRows(sqlmock.NewRows(columns). + AddRow("fami", "fami-delivering", []byte(`{}`), domain.OutboxStatusDelivering, 1, int64(10))) + mock.ExpectQuery(regexp.QuoteMeta(unsettledRetryQuery())). + WithArgs("lalu", domain.EventTypeWheelRewardSettlement, domain.OutboxStatusPending, domain.OutboxStatusRetryable, 1). + WillReturnRows(sqlmock.NewRows(columns). + AddRow("lalu", "lalu-oldest", []byte(`{}`), domain.OutboxStatusRetryable, 2, int64(5))) + mock.ExpectQuery(regexp.QuoteMeta(unsettledDeliveringQuery())). + WithArgs("lalu", domain.EventTypeWheelRewardSettlement, domain.OutboxStatusDelivering, 1). + WillReturnRows(sqlmock.NewRows(columns). + AddRow("lalu", "lalu-delivering", []byte(`{}`), domain.OutboxStatusDelivering, 1, int64(15))) + + rows, err := loadUnsettledRows(context.Background(), db, []string{"lalu", "fami", "fami"}, limit) + if err != nil { + t.Fatalf("loadUnsettledRows returned error: %v", err) + } + if len(rows) != limit { + t.Fatalf("global limit mismatch: got=%d want=%d rows=%+v", len(rows), limit, rows) + } + wantIDs := []string{"fami-delivering", "fami-pending", "lalu-oldest"} + for index, wantID := range wantIDs { + if rows[index].OutboxID != wantID { + t.Fatalf("deterministic order[%d]=%q want=%q rows=%+v", index, rows[index].OutboxID, wantID, rows) + } + } + if err := mock.ExpectationsWereMet(); err != nil { + t.Fatalf("unmet SQL expectations: %v", err) + } +} + +func TestNormalizeAuditAppCodesRequiresRepeatedExplicitScope(t *testing.T) { + if _, err := normalizeAuditAppCodes(nil); err == nil { + t.Fatal("missing --app-code must be rejected before opening or querying MySQL") + } + values, err := normalizeAuditAppCodes([]string{" LALU ", "fami", "lalu", "huwaa"}) + if err != nil { + t.Fatalf("normalizeAuditAppCodes returned error: %v", err) + } + want := []string{"fami", "huwaa", "lalu"} + if len(values) != len(want) { + t.Fatalf("normalized app count=%d want=%d values=%+v", len(values), len(want), values) + } + for index := range want { + if values[index] != want[index] { + t.Fatalf("normalized app[%d]=%q want=%q", index, values[index], want[index]) + } + } +} + +func auditOutboxRow(t *testing.T, payload rewardPayload) outboxRow { + t.Helper() + body, err := json.Marshal(payload) + if err != nil { + t.Fatalf("marshal payload: %v", err) + } + return outboxRow{AppCode: payload.AppCode, OutboxID: "wheel_reward_" + payload.DrawID, Payload: body, Status: domain.OutboxStatusPending} +} + +func resourceAuditRows() *sqlmock.Rows { + return sqlmock.NewRows([]string{ + "grant_id", + "status", + "target_user_id", + "grant_source", + "grant_subject_type", + "grant_subject_id", + "reason", + "operator_user_id", + "quantity", + "duration_ms", + "resource_id", + }) +} + +func coinAuditRows() *sqlmock.Rows { + return sqlmock.NewRows([]string{"transaction_id", "biz_type", "status", "metadata_json"}) +} + +func matchingCoinMetadata(payload rewardPayload) string { + body, _ := json.Marshal(map[string]any{ + "target_user_id": payload.UserID, + "amount": int64(200), + "draw_id": payload.DrawID, + "wheel_id": payload.WheelID, + "selected_tier_id": "coin-200", + "reason": "wheel_reward", + "visible_region_id": payload.VisibleRegionID, + }) + return string(body) +} diff --git a/services/activity-service/configs/config.docker.yaml b/services/activity-service/configs/config.docker.yaml index 8c6ff796..7ceff219 100644 --- a/services/activity-service/configs/config.docker.yaml +++ b/services/activity-service/configs/config.docker.yaml @@ -30,6 +30,13 @@ user_leaderboard_worker: redis_password: "" redis_db: 0 key_prefix: "activity:user_leaderboard" +wheel_reward_worker: + enabled: true + app_codes: ["lalu", "fami", "huwaa"] + poll_interval: "1s" + batch_size: 50 + lock_ttl: "30s" + max_retry: 16 message_action_confirm_worker: enabled: true outbox_poll_interval: "1s" diff --git a/services/activity-service/configs/config.tencent.example.yaml b/services/activity-service/configs/config.tencent.example.yaml index 1492aeb3..ec6d0461 100644 --- a/services/activity-service/configs/config.tencent.example.yaml +++ b/services/activity-service/configs/config.tencent.example.yaml @@ -30,6 +30,13 @@ user_leaderboard_worker: redis_password: "TENCENT_REDIS_PASSWORD" redis_db: 0 key_prefix: "activity:user_leaderboard" +wheel_reward_worker: + enabled: true + app_codes: ["lalu", "fami", "huwaa"] + poll_interval: "1s" + batch_size: 50 + lock_ttl: "30s" + max_retry: 16 message_action_confirm_worker: enabled: true outbox_poll_interval: "1s" diff --git a/services/activity-service/configs/config.yaml b/services/activity-service/configs/config.yaml index ae81e2eb..546d6ef3 100644 --- a/services/activity-service/configs/config.yaml +++ b/services/activity-service/configs/config.yaml @@ -30,6 +30,13 @@ user_leaderboard_worker: redis_password: "" redis_db: 0 key_prefix: "activity:user_leaderboard" +wheel_reward_worker: + enabled: true + app_codes: ["lalu", "fami", "huwaa"] + poll_interval: "1s" + batch_size: 50 + lock_ttl: "30s" + max_retry: 16 message_action_confirm_worker: enabled: true outbox_poll_interval: "1s" diff --git a/services/activity-service/internal/app/app.go b/services/activity-service/internal/app/app.go index b3e862b9..f8ef22dc 100644 --- a/services/activity-service/internal/app/app.go +++ b/services/activity-service/internal/app/app.go @@ -21,6 +21,7 @@ import ( firstrechargeservice "hyapp/services/activity-service/internal/service/firstrecharge" inviteactivityservice "hyapp/services/activity-service/internal/service/inviteactivity" messageservice "hyapp/services/activity-service/internal/service/message" + wheelservice "hyapp/services/activity-service/internal/service/wheel" mysqlstorage "hyapp/services/activity-service/internal/storage/mysql" ) @@ -37,10 +38,13 @@ type App struct { cumulativeRecharge *cumulativerechargeservice.Service inviteActivityReward *inviteactivityservice.Service actionConfirm *messageservice.ActionConfirmService + wheel *wheelservice.Service broadcastWorkerEnabled bool messageActionWorkerEnabled bool + wheelRewardWorkerEnabled bool workerNodeID string messageActionWorkerOptions config.MessageActionConfirmWorkerConfig + wheelRewardWorkerOptions config.WheelRewardWorkerConfig messageActionOutboxTopic string userLeaderboardRedisClose func() error mqConsumers []*rocketmqx.Consumer @@ -133,10 +137,13 @@ func New(cfg config.Config) (*App, error) { cumulativeRecharge: services.cumulativeRecharge, inviteActivityReward: services.inviteActivityReward, actionConfirm: services.actionConfirm, + wheel: services.wheel, broadcastWorkerEnabled: cfg.Broadcast.Enabled, messageActionWorkerEnabled: cfg.MessageActionConfirmWorker.Enabled, + wheelRewardWorkerEnabled: cfg.WheelRewardWorker.Enabled, workerNodeID: cfg.NodeID, messageActionWorkerOptions: cfg.MessageActionConfirmWorker, + wheelRewardWorkerOptions: cfg.WheelRewardWorker, messageActionOutboxTopic: cfg.RocketMQ.MessageActionOutbox.Topic, userLeaderboardRedisClose: userLeaderboardRedisClose, mqConsumers: mqConsumers, @@ -166,6 +173,18 @@ func (a *App) Run() error { a.runMessageActionOutboxWorker(ctx) }) } + if a.wheelRewardWorkerEnabled && a.wheel != nil && a.workers != nil { + a.workers.Go(func(ctx context.Context) { + a.wheel.RunRewardWorker(ctx, wheelservice.RewardWorkerOptions{ + WorkerID: a.workerNodeID + ":wheel-reward", + AppCodes: a.wheelRewardWorkerOptions.AppCodes, + PollInterval: a.wheelRewardWorkerOptions.PollInterval, + BatchSize: a.wheelRewardWorkerOptions.BatchSize, + LockTTL: a.wheelRewardWorkerOptions.LockTTL, + MaxRetry: a.wheelRewardWorkerOptions.MaxRetry, + }) + }) + } err := a.server.Serve(a.listener) if a.workers != nil { a.workers.StopAndWait() diff --git a/services/activity-service/internal/config/config.go b/services/activity-service/internal/config/config.go index a3e5d87e..a8c598ee 100644 --- a/services/activity-service/internal/config/config.go +++ b/services/activity-service/internal/config/config.go @@ -40,6 +40,8 @@ type Config struct { TaskEventWorker TaskEventWorkerConfig `yaml:"task_event_worker"` // UserLeaderboardWorker 控制用户送礼榜单对 wallet_outbox 送礼事实的 Redis 投影。 UserLeaderboardWorker UserLeaderboardWorkerConfig `yaml:"user_leaderboard_worker"` + // WheelRewardWorker 补偿抽奖事务已经确定、但同步钱包发奖未完成的 WheelRewardSettlement outbox。 + WheelRewardWorker WheelRewardWorkerConfig `yaml:"wheel_reward_worker"` // TencentIM 是 activity-service 发送全局/区域播报群消息的 REST 配置。 TencentIM TencentIMConfig `yaml:"tencent_im"` // Broadcast 控制 IM 播报群 reconciler、outbox worker 和贵重礼物阈值。 @@ -120,6 +122,17 @@ type UserLeaderboardWorkerConfig struct { KeyPrefix string `yaml:"key_prefix"` } +// WheelRewardWorkerConfig 控制转盘发奖 outbox 的租户分区、锁和重试边界。 +// AppCodes 必须显式覆盖线上业务 App；worker 每轮逐 App claim，禁止空 context 默认只处理 lalu。 +type WheelRewardWorkerConfig struct { + Enabled bool `yaml:"enabled"` + AppCodes []string `yaml:"app_codes"` + PollInterval time.Duration `yaml:"poll_interval"` + BatchSize int `yaml:"batch_size"` + LockTTL time.Duration `yaml:"lock_ttl"` + MaxRetry int `yaml:"max_retry"` +} + // TencentIMConfig 描述 activity-service 调用腾讯云 IM REST API 的配置。 type TencentIMConfig struct { Enabled bool `yaml:"enabled"` @@ -266,6 +279,14 @@ func Default() Config { UserLeaderboardWorker: UserLeaderboardWorkerConfig{ Enabled: false, }, + WheelRewardWorker: WheelRewardWorkerConfig{ + Enabled: true, + AppCodes: []string{"lalu", "fami", "huwaa"}, + PollInterval: time.Second, + BatchSize: 50, + LockTTL: 30 * time.Second, + MaxRetry: 16, + }, MessageActionConfirmWorker: MessageActionConfirmWorkerConfig{ Enabled: true, OutboxPollInterval: time.Second, @@ -447,6 +468,22 @@ func Load(path string) (Config, error) { } cfg.UserLeaderboardWorker.RedisAddr = strings.TrimSpace(cfg.UserLeaderboardWorker.RedisAddr) cfg.UserLeaderboardWorker.KeyPrefix = strings.TrimSpace(cfg.UserLeaderboardWorker.KeyPrefix) + cfg.WheelRewardWorker.AppCodes = normalizeAppCodes(cfg.WheelRewardWorker.AppCodes) + if len(cfg.WheelRewardWorker.AppCodes) == 0 { + cfg.WheelRewardWorker.AppCodes = append([]string(nil), Default().WheelRewardWorker.AppCodes...) + } + if cfg.WheelRewardWorker.PollInterval <= 0 { + cfg.WheelRewardWorker.PollInterval = time.Second + } + if cfg.WheelRewardWorker.BatchSize <= 0 { + cfg.WheelRewardWorker.BatchSize = 50 + } + if cfg.WheelRewardWorker.LockTTL <= 0 { + cfg.WheelRewardWorker.LockTTL = 30 * time.Second + } + if cfg.WheelRewardWorker.MaxRetry <= 0 { + cfg.WheelRewardWorker.MaxRetry = 16 + } rocketMQ, err := normalizeRocketMQConfig(cfg.RocketMQ) if err != nil { return Config{}, err diff --git a/services/activity-service/internal/config/config_test.go b/services/activity-service/internal/config/config_test.go index 1a59c7da..202a11df 100644 --- a/services/activity-service/internal/config/config_test.go +++ b/services/activity-service/internal/config/config_test.go @@ -27,6 +27,9 @@ func TestLoadLocalEnablesOutboxMQConsumers(t *testing.T) { if cfg.UserLeaderboardWorker.RedisAddr != "127.0.0.1:13379" || cfg.UserLeaderboardWorker.KeyPrefix != "activity:user_leaderboard" { t.Fatalf("local user leaderboard worker redis config mismatch: %+v", cfg.UserLeaderboardWorker) } + if !cfg.WheelRewardWorker.Enabled || len(cfg.WheelRewardWorker.AppCodes) != 3 || cfg.WheelRewardWorker.AppCodes[1] != "fami" || cfg.WheelRewardWorker.BatchSize <= 0 || cfg.WheelRewardWorker.LockTTL <= 0 { + t.Fatalf("local wheel reward compensation must cover every app with bounded claim settings: %+v", cfg.WheelRewardWorker) + } if cfg.RocketMQ.WalletOutbox.InviteActivityConsumerGroup == "" || cfg.RocketMQ.WalletOutbox.UserLeaderboardConsumerGroup == "" || cfg.RocketMQ.WalletOutbox.VIPRebateNoticeConsumerGroup == "" || cfg.RocketMQ.UserOutbox.InviteActivityConsumerGroup == "" { t.Fatalf("local config must configure invite activity consumer groups: wallet=%+v user=%+v", cfg.RocketMQ.WalletOutbox, cfg.RocketMQ.UserOutbox) } @@ -58,6 +61,9 @@ func TestLoadTencentExampleEnablesRoomOutboxMQ(t *testing.T) { if !cfg.UserLeaderboardWorker.Enabled || cfg.UserLeaderboardWorker.RedisAddr == "" || cfg.UserLeaderboardWorker.RedisPassword == "" || cfg.RocketMQ.WalletOutbox.UserLeaderboardConsumerGroup == "" { t.Fatalf("tencent example must configure user leaderboard worker: worker=%+v wallet=%+v", cfg.UserLeaderboardWorker, cfg.RocketMQ.WalletOutbox) } + if !cfg.WheelRewardWorker.Enabled || len(cfg.WheelRewardWorker.AppCodes) != 3 || cfg.WheelRewardWorker.AppCodes[2] != "huwaa" || cfg.WheelRewardWorker.MaxRetry != 16 { + t.Fatalf("tencent example must configure multi-app wheel reward compensation: %+v", cfg.WheelRewardWorker) + } } func TestLoadRejectsLuckyGiftConsumerWithoutBroadcastDelivery(t *testing.T) { diff --git a/services/activity-service/internal/domain/wheel/wheel.go b/services/activity-service/internal/domain/wheel/wheel.go index eec9c6f7..4753e83a 100644 --- a/services/activity-service/internal/domain/wheel/wheel.go +++ b/services/activity-service/internal/domain/wheel/wheel.go @@ -13,6 +13,12 @@ const ( RewardTypeDress = "dress" EventTypeWheelRewardSettlement = "WheelRewardSettlement" + + OutboxStatusPending = "pending" + OutboxStatusDelivering = "delivering" + OutboxStatusRetryable = "retryable" + OutboxStatusDelivered = "delivered" + OutboxStatusFailed = "failed" ) // RuleConfig 是转盘配置的不可变版本快照。 @@ -113,3 +119,19 @@ type DrawQuery struct { Page int32 PageSize int32 } + +// RewardOutboxRecord 是转盘发奖补偿 worker 的持久化工作项。 +// Payload 始终保留抽奖事务写入时的不可变奖品快照；补偿时不能回读当前奖档配置，否则运营改配置后会改变历史应发权益。 +type RewardOutboxRecord struct { + AppCode string + OutboxID string + Payload []byte + Status string + RetryCount int + NextRetryAtMS int64 + LockedBy string + LockUntilMS int64 + LastError string + CreatedAtMS int64 + UpdatedAtMS int64 +} diff --git a/services/activity-service/internal/service/agencyopening/service.go b/services/activity-service/internal/service/agencyopening/service.go index 9d27130e..34ce249e 100644 --- a/services/activity-service/internal/service/agencyopening/service.go +++ b/services/activity-service/internal/service/agencyopening/service.go @@ -221,6 +221,11 @@ func (s *Service) HandleRoomEvent(ctx context.Context, envelope *roomeventsv1.Ev } eventCtx := appcode.WithContext(ctx, envelope.GetAppCode()) ownerUserID, err := s.fetchRoomOwner(eventCtx, roomID) + if xerr.IsCode(err, xerr.NotFound) { + // RoomGiftSent 已经是 room-service 提交过的历史事实；若消费时房间已被删除，当前 owner 已不可恢复， + // 该事件对“开业房主流水”只能收敛为 skipped。这里只吞精确 NotFound，网络、超时和内部错误仍返回 MQ 重试，避免把临时依赖故障误判成业务终态。 + return 0, nil + } if err != nil || ownerUserID <= 0 { return 0, err } diff --git a/services/activity-service/internal/service/agencyopening/service_test.go b/services/activity-service/internal/service/agencyopening/service_test.go index ce87761b..92f46f50 100644 --- a/services/activity-service/internal/service/agencyopening/service_test.go +++ b/services/activity-service/internal/service/agencyopening/service_test.go @@ -6,6 +6,8 @@ import ( "time" "google.golang.org/grpc" + "google.golang.org/grpc/codes" + "google.golang.org/grpc/status" "google.golang.org/protobuf/proto" roomeventsv1 "hyapp.local/api/proto/events/room/v1" roomv1 "hyapp.local/api/proto/room/v1" @@ -109,6 +111,46 @@ func TestHandleRoomEventScoresApplicantRoomOwner(t *testing.T) { } } +func TestHandleRoomEventTreatsDeletedRoomAsTerminalSkip(t *testing.T) { + repo := &agencyOpeningFakeRepo{consumeStatus: domain.EventStatusConsumed} + room := &agencyOpeningFakeRoomClient{err: status.Error(codes.NotFound, "room not found")} + svc := New(repo, nil, nil, room) + body, err := proto.Marshal(&roomeventsv1.RoomGiftSent{SenderUserId: 9, TargetUserId: 88, GiftValue: 900}) + if err != nil { + t.Fatalf("marshal gift: %v", err) + } + + count, err := svc.HandleRoomEvent(context.Background(), &roomeventsv1.EventEnvelope{ + EventId: "deleted-room-gift", RoomId: "deleted-room", EventType: "RoomGiftSent", AppCode: "lalu", Body: body, + }) + if err != nil || count != 0 { + t.Fatalf("deleted room must be terminal skipped, count=%d err=%v", count, err) + } + if repo.consumed.EventID != "" { + t.Fatalf("deleted room must not create agency turnover: %+v", repo.consumed) + } +} + +func TestHandleRoomEventKeepsTransientRoomFailureRetryable(t *testing.T) { + repo := &agencyOpeningFakeRepo{consumeStatus: domain.EventStatusConsumed} + room := &agencyOpeningFakeRoomClient{err: status.Error(codes.Unavailable, "room service unavailable")} + svc := New(repo, nil, nil, room) + body, err := proto.Marshal(&roomeventsv1.RoomGiftSent{SenderUserId: 9, TargetUserId: 88, GiftValue: 900}) + if err != nil { + t.Fatalf("marshal gift: %v", err) + } + + count, err := svc.HandleRoomEvent(context.Background(), &roomeventsv1.EventEnvelope{ + EventId: "transient-room-gift", RoomId: "room-1", EventType: "RoomGiftSent", AppCode: "lalu", Body: body, + }) + if err == nil || status.Code(err) != codes.Unavailable || count != 0 { + t.Fatalf("transient room failure must remain retryable, count=%d err=%v", count, err) + } + if repo.consumed.EventID != "" { + t.Fatalf("transient room failure must not create partial projection: %+v", repo.consumed) + } +} + type agencyOpeningFakeRepo struct { currentCycle domain.Cycle hasCurrent bool @@ -179,10 +221,14 @@ func (s agencyOpeningFakeAgencySource) ResolveOwnerAgency(context.Context, int64 type agencyOpeningFakeRoomClient struct { ownerUserID int64 roomID string + err error } func (c *agencyOpeningFakeRoomClient) AdminGetRoom(_ context.Context, req *roomv1.AdminGetRoomRequest, _ ...grpc.CallOption) (*roomv1.AdminGetRoomResponse, error) { c.roomID = req.GetRoomId() + if c.err != nil { + return nil, c.err + } return &roomv1.AdminGetRoomResponse{ Room: &roomv1.AdminRoomListItem{OwnerUserId: c.ownerUserID}, }, nil diff --git a/services/activity-service/internal/service/wheel/service.go b/services/activity-service/internal/service/wheel/service.go index 17c5b24f..9368e70d 100644 --- a/services/activity-service/internal/service/wheel/service.go +++ b/services/activity-service/internal/service/wheel/service.go @@ -2,6 +2,7 @@ package wheel import ( "context" + "crypto/sha256" "encoding/json" "fmt" "log/slog" @@ -24,6 +25,9 @@ type Repository interface { ListWheelDraws(ctx context.Context, query domain.DrawQuery) ([]domain.DrawResult, int64, error) GetWheelDrawSummary(ctx context.Context, query domain.DrawQuery) (domain.DrawSummary, error) MarkWheelDrawsGranted(ctx context.Context, appCode string, drawIDs []string, transactionID string, nowMS int64) error + ClaimWheelRewardOutbox(ctx context.Context, workerID string, nowMS int64, lockUntilMS int64, limit int, maxRetry int) ([]domain.RewardOutboxRecord, error) + MarkWheelRewardOutboxDelivered(ctx context.Context, outboxID string, workerID string, drawIDs []string, transactionID string, nowMS int64) error + MarkWheelRewardOutboxFailed(ctx context.Context, outboxID string, workerID string, nowMS int64, nextRetryAtMS int64, maxRetry int, lastError string) error } type WalletClient interface { @@ -128,26 +132,41 @@ func (s *Service) fulfillWheelRewardFastPath(ctx context.Context, cmd domain.Dra if result.RewardStatus == domain.StatusGranted { return result } - if s.wallet == nil { - logx.Warn(ctx, "wheel_reward_wallet_not_configured", slog.String("draw_id", result.DrawID)) + fulfilled, err := s.fulfillWheelReward(ctx, cmd, result) + if err != nil { + // 抽奖结果已经在事务内确认，发放失败时不能回滚抽奖事实；保留 pending/outbox，补偿 worker 会使用相同 command_id 继续核账和补发。 + logx.Error(ctx, "wheel_reward_fast_path_failed", err, + slog.String("draw_id", result.DrawID), + slog.String("command_id", result.CommandID), + ) return result } + result = fulfilled + drawIDs := []string{result.DrawID} + if result.DrawID == "" { + drawIDs = append([]string(nil), result.DrawIDs...) + } + if err := s.repository.MarkWheelDrawsGranted(appcode.WithContext(context.WithoutCancel(ctx), appcode.FromContext(ctx)), appcode.FromContext(ctx), drawIDs, result.WalletTransactionID, s.now().UTC().UnixMilli()); err != nil { + logx.Error(ctx, "wheel_reward_fast_path_mark_granted_failed", err, slog.String("draw_id", result.DrawID), slog.String("wallet_transaction_id", result.WalletTransactionID)) + } + return result +} + +// fulfillWheelReward 调用 wallet owner 的幂等账务命令完成一组奖品发放。 +// CreditWheelReward/GrantResource 在钱包事务内都会先按稳定 command_id 和 request hash 查询既有交易或权益；因此 worker 在超时、进程崩溃或标记 delivered 失败后重放时，先得到原收据而不是再次入账。 +func (s *Service) fulfillWheelReward(ctx context.Context, cmd domain.DrawCommand, result domain.DrawResult) (domain.DrawResult, error) { + if s.wallet == nil { + return result, xerr.New(xerr.Unavailable, "wheel reward wallet client is not configured") + } rewards := wheelResultRewards(result) if len(rewards) == 0 { - return result + return result, xerr.New(xerr.InvalidArgument, "wheel reward snapshot is empty") } receipts := make([]string, 0, len(rewards)) for index, reward := range rewards { receipt, coinBalance, err := s.fulfillWheelRewardItem(ctx, cmd, result, reward, index, len(rewards)) if err != nil { - // 抽奖结果已经在事务内确认，发放失败时不能回滚抽奖事实；保留 pending/outbox，补偿 worker 或重试可以用同一 command_id 幂等补发。 - logx.Error(ctx, "wheel_reward_fast_path_failed", err, - slog.String("draw_id", result.DrawID), - slog.String("command_id", result.CommandID), - slog.String("reward_type", reward.RewardType), - slog.String("selected_tier_id", reward.SelectedTierID), - ) - return result + return result, fmt.Errorf("fulfill reward item %d type=%s tier=%s: %w", index+1, reward.RewardType, reward.SelectedTierID, err) } if receipt != "" { receipts = append(receipts, receipt) @@ -157,15 +176,20 @@ func (s *Service) fulfillWheelRewardFastPath(ctx context.Context, cmd domain.Dra } } result.RewardStatus = domain.StatusGranted - result.WalletTransactionID = strings.Join(receipts, ",") - drawIDs := []string{result.DrawID} - if result.DrawID == "" { - drawIDs = append([]string(nil), result.DrawIDs...) + result.WalletTransactionID = wheelRewardReceiptReference(receipts) + return result, nil +} + +func wheelRewardReceiptReference(receipts []string) string { + joined := strings.Join(receipts, ",") + const rewardTransactionIDMaxLength = 128 + if len(joined) <= rewardTransactionIDMaxLength { + return joined } - if err := s.repository.MarkWheelDrawsGranted(appcode.WithContext(context.WithoutCancel(ctx), appcode.FromContext(ctx)), appcode.FromContext(ctx), drawIDs, result.WalletTransactionID, s.now().UTC().UnixMilli()); err != nil { - logx.Error(ctx, "wheel_reward_fast_path_mark_granted_failed", err, slog.String("draw_id", result.DrawID), slog.String("wallet_transaction_id", result.WalletTransactionID)) - } - return result + // wheel_draw_records 的历史字段是 VARCHAR(128)，而钱包 transaction/grant ID 各自约 68 字符； + // 多奖品直接逗号拼接会让钱包已经全部发放后仍无法标记 granted。超长集合保存稳定摘要，逐项原收据仍由钱包 owner 按派生 command_id 可审计查询。 + sum := sha256.Sum256([]byte(joined)) + return fmt.Sprintf("batch:%d:%x", len(receipts), sum) } func (s *Service) fulfillWheelRewardItem(ctx context.Context, cmd domain.DrawCommand, result domain.DrawResult, reward domain.DrawReward, index, total int) (string, int64, error) { diff --git a/services/activity-service/internal/service/wheel/service_test.go b/services/activity-service/internal/service/wheel/service_test.go index b4aec8ea..bff56fd8 100644 --- a/services/activity-service/internal/service/wheel/service_test.go +++ b/services/activity-service/internal/service/wheel/service_test.go @@ -2,6 +2,8 @@ package wheel import ( "context" + "errors" + "strings" "testing" "time" @@ -96,11 +98,35 @@ func TestDrawFulfillsBatchCoinAndResourceRewards(t *testing.T) { if result.CoinBalanceAfter != 9900 { t.Fatalf("coin balance mismatch: %d", result.CoinBalanceAfter) } + if len(result.WalletTransactionID) > 128 { + t.Fatalf("batch reward receipt reference exceeds wheel_draw_records boundary: %d", len(result.WalletTransactionID)) + } +} + +func TestWheelRewardReceiptReferenceCompactsMultiRewardReceipts(t *testing.T) { + receipts := []string{ + "wtx_aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa", + "rgr_bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb", + } + first := wheelRewardReceiptReference(receipts) + second := wheelRewardReceiptReference(receipts) + if first != second || !strings.HasPrefix(first, "batch:2:") || len(first) > 128 { + t.Fatalf("receipt reference must be stable and bounded: first=%q second=%q", first, second) + } } type fakeWheelRepository struct { - executeResult domain.DrawResult - markGrantedDrawIDs []string + executeResult domain.DrawResult + markGrantedDrawIDs []string + claimRecords []domain.RewardOutboxRecord + claimApps []string + deliveredOutboxID string + deliveredWorkerID string + deliveredDrawIDs []string + deliveredTransactionID string + deliverErr error + failedOutboxID string + failedReason string } func (r *fakeWheelRepository) PublishWheelRuleConfig(context.Context, domain.RuleConfig, int64) (domain.RuleConfig, error) { @@ -128,6 +154,27 @@ func (r *fakeWheelRepository) MarkWheelDrawsGranted(_ context.Context, _ string, return nil } +func (r *fakeWheelRepository) ClaimWheelRewardOutbox(ctx context.Context, _ string, _ int64, _ int64, _ int, _ int) ([]domain.RewardOutboxRecord, error) { + r.claimApps = append(r.claimApps, appcode.FromContext(ctx)) + records := append([]domain.RewardOutboxRecord(nil), r.claimRecords...) + r.claimRecords = nil + return records, nil +} + +func (r *fakeWheelRepository) MarkWheelRewardOutboxDelivered(_ context.Context, outboxID string, workerID string, drawIDs []string, transactionID string, _ int64) error { + r.deliveredOutboxID = outboxID + r.deliveredWorkerID = workerID + r.deliveredDrawIDs = append([]string(nil), drawIDs...) + r.deliveredTransactionID = transactionID + return r.deliverErr +} + +func (r *fakeWheelRepository) MarkWheelRewardOutboxFailed(_ context.Context, outboxID string, _ string, _ int64, _ int64, _ int, lastError string) error { + r.failedOutboxID = outboxID + r.failedReason = lastError + return nil +} + type fakeWheelWallet struct { coinCredits []*walletv1.CreditWheelRewardRequest resourceGrants []*walletv1.GrantResourceRequest @@ -146,3 +193,85 @@ func (w *fakeWheelWallet) GrantResource(_ context.Context, req *walletv1.GrantRe w.resourceGrants = append(w.resourceGrants, req) return &walletv1.ResourceGrantResponse{Grant: &walletv1.ResourceGrant{GrantId: "grant-" + req.GetCommandId()}}, nil } + +func TestProcessPendingRewardsReplaysStableWalletCommandAndMarksDelivered(t *testing.T) { + repository := &fakeWheelRepository{claimRecords: []domain.RewardOutboxRecord{{ + AppCode: "fami", + OutboxID: "wheel_reward_wheel_draw_replay", + // 线上历史 payload 的 rewards 使用 Go 默认字段名；补偿解码必须兼容该不可变快照，不能改读当前奖档。 + Payload: []byte(`{"app_code":"fami","draw_id":"wheel_draw_replay","draw_ids":["wheel_draw_replay"],"command_id":"draw-command","user_id":1001,"wheel_id":"classic","rewards":[{"SelectedTierID":"tier-1","RewardType":"coin","RewardCoins":200}],"visible_region_id":9}`), + }}} + wallet := &fakeWheelWallet{coinBalanceAfter: 1200} + service := New(repository, WithWallet(wallet)) + service.SetClock(func() time.Time { return time.UnixMilli(1700000000000).UTC() }) + + result, err := service.ProcessPendingRewards(context.Background(), RewardWorkerOptions{ + WorkerID: "activity-1:wheel-reward", + AppCodes: []string{"fami"}, + }) + if err != nil { + t.Fatalf("ProcessPendingRewards returned error: %v", err) + } + if result.Claimed != 1 || result.Delivered != 1 || result.Failed != 0 { + t.Fatalf("unexpected batch result: %+v", result) + } + if len(wallet.coinCredits) != 1 { + t.Fatalf("wallet credit count = %d, want 1", len(wallet.coinCredits)) + } + request := wallet.coinCredits[0] + if request.GetCommandId() != "wheel_reward:wheel_draw_replay" || request.GetAppCode() != "fami" || request.GetTargetUserId() != 1001 || request.GetAmount() != 200 { + t.Fatalf("wallet reconciliation command mismatch: %+v", request) + } + if repository.deliveredOutboxID != "wheel_reward_wheel_draw_replay" || repository.deliveredWorkerID != "activity-1:wheel-reward" || repository.deliveredTransactionID != "tx-wheel_reward:wheel_draw_replay" { + t.Fatalf("delivered marker mismatch: %+v", repository) + } + if len(repository.deliveredDrawIDs) != 1 || repository.deliveredDrawIDs[0] != "wheel_draw_replay" { + t.Fatalf("delivered draw ids mismatch: %+v", repository.deliveredDrawIDs) + } + if len(repository.claimApps) != 1 || repository.claimApps[0] != "fami" { + t.Fatalf("claim must use explicit app partition, got %+v", repository.claimApps) + } +} + +func TestProcessPendingRewardsRejectsCrossAppPayloadBeforeWallet(t *testing.T) { + repository := &fakeWheelRepository{claimRecords: []domain.RewardOutboxRecord{{ + AppCode: "fami", + OutboxID: "wheel_reward_cross_app", + Payload: []byte(`{"app_code":"huwaa","draw_id":"cross_app","command_id":"cmd","user_id":1001,"wheel_id":"classic","reward_type":"coin","reward_coins":200,"selected_tier_id":"tier-1"}`), + }}} + wallet := &fakeWheelWallet{} + service := New(repository, WithWallet(wallet)) + service.SetClock(func() time.Time { return time.UnixMilli(1700000000000).UTC() }) + + result, err := service.ProcessPendingRewards(context.Background(), RewardWorkerOptions{WorkerID: "worker", AppCodes: []string{"fami"}}) + if err == nil || result.Claimed != 1 || result.Failed != 1 || result.Delivered != 0 { + t.Fatalf("cross-app payload must fail closed, result=%+v err=%v", result, err) + } + if len(wallet.coinCredits) != 0 || len(wallet.resourceGrants) != 0 { + t.Fatalf("cross-app payload must not reach wallet: coin=%d resource=%d", len(wallet.coinCredits), len(wallet.resourceGrants)) + } + if repository.failedOutboxID != "wheel_reward_cross_app" || repository.failedReason == "" { + t.Fatalf("failed outbox marker missing: %+v", repository) + } +} + +func TestProcessPendingRewardsLeavesCompletedWalletCommandForLeaseRecoveryWhenDeliveryMarkFails(t *testing.T) { + repository := &fakeWheelRepository{ + deliverErr: errors.New("mysql unavailable"), + claimRecords: []domain.RewardOutboxRecord{{ + AppCode: "lalu", + OutboxID: "wheel_reward_mark_failure", + Payload: []byte(`{"app_code":"lalu","draw_id":"mark_failure","command_id":"cmd","user_id":1001,"wheel_id":"classic","reward_type":"coin","reward_coins":20,"selected_tier_id":"tier-1"}`), + }}, + } + wallet := &fakeWheelWallet{} + service := New(repository, WithWallet(wallet)) + + result, err := service.ProcessPendingRewards(context.Background(), RewardWorkerOptions{WorkerID: "worker", AppCodes: []string{"lalu"}}) + if err == nil || result.Failed != 1 || len(wallet.coinCredits) != 1 { + t.Fatalf("delivery mark failure mismatch: result=%+v wallet=%d err=%v", result, len(wallet.coinCredits), err) + } + if repository.failedOutboxID != "" { + t.Fatalf("lost lease/delivery failure must stay delivering for safe idempotent reclaim, got failed marker %q", repository.failedOutboxID) + } +} diff --git a/services/activity-service/internal/service/wheel/worker.go b/services/activity-service/internal/service/wheel/worker.go new file mode 100644 index 00000000..b8de8c7f --- /dev/null +++ b/services/activity-service/internal/service/wheel/worker.go @@ -0,0 +1,218 @@ +package wheel + +import ( + "context" + "encoding/json" + "errors" + "fmt" + "log/slog" + "strings" + "time" + + "hyapp/pkg/appcode" + "hyapp/pkg/logx" + "hyapp/pkg/xerr" + domain "hyapp/services/activity-service/internal/domain/wheel" +) + +// RewardWorkerOptions 定义补偿 worker 的租户、锁和重试边界。 +type RewardWorkerOptions struct { + WorkerID string + AppCodes []string + PollInterval time.Duration + BatchSize int + LockTTL time.Duration + MaxRetry int +} + +// RewardWorkerBatchResult 只统计本轮 claim 后的终态；失败项已经释放为 retryable/failed，不会静默丢失。 +type RewardWorkerBatchResult struct { + Claimed int + Delivered int + Failed int +} + +type wheelRewardOutboxPayload struct { + AppCode string `json:"app_code"` + DrawID string `json:"draw_id"` + DrawIDs []string `json:"draw_ids"` + CommandID string `json:"command_id"` + UserID int64 `json:"user_id"` + WheelID string `json:"wheel_id"` + SelectedTierID string `json:"selected_tier_id"` + RewardType string `json:"reward_type"` + RewardID string `json:"reward_id"` + RewardCount int64 `json:"reward_count"` + RewardCoins int64 `json:"reward_coins"` + Rewards []domain.DrawReward `json:"rewards"` + VisibleRegionID int64 `json:"visible_region_id"` +} + +// RunRewardWorker 持续逐 App 分区补偿 WheelRewardSettlement。 +// AppCode 必须写进 context 后再 claim 和调用钱包，避免后台根 context 被 appcode 默认值悄悄收敛成 lalu。 +func (s *Service) RunRewardWorker(ctx context.Context, options RewardWorkerOptions) { + options = normalizeRewardWorkerOptions(options) + ticker := time.NewTicker(options.PollInterval) + defer ticker.Stop() + for { + if _, err := s.ProcessPendingRewards(ctx, options); err != nil && ctx.Err() == nil { + logx.Error(ctx, "wheel_reward_outbox_batch_failed", err, slog.String("worker_id", options.WorkerID)) + } + select { + case <-ctx.Done(): + return + case <-ticker.C: + } + } +} + +// ProcessPendingRewards 执行一轮有界补偿。 +// 钱包 RPC 使用 draw_id 派生的稳定 command_id；钱包 owner 会在同一事务内先查既有交易/权益及 request hash，命中则返回原收据，未命中才创建，因此 activity 只在收齐所有收据后标 delivered。 +func (s *Service) ProcessPendingRewards(ctx context.Context, options RewardWorkerOptions) (RewardWorkerBatchResult, error) { + if err := s.requireRepository(); err != nil { + return RewardWorkerBatchResult{}, err + } + options = normalizeRewardWorkerOptions(options) + var result RewardWorkerBatchResult + var batchErr error + for _, appCodeValue := range options.AppCodes { + if ctx.Err() != nil { + return result, errors.Join(batchErr, ctx.Err()) + } + appCtx := appcode.WithContext(ctx, appCodeValue) + nowMS := s.now().UTC().UnixMilli() + records, err := s.repository.ClaimWheelRewardOutbox(appCtx, options.WorkerID, nowMS, nowMS+options.LockTTL.Milliseconds(), options.BatchSize, options.MaxRetry) + if err != nil { + batchErr = errors.Join(batchErr, fmt.Errorf("claim app=%s: %w", appCodeValue, err)) + continue + } + result.Claimed += len(records) + for _, record := range records { + if err := s.processRewardOutboxRecord(appCtx, record, options); err != nil { + result.Failed++ + batchErr = errors.Join(batchErr, fmt.Errorf("process app=%s outbox=%s: %w", appCodeValue, record.OutboxID, err)) + continue + } + result.Delivered++ + } + } + return result, batchErr +} + +func (s *Service) processRewardOutboxRecord(ctx context.Context, record domain.RewardOutboxRecord, options RewardWorkerOptions) error { + payload, err := decodeWheelRewardOutboxPayload(record) + if err == nil { + var fulfilled domain.DrawResult + fulfilled, err = s.fulfillWheelReward(ctx, domain.DrawCommand{ + CommandID: payload.CommandID, + WheelID: payload.WheelID, + UserID: payload.UserID, + VisibleRegionID: payload.VisibleRegionID, + }, domain.DrawResult{ + DrawID: payload.DrawID, + DrawIDs: payload.DrawIDs, + CommandID: payload.CommandID, + WheelID: payload.WheelID, + SelectedTierID: payload.SelectedTierID, + RewardType: payload.RewardType, + RewardID: payload.RewardID, + RewardCount: payload.RewardCount, + RewardCoins: payload.RewardCoins, + RewardStatus: domain.StatusPending, + Rewards: payload.Rewards, + }) + if err == nil { + drawIDs := payload.DrawIDs + if len(drawIDs) == 0 { + drawIDs = []string{payload.DrawID} + } + // draw、统计和 outbox 的最终提交带 worker fencing token；若 lease 已被接管，这里会回滚而不是覆盖新 worker。 + return s.repository.MarkWheelRewardOutboxDelivered(ctx, record.OutboxID, options.WorkerID, drawIDs, fulfilled.WalletTransactionID, s.now().UTC().UnixMilli()) + } + } + + nowMS := s.now().UTC().UnixMilli() + nextRetryAtMS := nowMS + wheelRewardRetryDelay(record.RetryCount+1).Milliseconds() + markErr := s.repository.MarkWheelRewardOutboxFailed(ctx, record.OutboxID, options.WorkerID, nowMS, nextRetryAtMS, options.MaxRetry, err.Error()) + if markErr != nil { + return errors.Join(err, fmt.Errorf("mark wheel reward retryable: %w", markErr)) + } + return err +} + +func decodeWheelRewardOutboxPayload(record domain.RewardOutboxRecord) (wheelRewardOutboxPayload, error) { + var payload wheelRewardOutboxPayload + if err := json.Unmarshal(record.Payload, &payload); err != nil { + return payload, fmt.Errorf("decode wheel reward payload: %w", err) + } + recordApp := appcode.Normalize(record.AppCode) + payloadApp := appcode.Normalize(payload.AppCode) + if payload.AppCode != "" && payloadApp != recordApp { + return payload, xerr.New(xerr.Conflict, "wheel reward payload app_code does not match outbox partition") + } + payload.AppCode = recordApp + payload.DrawID = strings.TrimSpace(payload.DrawID) + payload.CommandID = strings.TrimSpace(payload.CommandID) + payload.WheelID = strings.TrimSpace(payload.WheelID) + if payload.DrawID == "" || payload.CommandID == "" || payload.WheelID == "" || payload.UserID <= 0 { + return payload, xerr.New(xerr.InvalidArgument, "wheel reward payload identity is incomplete") + } + if len(payload.DrawIDs) == 0 { + payload.DrawIDs = []string{payload.DrawID} + } + for index := range payload.DrawIDs { + payload.DrawIDs[index] = strings.TrimSpace(payload.DrawIDs[index]) + if payload.DrawIDs[index] == "" { + return payload, xerr.New(xerr.InvalidArgument, "wheel reward payload contains empty draw_id") + } + } + return payload, nil +} + +func normalizeRewardWorkerOptions(options RewardWorkerOptions) RewardWorkerOptions { + options.WorkerID = strings.TrimSpace(options.WorkerID) + if options.WorkerID == "" { + options.WorkerID = "activity-wheel-reward" + } + seen := make(map[string]struct{}, len(options.AppCodes)) + appCodes := make([]string, 0, len(options.AppCodes)) + for _, value := range options.AppCodes { + value = appcode.Normalize(value) + if _, exists := seen[value]; exists { + continue + } + seen[value] = struct{}{} + appCodes = append(appCodes, value) + } + if len(appCodes) == 0 { + appCodes = []string{"lalu", "fami", "huwaa"} + } + options.AppCodes = appCodes + if options.PollInterval <= 0 { + options.PollInterval = time.Second + } + if options.BatchSize <= 0 { + options.BatchSize = 50 + } + if options.LockTTL <= 0 { + options.LockTTL = 30 * time.Second + } + if options.MaxRetry <= 0 { + options.MaxRetry = 16 + } + return options +} + +func wheelRewardRetryDelay(attempt int) time.Duration { + if attempt < 1 { + attempt = 1 + } + if attempt > 7 { + attempt = 7 + } + delay := time.Second * time.Duration(1<<(attempt-1)) + if delay > time.Minute { + return time.Minute + } + return delay +} diff --git a/services/activity-service/internal/storage/mysql/wheel_outbox_repository.go b/services/activity-service/internal/storage/mysql/wheel_outbox_repository.go new file mode 100644 index 00000000..2021c1ff --- /dev/null +++ b/services/activity-service/internal/storage/mysql/wheel_outbox_repository.go @@ -0,0 +1,212 @@ +package mysql + +import ( + "context" + "strings" + + "hyapp/pkg/appcode" + "hyapp/pkg/xerr" + domain "hyapp/services/activity-service/internal/domain/wheel" +) + +// ClaimWheelRewardOutbox 抢占一个 App 分区内可补偿的转盘发奖事实。 +// 行锁和 SKIP LOCKED 允许双节点并行；过期 delivering 可被接管，而仍持有效 lease 的记录不会被第二个节点重复处理。 +func (r *Repository) ClaimWheelRewardOutbox(ctx context.Context, workerID string, nowMS int64, lockUntilMS int64, limit int, maxRetry int) ([]domain.RewardOutboxRecord, error) { + if r == nil || r.db == nil { + return nil, xerr.New(xerr.Unavailable, "mysql repository is not configured") + } + workerID = strings.TrimSpace(workerID) + if workerID == "" { + return nil, xerr.New(xerr.InvalidArgument, "wheel reward worker_id is required") + } + if limit <= 0 { + limit = 50 + } + if maxRetry <= 0 { + maxRetry = 16 + } + tx, err := r.db.BeginTx(ctx, nil) + if err != nil { + return nil, err + } + defer func() { _ = tx.Rollback() }() + appCode := appcode.FromContext(ctx) + rows, err := tx.QueryContext(ctx, ` + SELECT app_code, outbox_id, payload, status, retry_count, next_retry_at_ms, + locked_by, lock_until_ms, last_error, created_at_ms, updated_at_ms + FROM activity_outbox + WHERE app_code = ? + AND event_type = ? + AND retry_count < ? + AND ( + (status IN (?, ?) AND next_retry_at_ms <= ?) + OR (status = ? AND lock_until_ms <= ?) + ) + ORDER BY created_at_ms, outbox_id + LIMIT ? + FOR UPDATE SKIP LOCKED`, + appCode, + domain.EventTypeWheelRewardSettlement, + maxRetry, + domain.OutboxStatusPending, + domain.OutboxStatusRetryable, + nowMS, + domain.OutboxStatusDelivering, + nowMS, + limit, + ) + if err != nil { + return nil, err + } + records := make([]domain.RewardOutboxRecord, 0, limit) + for rows.Next() { + var record domain.RewardOutboxRecord + if err := rows.Scan( + &record.AppCode, + &record.OutboxID, + &record.Payload, + &record.Status, + &record.RetryCount, + &record.NextRetryAtMS, + &record.LockedBy, + &record.LockUntilMS, + &record.LastError, + &record.CreatedAtMS, + &record.UpdatedAtMS, + ); err != nil { + _ = rows.Close() + return nil, err + } + records = append(records, record) + } + if err := rows.Close(); err != nil { + return nil, err + } + if err := rows.Err(); err != nil { + return nil, err + } + for index := range records { + result, err := tx.ExecContext(ctx, ` + UPDATE activity_outbox + SET status = ?, locked_by = ?, lock_until_ms = ?, updated_at_ms = ? + WHERE app_code = ? AND outbox_id = ?`, + domain.OutboxStatusDelivering, + workerID, + lockUntilMS, + nowMS, + records[index].AppCode, + records[index].OutboxID, + ) + if err != nil { + return nil, err + } + if affected, err := result.RowsAffected(); err != nil || affected != 1 { + if err != nil { + return nil, err + } + return nil, xerr.New(xerr.Conflict, "wheel reward outbox claim lost") + } + records[index].Status = domain.OutboxStatusDelivering + records[index].LockedBy = workerID + records[index].LockUntilMS = lockUntilMS + } + if err := tx.Commit(); err != nil { + return nil, err + } + return records, nil +} + +// MarkWheelRewardOutboxDelivered 把钱包返回的既有或新建收据与 draw 状态、统计和 outbox 在一个 MySQL 事务中提交。 +// locked_by 是 fencing token；worker lease 过期并被接管后，旧 worker 即使晚返回也不能覆盖新 owner 的处理结果。 +func (r *Repository) MarkWheelRewardOutboxDelivered(ctx context.Context, outboxID string, workerID string, drawIDs []string, transactionID string, nowMS int64) error { + if r == nil || r.db == nil { + return xerr.New(xerr.Unavailable, "mysql repository is not configured") + } + outboxID = strings.TrimSpace(outboxID) + workerID = strings.TrimSpace(workerID) + if outboxID == "" || workerID == "" { + return xerr.New(xerr.InvalidArgument, "wheel reward outbox delivery identity is incomplete") + } + tx, err := r.db.BeginTx(ctx, nil) + if err != nil { + return err + } + defer func() { _ = tx.Rollback() }() + appCode := appcode.FromContext(ctx) + if err := markWheelDrawsGrantedTx(ctx, tx, appCode, drawIDs, transactionID, nowMS, true); err != nil { + return err + } + result, err := tx.ExecContext(ctx, ` + UPDATE activity_outbox + SET status = ?, locked_by = '', lock_until_ms = 0, last_error = '', updated_at_ms = ? + WHERE app_code = ? AND outbox_id = ? AND event_type = ? AND status = ? AND locked_by = ?`, + domain.OutboxStatusDelivered, + nowMS, + appCode, + outboxID, + domain.EventTypeWheelRewardSettlement, + domain.OutboxStatusDelivering, + workerID, + ) + if err != nil { + return err + } + if affected, err := result.RowsAffected(); err != nil || affected != 1 { + if err != nil { + return err + } + return xerr.New(xerr.Conflict, "wheel reward outbox delivery lock lost") + } + return tx.Commit() +} + +// MarkWheelRewardOutboxFailed 释放失败 lease，并以有界重试次数决定 retryable 或 failed。 +func (r *Repository) MarkWheelRewardOutboxFailed(ctx context.Context, outboxID string, workerID string, nowMS int64, nextRetryAtMS int64, maxRetry int, lastError string) error { + if r == nil || r.db == nil { + return xerr.New(xerr.Unavailable, "mysql repository is not configured") + } + if maxRetry <= 0 { + maxRetry = 16 + } + result, err := r.db.ExecContext(ctx, ` + UPDATE activity_outbox + SET status = CASE WHEN retry_count + 1 >= ? THEN ? ELSE ? END, + next_retry_at_ms = CASE WHEN retry_count + 1 >= ? THEN 0 ELSE ? END, + retry_count = retry_count + 1, + locked_by = '', + lock_until_ms = 0, + last_error = ?, + updated_at_ms = ? + WHERE app_code = ? AND outbox_id = ? AND event_type = ? AND status = ? AND locked_by = ?`, + maxRetry, + domain.OutboxStatusFailed, + domain.OutboxStatusRetryable, + maxRetry, + nextRetryAtMS, + truncateWheelOutboxError(lastError), + nowMS, + appcode.FromContext(ctx), + strings.TrimSpace(outboxID), + domain.EventTypeWheelRewardSettlement, + domain.OutboxStatusDelivering, + strings.TrimSpace(workerID), + ) + if err != nil { + return err + } + if affected, err := result.RowsAffected(); err != nil || affected != 1 { + if err != nil { + return err + } + return xerr.New(xerr.Conflict, "wheel reward outbox failure lock lost") + } + return nil +} + +func truncateWheelOutboxError(message string) string { + message = strings.TrimSpace(message) + if len(message) <= 512 { + return message + } + return message[:512] +} diff --git a/services/activity-service/internal/storage/mysql/wheel_outbox_repository_test.go b/services/activity-service/internal/storage/mysql/wheel_outbox_repository_test.go new file mode 100644 index 00000000..707828c2 --- /dev/null +++ b/services/activity-service/internal/storage/mysql/wheel_outbox_repository_test.go @@ -0,0 +1,79 @@ +package mysql + +import ( + "context" + "testing" + + "hyapp/internal/testutil/mysqlschema" + "hyapp/pkg/appcode" + domain "hyapp/services/activity-service/internal/domain/wheel" +) + +func TestWheelRewardOutboxClaimAndDeliveryAreLeaseFencedAndAtomic(t *testing.T) { + schema := mysqlschema.New(t, mysqlschema.Config{ + EnvVar: "ACTIVITY_SERVICE_MYSQL_TEST_DSN", + InitDBPath: mysqlschema.InitDBPath(t, mysqlschema.CallerFile(t, 1), "..", "..", "..", "deploy", "mysql", "initdb", "001_activity_service.sql"), + DatabasePrefix: "hyapp_activity_wheel_outbox_test", + }) + repository, err := Open(context.Background(), schema.DSN) + if err != nil { + t.Fatalf("open repository: %v", err) + } + t.Cleanup(func() { _ = repository.Close() }) + ctx := appcode.WithContext(context.Background(), "fami") + const ( + drawID = "wheel_draw_outbox_test" + outboxID = "wheel_reward_wheel_draw_outbox_test" + ) + if _, err := schema.DB.ExecContext(ctx, ` + INSERT INTO wheel_draw_stats ( + app_code, wheel_id, total_draws, unique_users, total_spent_coins, total_rtp_value_coins, + pending_draws, granted_draws, failed_draws, created_at_ms, updated_at_ms + ) VALUES ('fami', 'classic', 1, 1, 100, 20, 1, 0, 0, 1000, 1000)`); err != nil { + t.Fatalf("seed wheel stats: %v", err) + } + if _, err := schema.DB.ExecContext(ctx, ` + INSERT INTO wheel_draw_records ( + app_code, draw_id, command_id, user_id, device_id, visible_region_id, wheel_id, coin_spent, + rule_version, rtp_window_index, selected_tier_id, reward_type, reward_id, reward_count, + reward_coins, rtp_value_coins, candidate_tiers_json, rtp_snapshot_json, prize_metadata_json, + reward_status, paid_at_ms, created_at_ms, updated_at_ms + ) VALUES ('fami', ?, 'draw-command', 1001, 'device', 9, 'classic', 100, + 1, 1, 'tier-1', 'coin', '', 1, 20, 20, '{}', '{}', '{}', 'pending', 1000, 1000, 1000)`, drawID); err != nil { + t.Fatalf("seed wheel draw: %v", err) + } + if _, err := schema.DB.ExecContext(ctx, ` + INSERT INTO activity_outbox ( + app_code, outbox_id, event_type, payload, status, retry_count, next_retry_at_ms, + locked_by, lock_until_ms, last_error, created_at_ms, updated_at_ms + ) VALUES ('fami', ?, ?, '{}', 'pending', 0, 1000, '', 0, '', 1000, 1000)`, outboxID, domain.EventTypeWheelRewardSettlement); err != nil { + t.Fatalf("seed wheel outbox: %v", err) + } + + records, err := repository.ClaimWheelRewardOutbox(ctx, "worker-1", 1000, 31000, 10, 16) + if err != nil || len(records) != 1 || records[0].LockedBy != "worker-1" || records[0].Status != domain.OutboxStatusDelivering { + t.Fatalf("claim mismatch: records=%+v err=%v", records, err) + } + secondClaim, err := repository.ClaimWheelRewardOutbox(ctx, "worker-2", 2000, 32000, 10, 16) + if err != nil || len(secondClaim) != 0 { + t.Fatalf("active lease must block second worker: records=%+v err=%v", secondClaim, err) + } + if err := repository.MarkWheelRewardOutboxDelivered(ctx, outboxID, "worker-1", []string{drawID}, "wtx-existing", 3000); err != nil { + t.Fatalf("mark delivered: %v", err) + } + + var drawStatus, receiptRef, outboxStatus, lockedBy string + var pending, granted int64 + if err := schema.DB.QueryRowContext(ctx, `SELECT reward_status, reward_transaction_id FROM wheel_draw_records WHERE app_code = 'fami' AND draw_id = ?`, drawID).Scan(&drawStatus, &receiptRef); err != nil { + t.Fatalf("read draw state: %v", err) + } + if err := schema.DB.QueryRowContext(ctx, `SELECT pending_draws, granted_draws FROM wheel_draw_stats WHERE app_code = 'fami' AND wheel_id = 'classic'`).Scan(&pending, &granted); err != nil { + t.Fatalf("read wheel stats: %v", err) + } + if err := schema.DB.QueryRowContext(ctx, `SELECT status, locked_by FROM activity_outbox WHERE app_code = 'fami' AND outbox_id = ?`, outboxID).Scan(&outboxStatus, &lockedBy); err != nil { + t.Fatalf("read outbox state: %v", err) + } + if drawStatus != domain.StatusGranted || receiptRef != "wtx-existing" || pending != 0 || granted != 1 || outboxStatus != domain.OutboxStatusDelivered || lockedBy != "" { + t.Fatalf("delivery state mismatch: draw=%s receipt=%s pending=%d granted=%d outbox=%s lock=%q", drawStatus, receiptRef, pending, granted, outboxStatus, lockedBy) + } +} diff --git a/services/activity-service/internal/storage/mysql/wheel_repository.go b/services/activity-service/internal/storage/mysql/wheel_repository.go index 04d41a12..fa50c90d 100644 --- a/services/activity-service/internal/storage/mysql/wheel_repository.go +++ b/services/activity-service/internal/storage/mysql/wheel_repository.go @@ -317,11 +317,38 @@ func (r *Repository) MarkWheelDrawsGranted(ctx context.Context, appCode string, return err } defer func() { _ = tx.Rollback() }() + if err := markWheelDrawsGrantedTx(ctx, tx, appCode, drawIDs, transactionID, nowMS, false); err != nil { + return err + } for _, drawID := range drawIDs { drawID = strings.TrimSpace(drawID) if drawID == "" { continue } + if _, err := tx.ExecContext(ctx, ` + UPDATE activity_outbox + SET status = 'delivered', locked_by = '', lock_until_ms = 0, last_error = '', updated_at_ms = ? + WHERE app_code = ? AND outbox_id = ?`, + nowMS, appCode, "wheel_reward_"+drawID, + ); err != nil { + return err + } + } + return tx.Commit() +} + +// markWheelDrawsGrantedTx 原子搬移 draw 和汇总状态；requireDraw 用于补偿 worker 防止损坏 payload 把 outbox 错标 delivered。 +func markWheelDrawsGrantedTx(ctx context.Context, tx *sql.Tx, appCode string, drawIDs []string, transactionID string, nowMS int64, requireDraw bool) error { + seen := make(map[string]struct{}, len(drawIDs)) + for _, drawID := range drawIDs { + drawID = strings.TrimSpace(drawID) + if drawID == "" { + continue + } + if _, exists := seen[drawID]; exists { + continue + } + seen[drawID] = struct{}{} var wheelID, oldStatus string if err := tx.QueryRowContext(ctx, ` SELECT wheel_id, reward_status @@ -331,6 +358,9 @@ func (r *Repository) MarkWheelDrawsGranted(ctx context.Context, appCode string, appCode, drawID, ).Scan(&wheelID, &oldStatus); err != nil { if errors.Is(err, sql.ErrNoRows) { + if requireDraw { + return xerr.New(xerr.NotFound, "wheel draw record not found") + } continue } return err @@ -356,16 +386,11 @@ func (r *Repository) MarkWheelDrawsGranted(ctx context.Context, appCode string, return err } } - if _, err := tx.ExecContext(ctx, ` - UPDATE activity_outbox - SET status = 'delivered', locked_by = '', lock_until_ms = 0, last_error = '', updated_at_ms = ? - WHERE app_code = ? AND outbox_id = ?`, - nowMS, appCode, "wheel_reward_"+drawID, - ); err != nil { - return err - } } - return tx.Commit() + if requireDraw && len(seen) == 0 { + return xerr.New(xerr.InvalidArgument, "wheel draw_ids are empty") + } + return nil } func (r *Repository) executeSingleWheelDraw(cmd domain.DrawCommand, commandID string, unitSpent int64, config domain.RuleConfig, window *wheelRTPWindow, pool *wheelPool, nowMS int64) (domain.DrawResult, int64, error) { diff --git a/services/game-service/configs/config.docker.yaml b/services/game-service/configs/config.docker.yaml index 73bed56c..331b1083 100644 --- a/services/game-service/configs/config.docker.yaml +++ b/services/game-service/configs/config.docker.yaml @@ -34,6 +34,7 @@ rocketmq: producer_group: "hyapp-game-outbox-producer" outbox_worker: enabled: true + app_codes: [] poll_interval: "1s" batch_size: 100 publish_timeout: "3s" diff --git a/services/game-service/configs/config.tencent.example.yaml b/services/game-service/configs/config.tencent.example.yaml index e422243c..ae4eb0dc 100644 --- a/services/game-service/configs/config.tencent.example.yaml +++ b/services/game-service/configs/config.tencent.example.yaml @@ -34,6 +34,8 @@ rocketmq: producer_group: "hyapp-game-outbox-producer" outbox_worker: enabled: true + # 空值自动发现全部 App；历史积压回放时先设 ["lalu", "fami"]，核对后再加入 "huwaa"。 + app_codes: [] poll_interval: "60s" batch_size: 10 publish_timeout: "3s" diff --git a/services/game-service/configs/config.yaml b/services/game-service/configs/config.yaml index e985c80f..c20e4de7 100644 --- a/services/game-service/configs/config.yaml +++ b/services/game-service/configs/config.yaml @@ -34,6 +34,7 @@ rocketmq: producer_group: "hyapp-game-outbox-producer" outbox_worker: enabled: true + app_codes: [] poll_interval: "1s" batch_size: 100 publish_timeout: "3s" diff --git a/services/game-service/internal/app/app.go b/services/game-service/internal/app/app.go index ef3f796c..1fa4ae65 100644 --- a/services/game-service/internal/app/app.go +++ b/services/game-service/internal/app/app.go @@ -11,10 +11,12 @@ import ( "google.golang.org/grpc" "google.golang.org/grpc/credentials/insecure" gamev1 "hyapp.local/api/proto/game/v1" + "hyapp/pkg/appcode" "hyapp/pkg/gamemq" "hyapp/pkg/grpchealth" "hyapp/pkg/healthhttp" "hyapp/pkg/logx" + "hyapp/pkg/outboxpartition" "hyapp/pkg/rocketmqx" servicegrpc "hyapp/pkg/servicekit/grpcserver" servicehealth "hyapp/pkg/servicekit/health" @@ -40,11 +42,13 @@ type App struct { activityConn *grpc.ClientConn robotConn *grpc.ClientConn outboxProducer *rocketmqx.Producer - outboxCtx context.Context - outboxCancel context.CancelFunc - outboxWG sync.WaitGroup - cfg config.Config - closeOnce sync.Once + // outboxPartitions 缓存真实 App 列表并让双实例从不同轮转起点公平 claim。 + outboxPartitions *outboxpartition.Partitions + outboxCtx context.Context + outboxCancel context.CancelFunc + outboxWG sync.WaitGroup + cfg config.Config + closeOnce sync.Once } func New(cfg config.Config) (*App, error) { @@ -154,7 +158,7 @@ func New(cfg config.Config) (*App, error) { } outboxCtx, outboxCancel := context.WithCancel(context.Background()) - return &App{server: server, listener: listener, health: health, healthHTTP: healthHTTP, repo: repo, walletConn: walletConn, userConn: userConn, activityConn: activityConn, robotConn: robotConn, outboxProducer: outboxProducer, outboxCtx: outboxCtx, outboxCancel: outboxCancel, cfg: cfg}, nil + return &App{server: server, listener: listener, health: health, healthHTTP: healthHTTP, repo: repo, walletConn: walletConn, userConn: userConn, activityConn: activityConn, robotConn: robotConn, outboxProducer: outboxProducer, outboxPartitions: outboxpartition.New(cfg.OutboxWorker.AppCodes, outboxpartition.DefaultDiscoveryRefreshInterval), outboxCtx: outboxCtx, outboxCancel: outboxCancel, cfg: cfg}, nil } func (a *App) Run() error { @@ -229,7 +233,17 @@ func (a *App) runOutboxWorker() { func (a *App) processOutboxBatch() (int, error) { workerID := "game-outbox-" + a.cfg.NodeID - records, err := a.repo.ClaimPendingGameOutbox(a.outboxCtx, workerID, a.cfg.OutboxWorker.BatchSize) + appCodes, discoveryErr := a.outboxPartitions.Resolve(a.outboxCtx, a.repo.ListGameOutboxAppCodes) + if discoveryErr != nil { + if len(appCodes) == 0 { + return 0, discoveryErr + } + // 已缓存 App 继续发布；低频刷新失败不应把整个 game outbox worker 停掉。 + logx.Warn(a.outboxCtx, "game_outbox_app_discovery_stale", slog.String("worker_id", workerID), slog.String("error", discoveryErr.Error())) + } + records, err := outboxpartition.ClaimFair(a.outboxCtx, a.outboxPartitions.Order(appCodes), a.cfg.OutboxWorker.BatchSize, func(claimCtx context.Context, appCode string, limit int) ([]mysqlstorage.GameOutboxRecord, error) { + return a.repo.ClaimPendingGameOutbox(appcode.WithContext(claimCtx, appCode), workerID, limit) + }) if err != nil { return 0, err } diff --git a/services/game-service/internal/config/config.go b/services/game-service/internal/config/config.go index 1e26ccdf..1903874e 100644 --- a/services/game-service/internal/config/config.go +++ b/services/game-service/internal/config/config.go @@ -9,6 +9,7 @@ import ( "hyapp/pkg/configx" "hyapp/pkg/logx" + "hyapp/pkg/outboxpartition" "hyapp/pkg/tencentim" ) @@ -95,7 +96,9 @@ type GameOutboxConfig struct { } type OutboxWorkerConfig struct { - Enabled bool `yaml:"enabled"` + Enabled bool `yaml:"enabled"` + // AppCodes 是可投递租户 allowlist；空值自动发现，非空用于分阶段回放历史积压。 + AppCodes []string `yaml:"app_codes"` PollInterval time.Duration `yaml:"poll_interval"` BatchSize int `yaml:"batch_size"` PublishTimeout time.Duration `yaml:"publish_timeout"` @@ -210,6 +213,7 @@ func (cfg *Config) Normalize() error { if cfg.OutboxWorker.PublishTimeout <= 0 { cfg.OutboxWorker.PublishTimeout = 3 * time.Second } + cfg.OutboxWorker.AppCodes = outboxpartition.Normalize(cfg.OutboxWorker.AppCodes, false) if cfg.OutboxWorker.Enabled && !cfg.RocketMQ.GameOutbox.Enabled { return errors.New("outbox_worker requires rocketmq.game_outbox.enabled") } diff --git a/services/game-service/internal/config/config_test.go b/services/game-service/internal/config/config_test.go index 9b0b498e..e5c92ddb 100644 --- a/services/game-service/internal/config/config_test.go +++ b/services/game-service/internal/config/config_test.go @@ -2,6 +2,27 @@ package config import "testing" +func TestOutboxWorkerAppCodesNormalizeForStagedReplay(t *testing.T) { + cfg := Default() + cfg.OutboxWorker.AppCodes = []string{" LALU ", "fami", "HUWAA", "fami"} + if err := cfg.Normalize(); err != nil { + t.Fatalf("normalize staged outbox config: %v", err) + } + if len(cfg.OutboxWorker.AppCodes) != 3 || cfg.OutboxWorker.AppCodes[0] != "lalu" || cfg.OutboxWorker.AppCodes[1] != "fami" || cfg.OutboxWorker.AppCodes[2] != "huwaa" { + t.Fatalf("normalized outbox app_codes = %v", cfg.OutboxWorker.AppCodes) + } +} + +func TestLocalOutboxWorkerEmptyAppCodesUsesDiscovery(t *testing.T) { + cfg, err := Load("../../configs/config.yaml") + if err != nil { + t.Fatalf("load local config: %v", err) + } + if len(cfg.OutboxWorker.AppCodes) != 0 { + t.Fatalf("local empty app_codes must retain automatic discovery: %v", cfg.OutboxWorker.AppCodes) + } +} + func TestLoadConfigsKeepActivityServiceAddr(t *testing.T) { tests := []struct { name string diff --git a/services/game-service/internal/storage/mysql/outbox_app_codes_test.go b/services/game-service/internal/storage/mysql/outbox_app_codes_test.go new file mode 100644 index 00000000..71a0c17c --- /dev/null +++ b/services/game-service/internal/storage/mysql/outbox_app_codes_test.go @@ -0,0 +1,30 @@ +package mysql + +import ( + "context" + "regexp" + "testing" + + "github.com/DATA-DOG/go-sqlmock" +) + +func TestListGameOutboxAppCodesForcesPrimaryPrefixIndex(t *testing.T) { + db, mock, err := sqlmock.New() + if err != nil { + t.Fatalf("create sqlmock: %v", err) + } + defer db.Close() + mock.ExpectQuery(regexp.QuoteMeta(`SELECT DISTINCT app_code FROM game_outbox FORCE INDEX (PRIMARY) ORDER BY app_code`)). + WillReturnRows(sqlmock.NewRows([]string{"app_code"}).AddRow("fami").AddRow("huwaa").AddRow("lalu")) + + apps, err := (&Repository{db: db}).ListGameOutboxAppCodes(context.Background()) + if err != nil { + t.Fatalf("list game outbox apps: %v", err) + } + if len(apps) != 3 || apps[0] != "fami" || apps[1] != "huwaa" || apps[2] != "lalu" { + t.Fatalf("game outbox apps mismatch: %v", apps) + } + if err := mock.ExpectationsWereMet(); err != nil { + t.Fatalf("game app discovery query mismatch: %v", err) + } +} diff --git a/services/game-service/internal/storage/mysql/repository.go b/services/game-service/internal/storage/mysql/repository.go index 7fa55b19..62f6ba0a 100644 --- a/services/game-service/internal/storage/mysql/repository.go +++ b/services/game-service/internal/storage/mysql/repository.go @@ -1152,6 +1152,29 @@ type GameOutboxRecord struct { LockUntilMS int64 } +// ListGameOutboxAppCodes enumerates real tenant keys from the game owner table. +// PRIMARY is ordered by (app_code,event_id), allowing cached discovery without +// weakening the app_code prefix used by every SKIP LOCKED claim transaction. +func (r *Repository) ListGameOutboxAppCodes(ctx context.Context) ([]string, error) { + if r == nil || r.db == nil { + return nil, xerr.New(xerr.Unavailable, "mysql repository is not configured") + } + rows, err := r.db.QueryContext(ctx, `SELECT DISTINCT app_code FROM game_outbox FORCE INDEX (PRIMARY) ORDER BY app_code`) + if err != nil { + return nil, err + } + defer rows.Close() + appCodes := make([]string, 0, 4) + for rows.Next() { + var appCode string + if err := rows.Scan(&appCode); err != nil { + return nil, err + } + appCodes = append(appCodes, appCode) + } + return appCodes, rows.Err() +} + // ClaimPendingGameOutbox locks pending game facts so one worker publishes them. func (r *Repository) ClaimPendingGameOutbox(ctx context.Context, workerID string, batchSize int) ([]GameOutboxRecord, error) { if r == nil || r.db == nil { @@ -1356,23 +1379,34 @@ func (r *Repository) queryUnlockedLevelEventClaimKeys(ctx context.Context, tx *s } func (r *Repository) queryExpiredLevelEventClaimKeys(ctx context.Context, tx *sql.Tx, appCode string, nowMS int64, batchSize int) ([]levelEventClaimKey, error) { - // 过期锁接管仍使用 retry/lock 前缀缩小范围，但只锁主键，避免大 JSON 字段进入排序和慢日志。 - return queryLevelEventClaimKeysByStatus(batchSize, func(status string) ([]levelEventClaimKey, error) { + // 过期接管只处理 running：pending/failed 已由 unlocked path 负责。拆开 legacy lock=0 + // 与正常过期锁可继续利用 (app_code,status,locked_until_ms,...) 前缀，同时明确 + // 排除未来锁，避免并发 cron 抢走仍由其它 worker 处理的事实。 + query := func(lockCondition string, args ...any) ([]levelEventClaimKey, error) { + queryArgs := []any{appCode, "running"} + queryArgs = append(queryArgs, args...) + queryArgs = append(queryArgs, nowMS, batchSize) rows, err := tx.QueryContext(ctx, ` SELECT event_id, created_at_ms FROM game_level_event_outbox FORCE INDEX (idx_game_level_event_claim) - WHERE app_code = ? AND status = ? AND next_retry_at_ms <= ? - AND locked_until_ms > 0 AND locked_until_ms <= ? + WHERE app_code = ? AND status = ? AND `+lockCondition+` AND next_retry_at_ms <= ? ORDER BY created_at_ms ASC, event_id ASC - LIMIT ? FOR UPDATE SKIP LOCKED`, - appCode, status, nowMS, nowMS, batchSize, - ) + LIMIT ? FOR UPDATE SKIP LOCKED`, queryArgs...) if err != nil { return nil, err } defer rows.Close() return scanLevelEventClaimKeys(rows) - }) + } + legacyUnlocked, err := query("locked_until_ms = 0") + if err != nil { + return nil, err + } + expired, err := query("locked_until_ms > 0 AND locked_until_ms <= ?", nowMS) + if err != nil { + return nil, err + } + return mergeLevelEventClaimKeys(batchSize, legacyUnlocked, expired), nil } func queryLevelEventClaimKeysByStatus(batchSize int, query func(status string) ([]levelEventClaimKey, error)) ([]levelEventClaimKey, error) { diff --git a/services/game-service/internal/storage/mysql/repository_test.go b/services/game-service/internal/storage/mysql/repository_test.go index 3af80bd6..f68e83c3 100644 --- a/services/game-service/internal/storage/mysql/repository_test.go +++ b/services/game-service/internal/storage/mysql/repository_test.go @@ -483,6 +483,65 @@ func TestClaimPendingLevelEventsPreservesCreatedOrderAcrossStatuses(t *testing.T } } +func TestClaimPendingLevelEventsReclaimsLegacyAndExpiredRunningLocks(t *testing.T) { + caller := mysqlschema.CallerFile(t, 1) + schema := mysqlschema.New(t, mysqlschema.Config{ + InitDBPath: mysqlschema.InitDBPath(t, caller, "../../../deploy/mysql/initdb/001_game_service.sql"), + DatabasePrefix: "hy_game_test", + }) + + ctx := appcode.WithContext(context.Background(), "lalu") + repo, err := Open(ctx, schema.DSN) + if err != nil { + t.Fatalf("open repository failed: %v", err) + } + t.Cleanup(func() { _ = repo.Close() }) + + insertRunning := func(eventID string, lockedUntilMS int64, createdAtMS int64) { + t.Helper() + if _, err := repo.db.ExecContext(ctx, ` + INSERT INTO game_level_event_outbox ( + app_code, event_id, order_id, user_id, game_id, provider_order_id, + provider_round_id, coin_amount, wallet_transaction_id, payload_json, + status, attempt_count, next_retry_at_ms, locked_by, locked_until_ms, + failure_reason, occurred_at_ms, created_at_ms, updated_at_ms + ) VALUES ('lalu', ?, ?, 42, 'game_1', ?, '', 100, 'wallet_tx_1', '{}', + 'running', 1, 0, 'old-worker', ?, '', ?, ?, ?)`, + eventID, "order_"+eventID, "provider_"+eventID, lockedUntilMS, createdAtMS, createdAtMS, createdAtMS, + ); err != nil { + t.Fatalf("insert running level event %s failed: %v", eventID, err) + } + } + insertRunning("legacy_zero_lock", 0, 1000) + insertRunning("expired_lock", 2500, 1100) + insertRunning("future_lock", 4000, 1200) + + events, err := repo.ClaimPendingLevelEvents(ctx, "replacement-worker", 3000, 30*time.Second, 10) + if err != nil { + t.Fatalf("claim expired running events failed: %v", err) + } + if len(events) != 2 || events[0].EventID != "legacy_zero_lock" || events[1].EventID != "expired_lock" { + t.Fatalf("reclaimed running events mismatch: %+v", events) + } + for _, event := range events { + if event.Status != "running" { + t.Fatalf("reclaimed event must expose its pre-claim running state: %+v", event) + } + } + + var lockedBy string + var lockedUntilMS int64 + if err := repo.db.QueryRowContext(ctx, ` + SELECT locked_by, locked_until_ms FROM game_level_event_outbox + WHERE app_code = 'lalu' AND event_id = 'future_lock'`, + ).Scan(&lockedBy, &lockedUntilMS); err != nil { + t.Fatalf("query future running lock failed: %v", err) + } + if lockedBy != "old-worker" || lockedUntilMS != 4000 { + t.Fatalf("future lock was stolen: locked_by=%q locked_until_ms=%d", lockedBy, lockedUntilMS) + } +} + func TestLevelEventClaimKeyQueryUsesOrderIndexWithoutFilesort(t *testing.T) { caller := mysqlschema.CallerFile(t, 1) schema := mysqlschema.New(t, mysqlschema.Config{ diff --git a/services/gateway-service/internal/transport/http/appapi/app_event_handler.go b/services/gateway-service/internal/transport/http/appapi/app_event_handler.go index 3c7ba23b..b4e9b65a 100644 --- a/services/gateway-service/internal/transport/http/appapi/app_event_handler.go +++ b/services/gateway-service/internal/transport/http/appapi/app_event_handler.go @@ -1,7 +1,6 @@ package appapi import ( - "bytes" "encoding/json" "net/http" "strings" @@ -9,16 +8,12 @@ import ( userv1 "hyapp.local/api/proto/user/v1" "hyapp/pkg/appcode" + "hyapp/pkg/apptracking" "hyapp/services/gateway-service/internal/auth" "hyapp/services/gateway-service/internal/client" "hyapp/services/gateway-service/internal/transport/http/httpkit" ) -const ( - appEventMaxBatchSize = 50 - appEventMaxPropertiesBytes = 4096 -) - type appEventsRequest struct { Events []appEventRequest `json:"events"` } @@ -44,26 +39,17 @@ type appEventRequest struct { } func (h *Handler) reportAppEvents(writer http.ResponseWriter, request *http.Request) { - if h.statisticsClient == nil { - // App 埋点事实由 statistics-service 持久化；gateway 未注入写入边界时不能假成功，否则客户端会丢掉重试机会。 - httpkit.WriteError(writer, request, http.StatusBadGateway, httpkit.CodeUpstreamError, "upstream service error") - return - } var body appEventsRequest if !httpkit.Decode(writer, request, &body) { return } - if len(body.Events) == 0 || len(body.Events) > appEventMaxBatchSize { + if !apptracking.ValidBatchSize(len(body.Events)) { httpkit.WriteError(writer, request, http.StatusBadRequest, httpkit.CodeInvalidArgument, "invalid argument") return } nowMS := time.Now().UTC().UnixMilli() userID := auth.UserIDFromContext(request.Context()) - countryID, regionID, ok := h.appEventUserDimensions(writer, request, userID) - if !ok { - return - } headerDeviceID := httpkit.FirstHeader(request, "X-Device-ID", "X-HY-Device-ID") headerPlatform := httpkit.FirstHeader(request, "X-App-Platform", "X-Platform") headerAppVersion := httpkit.FirstHeader(request, "X-App-Version", "X-Client-Version") @@ -75,8 +61,6 @@ func (h *Handler) reportAppEvents(writer http.ResponseWriter, request *http.Requ event, valid := appTrackingEventFromRequest(item, appTrackingEventDefaults{ appCode: appcode.FromContext(request.Context()), userID: userID, - countryID: countryID, - regionID: regionID, deviceID: headerDeviceID, sessionID: auth.SessionIDFromContext(request.Context()), platform: headerPlatform, @@ -91,6 +75,21 @@ func (h *Handler) reportAppEvents(writer http.ResponseWriter, request *http.Requ } events = append(events, event) } + if h.statisticsClient == nil { + // App 埋点事实由 statistics-service 持久化；gateway 未注入写入边界时不能假成功，否则客户端会丢掉重试机会。 + httpkit.WriteError(writer, request, http.StatusBadGateway, httpkit.CodeUpstreamError, "upstream service error") + return + } + // 先完成客户端可控字段的全部长度校验，再读取 user-service。这样超限 + // 请求稳定返回 400，不会制造无意义的下游调用，更不会被包装成 502。 + countryID, regionID, ok := h.appEventUserDimensions(writer, request, userID) + if !ok { + return + } + for index := range events { + events[index].CountryID = countryID + events[index].RegionID = regionID + } result, err := h.statisticsClient.ReportAppTrackingEvents(request.Context(), events) if err != nil { @@ -128,8 +127,6 @@ func (h *Handler) appEventUserDimensions(writer http.ResponseWriter, request *ht type appTrackingEventDefaults struct { appCode string userID int64 - countryID int64 - regionID int64 deviceID string sessionID string platform string @@ -140,16 +137,7 @@ type appTrackingEventDefaults struct { } func appTrackingEventFromRequest(item appEventRequest, defaults appTrackingEventDefaults) (client.AppTrackingEvent, bool) { - item.EventID = strings.TrimSpace(item.EventID) - item.EventName = strings.TrimSpace(item.EventName) - if item.EventID == "" || item.EventName == "" { - return client.AppTrackingEvent{}, false - } deviceID := firstTrimmed(item.DeviceID, defaults.deviceID) - if defaults.userID <= 0 && deviceID == "" { - // 匿名埋点没有服务端 user_id，只能依赖稳定 device_id 做排障和后续粗粒度去重。 - return client.AppTrackingEvent{}, false - } properties, ok := normalizeAppEventProperties(item.Properties) if !ok { return client.AppTrackingEvent{}, false @@ -158,14 +146,14 @@ func appTrackingEventFromRequest(item appEventRequest, defaults appTrackingEvent if occurredAtMS <= 0 { occurredAtMS = defaults.receivedAtMS } - return client.AppTrackingEvent{ + event := client.AppTrackingEvent{ AppCode: defaults.appCode, EventID: item.EventID, EventName: item.EventName, - EventType: strings.TrimSpace(item.EventType), - Screen: strings.TrimSpace(item.Screen), - TargetType: strings.TrimSpace(item.TargetType), - TargetID: strings.TrimSpace(item.TargetID), + EventType: item.EventType, + Screen: item.Screen, + TargetType: item.TargetType, + TargetID: item.TargetID, UserID: defaults.userID, DeviceID: deviceID, SessionID: firstTrimmed(defaults.sessionID, item.SessionID), @@ -173,25 +161,60 @@ func appTrackingEventFromRequest(item appEventRequest, defaults appTrackingEvent AppVersion: firstTrimmed(item.AppVersion, defaults.appVersion), Language: firstTrimmed(item.Language, defaults.language), Timezone: firstTrimmed(item.Timezone, defaults.timezone), - CountryID: defaults.countryID, - RegionID: defaults.regionID, DurationMS: item.DurationMS, Success: item.Success, - ErrorCode: strings.TrimSpace(item.ErrorCode), + ErrorCode: item.ErrorCode, Properties: properties, OccurredAtMS: occurredAtMS, - }, true + } + if !normalizeAppTrackingEventText(&event) || event.EventID == "" || event.EventName == "" { + return client.AppTrackingEvent{}, false + } + if defaults.userID <= 0 && event.DeviceID == "" { + // 匿名埋点没有服务端 user_id，只能依赖稳定 device_id 做排障和后续粗粒度去重。 + return client.AppTrackingEvent{}, false + } + return event, true } func normalizeAppEventProperties(raw json.RawMessage) (json.RawMessage, bool) { - trimmed := bytes.TrimSpace(raw) - if len(trimmed) == 0 || bytes.Equal(trimmed, []byte("null")) { - return nil, true + return apptracking.NormalizeProperties(raw) +} + +func normalizeAppTrackingEventText(event *client.AppTrackingEvent) bool { + if event == nil { + return false } - if len(trimmed) > appEventMaxPropertiesBytes || !json.Valid(trimmed) { - return nil, false + // Every field uses the same byte limit as statistics persistence. Returning + // false is deliberate: identifiers and diagnostic error codes must be + // accepted intact or rejected, never silently truncated into another fact. + fields := []struct { + value *string + maxBytes int + }{ + {value: &event.AppCode, maxBytes: apptracking.MaxAppCodeBytes}, + {value: &event.EventID, maxBytes: apptracking.MaxEventIDBytes}, + {value: &event.EventName, maxBytes: apptracking.MaxEventNameBytes}, + {value: &event.EventType, maxBytes: apptracking.MaxEventTypeBytes}, + {value: &event.Screen, maxBytes: apptracking.MaxScreenBytes}, + {value: &event.TargetType, maxBytes: apptracking.MaxTargetTypeBytes}, + {value: &event.TargetID, maxBytes: apptracking.MaxTargetIDBytes}, + {value: &event.DeviceID, maxBytes: apptracking.MaxDeviceIDBytes}, + {value: &event.SessionID, maxBytes: apptracking.MaxSessionIDBytes}, + {value: &event.Platform, maxBytes: apptracking.MaxPlatformBytes}, + {value: &event.AppVersion, maxBytes: apptracking.MaxAppVersionBytes}, + {value: &event.Language, maxBytes: apptracking.MaxLanguageBytes}, + {value: &event.Timezone, maxBytes: apptracking.MaxTimezoneBytes}, + {value: &event.ErrorCode, maxBytes: apptracking.MaxErrorCodeBytes}, } - return json.RawMessage(trimmed), true + for _, field := range fields { + normalized, ok := apptracking.NormalizeText(*field.value, field.maxBytes) + if !ok { + return false + } + *field.value = normalized + } + return true } func appEventRequestLanguage(request *http.Request) string { diff --git a/services/gateway-service/internal/transport/http/response_test.go b/services/gateway-service/internal/transport/http/response_test.go index f0c77e8b..9e0978d3 100644 --- a/services/gateway-service/internal/transport/http/response_test.go +++ b/services/gateway-service/internal/transport/http/response_test.go @@ -4625,6 +4625,36 @@ func TestReportAppEventsRejectsInvalidRequests(t *testing.T) { } } +func TestReportAppEventsRejects185ByteErrorCodeBeforeDownstream(t *testing.T) { + statisticsClient := &fakeStatisticsClient{} + handler := NewHandlerWithClients(&fakeRoomClient{}, nil, nil, &fakeUserProfileClient{}) + handler.SetStatisticsClient(statisticsClient) + router := handler.Routes(auth.NewVerifier("secret")) + body, err := json.Marshal(map[string]any{ + "events": []map[string]any{{ + "event_id": "evt-error-code-too-long", + "event_name": "pay_failed", + "device_id": "dev-1", + "error_code": strings.Repeat("e", 185), + }}, + }) + if err != nil { + t.Fatalf("marshal app events request: %v", err) + } + request := httptest.NewRequest(http.MethodPost, "/api/v1/app/events", bytes.NewReader(body)) + request.Header.Set("X-Request-ID", "req-app-events-error-code-limit") + recorder := httptest.NewRecorder() + + router.ServeHTTP(recorder, request) + + // Regression: statistics-service used to reject this after forwarding and + // gateway translated that downstream 400 into a misleading 502. + assertEnvelope(t, recorder, http.StatusBadRequest, httpkit.CodeInvalidArgument, "req-app-events-error-code-limit") + if len(statisticsClient.lastAppEvents) != 0 { + t.Fatalf("oversized error_code must not reach statistics-service: %+v", statisticsClient.lastAppEvents) + } +} + func TestReportAppEventsUpstreamFailure(t *testing.T) { statisticsClient := &fakeStatisticsClient{appErr: errors.New("statistics unavailable")} handler := NewHandlerWithClients(&fakeRoomClient{}, nil, nil, &fakeUserProfileClient{}) diff --git a/services/gateway-service/internal/transport/http/roomapi/gift_panel_pipeline.go b/services/gateway-service/internal/transport/http/roomapi/gift_panel_pipeline.go index c18b991c..d0b4fe5a 100644 --- a/services/gateway-service/internal/transport/http/roomapi/gift_panel_pipeline.go +++ b/services/gateway-service/internal/transport/http/roomapi/gift_panel_pipeline.go @@ -79,6 +79,12 @@ func logRoomGiftPanelStage(ctx context.Context, requestID string, roomID string, slog.Bool("degraded", degraded), } switch { + case roomGiftPanelStageCanceled(err): + // 首个必要阶段失败后 handler 会取消仍在运行的 sibling RPC；这些 Canceled + // 只是请求内并发已被收敛，不是新的下游故障。保留 INFO 轨迹用于 request_id + // 关联，但不能为同一个主错误再制造多条 ERROR 告警。 + attrs = append(attrs, slog.String("error", err.Error()), slog.Bool("canceled", true)) + logx.Info(logCtx, "gateway_room_gift_panel_stage", attrs...) case err != nil && degraded: attrs = append(attrs, slog.String("error", err.Error())) logx.Warn(logCtx, "gateway_room_gift_panel_stage", attrs...) @@ -91,6 +97,10 @@ func logRoomGiftPanelStage(ctx context.Context, requestID string, roomID string, } } +func roomGiftPanelStageCanceled(err error) bool { + return err != nil && (errors.Is(err, context.Canceled) || status.Code(err) == codes.Canceled) +} + // serveRoomGiftPanel 在单个明确预算内聚合 owner service 读模型。 // 第一阶段只并发互不依赖的 RPC；依赖 visible_region_id 的礼物配置在快照返回后启动，避免猜测区域。 func (h *Handler) serveRoomGiftPanel(writer http.ResponseWriter, request *http.Request) { diff --git a/services/gateway-service/internal/transport/http/roomapi/gift_panel_pipeline_test.go b/services/gateway-service/internal/transport/http/roomapi/gift_panel_pipeline_test.go index a9001df8..c08b833d 100644 --- a/services/gateway-service/internal/transport/http/roomapi/gift_panel_pipeline_test.go +++ b/services/gateway-service/internal/transport/http/roomapi/gift_panel_pipeline_test.go @@ -4,6 +4,7 @@ import ( "context" "encoding/json" "errors" + "fmt" "net/http" "net/http/httptest" "sync/atomic" @@ -18,6 +19,9 @@ import ( "hyapp/services/gateway-service/internal/auth" gatewayclient "hyapp/services/gateway-service/internal/client" "hyapp/services/gateway-service/internal/transport/http/httpkit" + + "google.golang.org/grpc/codes" + "google.golang.org/grpc/status" ) type giftPanelTestRoomQueryClient struct { @@ -164,6 +168,21 @@ func TestRoomGiftPanelProductionBudgetsStayBelowFlutterTimeout(t *testing.T) { } } +func TestRoomGiftPanelStageCanceledRecognizesLocalAndGRPCCancellation(t *testing.T) { + for _, err := range []error{ + context.Canceled, + fmt.Errorf("query interrupted: %w", context.Canceled), + status.Error(codes.Canceled, "context canceled"), + } { + if !roomGiftPanelStageCanceled(err) { + t.Fatalf("expected cancellation for %v", err) + } + } + if roomGiftPanelStageCanceled(status.Error(codes.DeadlineExceeded, "deadline exceeded")) { + t.Fatal("deadline exhaustion is a real latency failure and must remain error-level") + } +} + func TestRoomGiftPanelRunsIndependentStagesConcurrentlyAndSkipsUnusedEnrichment(t *testing.T) { phaseOne := newGiftPanelStageGate() phaseTwo := newGiftPanelStageGate() diff --git a/services/notice-service/cmd/replay-deliveries/main.go b/services/notice-service/cmd/replay-deliveries/main.go new file mode 100644 index 00000000..81aca04d --- /dev/null +++ b/services/notice-service/cmd/replay-deliveries/main.go @@ -0,0 +1,505 @@ +// Command replay-deliveries replays an explicitly selected set of expired +// notice delivery leases from immutable owner outbox facts. +package main + +import ( + "context" + "database/sql" + "encoding/json" + "errors" + "flag" + "fmt" + "log" + "os" + "sort" + "strings" + "time" + + roomeventsv1 "hyapp.local/api/proto/events/room/v1" + "hyapp/pkg/appcode" + "hyapp/pkg/tencentim" + "hyapp/pkg/walletmq" + "hyapp/services/notice-service/internal/config" + "hyapp/services/notice-service/internal/modules/roomnotice" + "hyapp/services/notice-service/internal/modules/walletnotice" + mysqlplatform "hyapp/services/notice-service/internal/platform/mysql" +) + +const ( + walletSource = "wallet_outbox" + roomSource = "room_outbox" + c2cChannel = "tencent_im_c2c" +) + +type repeatedFlag []string + +func (values *repeatedFlag) String() string { return strings.Join(*values, ",") } + +func (values *repeatedFlag) Set(value string) error { + value = strings.TrimSpace(value) + if value == "" { + return errors.New("flag value cannot be empty") + } + *values = append(*values, value) + return nil +} + +type deliverySelector struct { + SourceName string + AppCode string + EventID string +} + +func (selector deliverySelector) String() string { + return selector.SourceName + ":" + selector.AppCode + ":" + selector.EventID +} + +type deliverySelectorFlag []deliverySelector + +func (values *deliverySelectorFlag) String() string { + parts := make([]string, 0, len(*values)) + for _, selector := range *values { + parts = append(parts, selector.String()) + } + return strings.Join(parts, ",") +} + +func (values *deliverySelectorFlag) Set(value string) error { + parts := strings.SplitN(strings.TrimSpace(value), ":", 3) + if len(parts) != 3 { + return errors.New("delivery must use source_name:app_code:event_id") + } + selector := deliverySelector{ + SourceName: strings.TrimSpace(parts[0]), + AppCode: strings.TrimSpace(parts[1]), + EventID: strings.TrimSpace(parts[2]), + } + if selector.SourceName != walletSource && selector.SourceName != roomSource { + return fmt.Errorf("unsupported delivery source %q", selector.SourceName) + } + if selector.AppCode == "" || selector.EventID == "" { + return errors.New("delivery app_code and event_id are required") + } + selector.AppCode = appcode.Normalize(selector.AppCode) + *values = append(*values, selector) + return nil +} + +type deliveryCandidate struct { + Selector string `json:"selector"` + SourceName string `json:"source_name"` + AppCode string `json:"app_code"` + EventID string `json:"event_id"` + Status string `json:"status"` + RetryCount int `json:"retry_count"` + UpdatedAtMS int64 `json:"updated_at_ms"` +} + +type replayItem struct { + Candidate deliveryCandidate + WalletMessage *walletmq.WalletOutboxMessage + RoomEnvelope *roomeventsv1.EventEnvelope +} + +func main() { + configPath := flag.String("config", "services/notice-service/configs/config.yaml", "path to notice-service config") + execute := flag.Bool("execute", false, "publish selected due deliveries; default is a read-only preview") + limit := flag.Int("limit", 100, "maximum candidate count, 1..500") + var eventIDs repeatedFlag + var appCodes repeatedFlag + var deliveries deliverySelectorFlag + flag.Var(&eventIDs, "event-id", "preview filter for a source event ID; repeat for multiple events") + flag.Var(&appCodes, "app-code", "required App partition for indexed preview; repeat for multiple Apps") + flag.Var(&deliveries, "delivery", "exact source_name:app_code:event_id selector copied from preview; repeat for multiple deliveries") + flag.Parse() + + if *limit <= 0 || *limit > 500 { + log.Fatal("limit must be between 1 and 500") + } + if len(deliveries) > 0 && (len(eventIDs) > 0 || len(appCodes) > 0) { + log.Fatal("--delivery cannot be combined with --event-id or --app-code") + } + if *execute && len(deliveries) == 0 { + // A write run is pinned to full primary-key selectors copied from preview; + // separate App/event filters can form an accidental cross product. + log.Fatal("--execute requires at least one exact --delivery from a preview") + } + if len(deliveries) > *limit { + log.Fatal("exact delivery count exceeds --limit") + } + if len(deliveries) == 0 && len(appCodes) == 0 { + // notice_delivery_events indexes start with source_name,app_code. Refusing + // an unpartitioned preview prevents a LIMIT query from scanning a 20+ GiB table. + log.Fatal("preview requires at least one --app-code or exact --delivery") + } + + cfg, err := config.Load(*configPath) + if err != nil { + log.Fatal(err) + } + ctx, cancel := context.WithTimeout(context.Background(), 2*time.Minute) + defer cancel() + store, err := mysqlplatform.Open(ctx, cfg.MySQLDSN) + if err != nil { + log.Fatal(err) + } + defer func() { _ = store.Close() }() + + nowMS := time.Now().UnixMilli() + var candidates []deliveryCandidate + if len(deliveries) > 0 { + candidates, err = loadExactDueDeliveries(ctx, store.DB, deliveries, nowMS) + } else { + candidates, err = listDueDeliveries(ctx, store.DB, eventIDs, appCodes, *limit, nowMS) + } + if err != nil { + log.Fatal(err) + } + if !*execute { + encoder := json.NewEncoder(os.Stdout) + for _, candidate := range candidates { + if err := encoder.Encode(candidate); err != nil { + log.Fatal(err) + } + } + fmt.Fprintf(os.Stderr, "previewed %d due delivery rows; no writes performed\n", len(candidates)) + return + } + if !cfg.TencentIM.Enabled { + log.Fatal("tencent_im.enabled is required for replay") + } + walletRepo, err := walletnotice.NewMySQLRepository(store.DB, cfg.WalletDatabase) + if err != nil { + log.Fatal(err) + } + roomRepo, err := roomnotice.NewMySQLRepository(store.DB, cfg.RoomDatabase) + if err != nil { + log.Fatal(err) + } + items, err := loadReplayItems(ctx, candidates, walletRepo, roomRepo) + if err != nil { + // Preload every immutable owner fact before creating an IM client or + // publishing the first item. One missing/corrupt fact therefore aborts the + // exact set without producing a partial operator repair. + log.Fatal(err) + } + publisher, err := tencentim.NewRESTClient(cfg.TencentIM.RESTConfig()) + if err != nil { + log.Fatal(err) + } + walletService := walletnotice.New(walletnotice.Config{NodeID: cfg.NodeID}, walletRepo, publisher) + roomService := roomnotice.New(roomnotice.Config{NodeID: cfg.NodeID}, roomRepo, publisher) + walletOptions := walletWorkerOptions(cfg) + roomOptions := roomWorkerOptions(cfg) + + failures := 0 + for _, item := range items { + candidate := item.Candidate + candidateCtx := appcode.WithContext(ctx, candidate.AppCode) + var replayErr error + switch candidate.SourceName { + case walletSource: + if item.WalletMessage == nil { + replayErr = errors.New("wallet owner snapshot is missing") + } else { + _, replayErr = walletService.ProcessWalletOutboxMessage(candidateCtx, *item.WalletMessage, walletOptions) + } + case roomSource: + if item.RoomEnvelope == nil { + replayErr = errors.New("room owner snapshot is missing") + } else { + _, replayErr = roomService.ProcessRoomOutboxEnvelope(candidateCtx, item.RoomEnvelope, roomOptions) + } + default: + replayErr = fmt.Errorf("unsupported source %q", candidate.SourceName) + } + if replayErr == nil { + replayErr = requireDelivered(ctx, store.DB, candidate) + } + if replayErr != nil { + failures++ + fmt.Fprintf(os.Stderr, "replay failed source=%s app_code=%s event_id=%s: %v\n", candidate.SourceName, candidate.AppCode, candidate.EventID, replayErr) + continue + } + fmt.Printf("delivered source=%s app_code=%s event_id=%s\n", candidate.SourceName, candidate.AppCode, candidate.EventID) + } + if failures > 0 { + log.Fatalf("replay finished with %d failures out of %d selected rows", failures, len(candidates)) + } + fmt.Printf("replay delivered %d selected rows\n", len(candidates)) +} + +func loadReplayItems(ctx context.Context, candidates []deliveryCandidate, walletRepo *walletnotice.MySQLRepository, roomRepo *roomnotice.MySQLRepository) ([]replayItem, error) { + items := make([]replayItem, 0, len(candidates)) + for _, candidate := range candidates { + candidateCtx := appcode.WithContext(ctx, candidate.AppCode) + item := replayItem{Candidate: candidate} + switch candidate.SourceName { + case walletSource: + if walletRepo == nil { + return nil, errors.New("wallet owner repository is not configured") + } + message, err := walletRepo.LoadWalletOutboxMessage(candidateCtx, candidate.AppCode, candidate.EventID) + if err != nil { + return nil, fmt.Errorf("load owner snapshot source=%s app_code=%s event_id=%s: %w", candidate.SourceName, candidate.AppCode, candidate.EventID, err) + } + item.WalletMessage = &message + case roomSource: + if roomRepo == nil { + return nil, errors.New("room owner repository is not configured") + } + envelope, err := roomRepo.LoadRoomOutboxEnvelope(candidateCtx, candidate.AppCode, candidate.EventID) + if err != nil { + return nil, fmt.Errorf("load owner snapshot source=%s app_code=%s event_id=%s: %w", candidate.SourceName, candidate.AppCode, candidate.EventID, err) + } + item.RoomEnvelope = envelope + default: + return nil, fmt.Errorf("unsupported source %q", candidate.SourceName) + } + items = append(items, item) + } + return items, nil +} + +func listDueDeliveries(ctx context.Context, db *sql.DB, eventIDs []string, appCodes []string, limit int, nowMS int64) ([]deliveryCandidate, error) { + if db == nil { + return nil, errors.New("notice database is not configured") + } + if limit <= 0 || limit > 500 { + return nil, errors.New("limit must be between 1 and 500") + } + apps := normalizeAppCodes(appCodes) + if len(apps) == 0 { + return nil, errors.New("at least one app_code is required for indexed preview") + } + if len(apps) > 32 { + return nil, errors.New("app_code filter count exceeds 32") + } + ids := normalizeEventIDs(eventIDs) + if len(ids) > 500 { + return nil, errors.New("event_id filter count exceeds 500") + } + candidates := make([]deliveryCandidate, 0, limit) + for _, sourceName := range []string{walletSource, roomSource} { + for _, appCodeValue := range apps { + branches := []struct { + indexName string + status string + dueColumn string + }{ + {indexName: "idx_notice_delivery_retry", status: "retryable", dueColumn: "next_retry_at_ms"}, + {indexName: "idx_notice_delivery_lock", status: "delivering", dueColumn: "lock_until_ms"}, + } + for _, branch := range branches { + query, args := dueDeliveryBranch(sourceName, appCodeValue, branch.indexName, branch.status, branch.dueColumn, ids, limit, nowMS) + branchCandidates, err := queryDueDeliveryCandidates(ctx, db, query, args...) + if err != nil { + return nil, err + } + candidates = append(candidates, branchCandidates...) + } + } + } + sort.Slice(candidates, func(i, j int) bool { + if candidates[i].UpdatedAtMS != candidates[j].UpdatedAtMS { + return candidates[i].UpdatedAtMS < candidates[j].UpdatedAtMS + } + return candidates[i].Selector < candidates[j].Selector + }) + if len(candidates) > limit { + candidates = candidates[:limit] + } + return candidates, nil +} + +func loadExactDueDeliveries(ctx context.Context, db *sql.DB, selectors []deliverySelector, nowMS int64) ([]deliveryCandidate, error) { + if db == nil { + return nil, errors.New("notice database is not configured") + } + normalized, err := normalizeDeliverySelectors(selectors) + if err != nil { + return nil, err + } + candidates := make([]deliveryCandidate, 0, len(selectors)) + for _, selector := range normalized { + key := selector.String() + var candidate deliveryCandidate + err = db.QueryRowContext(ctx, ` + SELECT source_name, app_code, source_event_id, status, retry_count, updated_at_ms + FROM notice_delivery_events FORCE INDEX (PRIMARY) + WHERE source_name = ? AND app_code = ? AND source_event_id = ? AND channel = ? + AND ((status = 'retryable' AND next_retry_at_ms <= ?) + OR (status = 'delivering' AND lock_until_ms <= ?))`, + selector.SourceName, + selector.AppCode, + selector.EventID, + c2cChannel, + nowMS, + nowMS, + ).Scan(&candidate.SourceName, &candidate.AppCode, &candidate.EventID, &candidate.Status, &candidate.RetryCount, &candidate.UpdatedAtMS) + if errors.Is(err, sql.ErrNoRows) { + return nil, fmt.Errorf("delivery %q does not exist or is no longer due", key) + } + if err != nil { + return nil, err + } + if candidate.SourceName != selector.SourceName || candidate.AppCode != selector.AppCode || candidate.EventID != selector.EventID { + return nil, fmt.Errorf("delivery %q returned a mismatched primary key", key) + } + candidate.Selector = key + candidates = append(candidates, candidate) + } + return candidates, nil +} + +func normalizeDeliverySelectors(values []deliverySelector) ([]deliverySelector, error) { + normalized := make([]deliverySelector, 0, len(values)) + seen := make(map[string]struct{}, len(values)) + for _, value := range values { + selector := deliverySelector{ + SourceName: strings.TrimSpace(value.SourceName), + AppCode: strings.TrimSpace(value.AppCode), + EventID: strings.TrimSpace(value.EventID), + } + if selector.SourceName != walletSource && selector.SourceName != roomSource { + return nil, fmt.Errorf("unsupported delivery source %q", selector.SourceName) + } + if selector.AppCode == "" || selector.EventID == "" { + return nil, errors.New("delivery app_code and event_id are required") + } + selector.AppCode = appcode.Normalize(selector.AppCode) + key := selector.String() + if _, exists := seen[key]; exists { + return nil, fmt.Errorf("duplicate delivery selector %q", key) + } + seen[key] = struct{}{} + normalized = append(normalized, selector) + } + return normalized, nil +} + +func dueDeliveryBranch(sourceName string, appCodeValue string, indexName string, status string, dueColumn string, eventIDs []string, limit int, nowMS int64) (string, []any) { + // Callers pass only compile-time index/status/column tuples. Splitting status + // branches gives MySQL the complete (source,app,channel,status,due_time) prefix. + query := ` + SELECT source_name, app_code, source_event_id, status, retry_count, updated_at_ms + FROM notice_delivery_events FORCE INDEX (` + indexName + `) + WHERE source_name = ? AND app_code = ? AND channel = ? AND status = ? + AND ` + dueColumn + ` <= ?` + args := []any{sourceName, appCodeValue, c2cChannel, status, nowMS} + if len(eventIDs) > 0 { + query += " AND source_event_id IN (" + placeholders(len(eventIDs)) + ")" + for _, eventID := range eventIDs { + args = append(args, eventID) + } + } + query += " ORDER BY updated_at_ms ASC, source_event_id ASC LIMIT ?" + args = append(args, limit) + return query, args +} + +func queryDueDeliveryCandidates(ctx context.Context, db *sql.DB, query string, args ...any) ([]deliveryCandidate, error) { + rows, err := db.QueryContext(ctx, query, args...) + if err != nil { + return nil, err + } + defer rows.Close() + candidates := make([]deliveryCandidate, 0) + for rows.Next() { + var candidate deliveryCandidate + if err := rows.Scan(&candidate.SourceName, &candidate.AppCode, &candidate.EventID, &candidate.Status, &candidate.RetryCount, &candidate.UpdatedAtMS); err != nil { + return nil, err + } + candidate.Selector = deliverySelector{SourceName: candidate.SourceName, AppCode: candidate.AppCode, EventID: candidate.EventID}.String() + candidates = append(candidates, candidate) + } + return candidates, rows.Err() +} + +func normalizeAppCodes(values []string) []string { + normalized := make([]string, 0, len(values)) + seen := make(map[string]struct{}, len(values)) + for _, value := range values { + if strings.TrimSpace(value) == "" { + continue + } + value = appcode.Normalize(value) + if _, exists := seen[value]; exists { + continue + } + seen[value] = struct{}{} + normalized = append(normalized, value) + } + return normalized +} + +func normalizeEventIDs(values []string) []string { + normalized := make([]string, 0, len(values)) + seen := make(map[string]struct{}, len(values)) + for _, value := range values { + value = strings.TrimSpace(value) + if value == "" { + continue + } + if _, exists := seen[value]; exists { + continue + } + seen[value] = struct{}{} + normalized = append(normalized, value) + } + return normalized +} + +func requireDelivered(ctx context.Context, db *sql.DB, candidate deliveryCandidate) error { + var status string + err := db.QueryRowContext(ctx, ` + SELECT status + FROM notice_delivery_events + WHERE source_name = ? AND app_code = ? AND source_event_id = ? AND channel = ?`, + candidate.SourceName, + candidate.AppCode, + candidate.EventID, + c2cChannel, + ).Scan(&status) + if err != nil { + return err + } + if status != "delivered" { + return fmt.Errorf("delivery status is %q after replay", status) + } + return nil +} + +func walletWorkerOptions(cfg config.Config) walletnotice.WalletNoticeWorkerOptions { + worker := cfg.WalletNoticeWorker + return walletnotice.WalletNoticeWorkerOptions{ + WorkerID: "wallet-notice-replay-" + cfg.NodeID, + BatchSize: worker.BatchSize, + LockTTL: worker.LockTTL, + PublishTimeout: worker.PublishTimeout, + MaxRetryCount: worker.MaxRetryCount, + InitialBackoff: worker.InitialBackoff, + MaxBackoff: worker.MaxBackoff, + } +} + +func roomWorkerOptions(cfg config.Config) roomnotice.RoomNoticeWorkerOptions { + worker := cfg.RoomNoticeWorker + return roomnotice.RoomNoticeWorkerOptions{ + WorkerID: "room-notice-replay-" + cfg.NodeID, + BatchSize: worker.BatchSize, + LockTTL: worker.LockTTL, + PublishTimeout: worker.PublishTimeout, + MaxRetryCount: worker.MaxRetryCount, + InitialBackoff: worker.InitialBackoff, + MaxBackoff: worker.MaxBackoff, + } +} + +func placeholders(count int) string { + values := make([]string, count) + for index := range values { + values[index] = "?" + } + return strings.Join(values, ",") +} diff --git a/services/notice-service/cmd/replay-deliveries/main_test.go b/services/notice-service/cmd/replay-deliveries/main_test.go new file mode 100644 index 00000000..d651ea87 --- /dev/null +++ b/services/notice-service/cmd/replay-deliveries/main_test.go @@ -0,0 +1,112 @@ +package main + +import ( + "context" + "database/sql" + "regexp" + "strings" + "testing" + + "github.com/DATA-DOG/go-sqlmock" +) + +func TestDeliverySelectorFlagKeepsColonInsideEventID(t *testing.T) { + var values deliverySelectorFlag + if err := values.Set("wallet_outbox: FAMI :wallet:balance:evt-1"); err != nil { + t.Fatalf("parse exact delivery selector: %v", err) + } + if len(values) != 1 || values[0].SourceName != walletSource || values[0].AppCode != "fami" || values[0].EventID != "wallet:balance:evt-1" { + t.Fatalf("parsed selector mismatch: %+v", values) + } + if err := values.Set("unknown:lalu:event-2"); err == nil { + t.Fatal("unsupported source must be rejected before any database access") + } + if err := values.Set("room_outbox::event-3"); err == nil { + t.Fatal("empty app_code must not default to lalu") + } +} + +func TestLoadExactDueDeliveriesUsesFullPrimaryKey(t *testing.T) { + db, mock, err := sqlmock.New() + if err != nil { + t.Fatalf("create sqlmock: %v", err) + } + defer db.Close() + nowMS := int64(1_700_000_000_000) + query := regexp.QuoteMeta(` + SELECT source_name, app_code, source_event_id, status, retry_count, updated_at_ms + FROM notice_delivery_events FORCE INDEX (PRIMARY) + WHERE source_name = ? AND app_code = ? AND source_event_id = ? AND channel = ? + AND ((status = 'retryable' AND next_retry_at_ms <= ?) + OR (status = 'delivering' AND lock_until_ms <= ?))`) + mock.ExpectQuery(query). + WithArgs(walletSource, "fami", "wallet:evt-1", c2cChannel, nowMS, nowMS). + WillReturnRows(sqlmock.NewRows([]string{"source_name", "app_code", "source_event_id", "status", "retry_count", "updated_at_ms"}). + AddRow(walletSource, "fami", "wallet:evt-1", "retryable", 2, nowMS-1000)) + + candidates, err := loadExactDueDeliveries(context.Background(), db, []deliverySelector{{SourceName: walletSource, AppCode: " FAMI ", EventID: "wallet:evt-1"}}, nowMS) + if err != nil { + t.Fatalf("load exact due delivery: %v", err) + } + if len(candidates) != 1 || candidates[0].Selector != "wallet_outbox:fami:wallet:evt-1" || candidates[0].RetryCount != 2 { + t.Fatalf("exact candidate mismatch: %+v", candidates) + } + if err := mock.ExpectationsWereMet(); err != nil { + t.Fatalf("exact query did not use the expected primary key: %v", err) + } +} + +func TestLoadExactDueDeliveriesRejectsDuplicateBeforeQuery(t *testing.T) { + db, mock, err := sqlmock.New() + if err != nil { + t.Fatalf("create sqlmock: %v", err) + } + defer db.Close() + selectors := []deliverySelector{ + {SourceName: roomSource, AppCode: "lalu", EventID: "evt-1"}, + {SourceName: roomSource, AppCode: " LALU ", EventID: "evt-1"}, + } + if _, err := loadExactDueDeliveries(context.Background(), db, selectors, 1000); err == nil || !strings.Contains(err.Error(), "duplicate") { + t.Fatalf("duplicate selector error = %v", err) + } + if err := mock.ExpectationsWereMet(); err != nil { + t.Fatalf("duplicate validation unexpectedly queried MySQL: %v", err) + } +} + +func TestLoadExactDueDeliveriesRejectsNotDueRow(t *testing.T) { + db, mock, err := sqlmock.New() + if err != nil { + t.Fatalf("create sqlmock: %v", err) + } + defer db.Close() + mock.ExpectQuery("FROM notice_delivery_events FORCE INDEX \\(PRIMARY\\)"). + WillReturnError(sql.ErrNoRows) + _, err = loadExactDueDeliveries(context.Background(), db, []deliverySelector{{SourceName: roomSource, AppCode: "lalu", EventID: "evt-1"}}, 1000) + if err == nil || !strings.Contains(err.Error(), "no longer due") { + t.Fatalf("not-due selector error = %v", err) + } +} + +func TestDueDeliveryBranchUsesCompleteRetryIndexPrefix(t *testing.T) { + query, args := dueDeliveryBranch(walletSource, "huwaa", "idx_notice_delivery_retry", "retryable", "next_retry_at_ms", []string{"evt-1", "evt-2"}, 20, 1234) + for _, fragment := range []string{ + "FORCE INDEX (idx_notice_delivery_retry)", + "source_name = ? AND app_code = ? AND channel = ? AND status = ?", + "next_retry_at_ms <= ?", + "source_event_id IN (?,?)", + } { + if !strings.Contains(query, fragment) { + t.Fatalf("indexed preview query is missing %q:\n%s", fragment, query) + } + } + wantArgs := []any{walletSource, "huwaa", c2cChannel, "retryable", int64(1234), "evt-1", "evt-2", 20} + if len(args) != len(wantArgs) { + t.Fatalf("preview args = %v, want %v", args, wantArgs) + } + for index := range args { + if args[index] != wantArgs[index] { + t.Fatalf("preview arg[%d] = %#v, want %#v", index, args[index], wantArgs[index]) + } + } +} diff --git a/services/notice-service/internal/modules/roomnotice/mysql_repository.go b/services/notice-service/internal/modules/roomnotice/mysql_repository.go index 8181f24f..b1b46bdd 100644 --- a/services/notice-service/internal/modules/roomnotice/mysql_repository.go +++ b/services/notice-service/internal/modules/roomnotice/mysql_repository.go @@ -251,6 +251,47 @@ func (r *MySQLRepository) ClaimRoomKickEnvelope(ctx context.Context, workerID st return event, true, nil } +// LoadRoomOutboxEnvelope loads one immutable owner event for an exact offline +// replay. Requiring the full primary key prevents repair tooling from scanning or +// claiming room-service state that was not selected by an operator. +func (r *MySQLRepository) LoadRoomOutboxEnvelope(ctx context.Context, appCode string, eventID string) (*roomeventsv1.EventEnvelope, error) { + if r == nil || r.db == nil { + return nil, xerr.New(xerr.Unavailable, "notice repository is not configured") + } + appCode = strings.TrimSpace(appCode) + eventID = strings.TrimSpace(eventID) + if appCode == "" || eventID == "" { + return nil, xerr.New(xerr.InvalidArgument, "app_code and event_id are required") + } + appCode = appcode.Normalize(appCode) + + var ( + ownerRoomID string + envelopeBytes []byte + ) + query := fmt.Sprintf(` + SELECT room_id, envelope + FROM %s + WHERE app_code = ? AND event_id = ? AND event_type = ?`, r.roomOutboxTable) + if err := r.db.QueryRowContext(ctx, query, appCode, eventID, eventRoomUserKicked).Scan(&ownerRoomID, &envelopeBytes); err != nil { + return nil, err + } + var envelope roomeventsv1.EventEnvelope + if err := proto.Unmarshal(envelopeBytes, &envelope); err != nil { + return nil, err + } + // Validate both the requested identity and the current consumer contract. A + // mismatched historical blob must never be delivered under another event's + // notice_delivery_events key. + if envelope.GetAppCode() != appCode || envelope.GetEventId() != eventID || envelope.GetRoomId() != ownerRoomID { + return nil, fmt.Errorf("room outbox envelope identity does not match requested key") + } + if _, err := roomKickEventFromEnvelope(&envelope, envelope.GetOccurredAtMs()); err != nil { + return nil, err + } + return &envelope, nil +} + func (r *MySQLRepository) lockRoomKickEvent(ctx context.Context, tx *sql.Tx, candidate roomKickCandidate) (RoomKickEvent, error) { var ( event RoomKickEvent diff --git a/services/notice-service/internal/modules/roomnotice/replay_repository_test.go b/services/notice-service/internal/modules/roomnotice/replay_repository_test.go new file mode 100644 index 00000000..b5b3f23e --- /dev/null +++ b/services/notice-service/internal/modules/roomnotice/replay_repository_test.go @@ -0,0 +1,68 @@ +package roomnotice + +import ( + "context" + "regexp" + "testing" + + "github.com/DATA-DOG/go-sqlmock" + "google.golang.org/protobuf/proto" + roomeventsv1 "hyapp.local/api/proto/events/room/v1" +) + +func TestLoadRoomOutboxEnvelopeRejectsEmptyAppInsteadOfDefaultingToLalu(t *testing.T) { + db, mock, err := sqlmock.New() + if err != nil { + t.Fatalf("create sqlmock: %v", err) + } + defer db.Close() + repository := &MySQLRepository{db: db, roomOutboxTable: "`hyapp_room`.`room_outbox`"} + if _, err := repository.LoadRoomOutboxEnvelope(context.Background(), "", "evt-1"); err == nil { + t.Fatal("empty app_code must be rejected before owner table access") + } + if err := mock.ExpectationsWereMet(); err != nil { + t.Fatalf("empty key unexpectedly queried room owner table: %v", err) + } +} + +func TestLoadRoomOutboxEnvelopeUsesExactImmutableOwnerEnvelope(t *testing.T) { + db, mock, err := sqlmock.New() + if err != nil { + t.Fatalf("create sqlmock: %v", err) + } + defer db.Close() + body, err := proto.Marshal(&roomeventsv1.RoomUserKicked{ActorUserId: 7, TargetUserId: 42}) + if err != nil { + t.Fatalf("marshal room kick body: %v", err) + } + envelopeBytes, err := proto.Marshal(&roomeventsv1.EventEnvelope{ + AppCode: "huwaa", + EventId: "room:kick:evt-1", + EventType: eventRoomUserKicked, + RoomId: "room-1", + RoomVersion: 9, + OccurredAtMs: 1700000000000, + Body: body, + }) + if err != nil { + t.Fatalf("marshal room envelope: %v", err) + } + repository := &MySQLRepository{db: db, roomOutboxTable: "`hyapp_room`.`room_outbox`"} + mock.ExpectQuery(regexp.QuoteMeta(` + SELECT room_id, envelope + FROM `+"`hyapp_room`.`room_outbox`"+` + WHERE app_code = ? AND event_id = ? AND event_type = ?`)). + WithArgs("huwaa", "room:kick:evt-1", eventRoomUserKicked). + WillReturnRows(sqlmock.NewRows([]string{"room_id", "envelope"}).AddRow("room-1", envelopeBytes)) + + envelope, err := repository.LoadRoomOutboxEnvelope(context.Background(), " HUWAA ", "room:kick:evt-1") + if err != nil { + t.Fatalf("load exact room owner envelope: %v", err) + } + if envelope.GetAppCode() != "huwaa" || envelope.GetEventId() != "room:kick:evt-1" || envelope.GetRoomId() != "room-1" { + t.Fatalf("room owner snapshot mismatch: %+v", envelope) + } + if err := mock.ExpectationsWereMet(); err != nil { + t.Fatalf("room replay query mismatch: %v", err) + } +} diff --git a/services/notice-service/internal/modules/walletnotice/mysql_repository.go b/services/notice-service/internal/modules/walletnotice/mysql_repository.go index 9407605d..665ed4fd 100644 --- a/services/notice-service/internal/modules/walletnotice/mysql_repository.go +++ b/services/notice-service/internal/modules/walletnotice/mysql_repository.go @@ -256,6 +256,61 @@ func (r *MySQLRepository) ClaimWalletBalanceMessage(ctx context.Context, workerI return event, true, nil } +// LoadWalletOutboxMessage loads one immutable owner event for an operator-selected +// delivery replay. The exact (app_code, event_id) key is mandatory so an offline +// repair cannot accidentally turn into an unbounded wallet_outbox scan. +func (r *MySQLRepository) LoadWalletOutboxMessage(ctx context.Context, appCode string, eventID string) (walletmq.WalletOutboxMessage, error) { + if r == nil || r.db == nil { + return walletmq.WalletOutboxMessage{}, xerr.New(xerr.Unavailable, "notice repository is not configured") + } + appCode = strings.TrimSpace(appCode) + eventID = strings.TrimSpace(eventID) + if appCode == "" || eventID == "" { + return walletmq.WalletOutboxMessage{}, xerr.New(xerr.InvalidArgument, "app_code and event_id are required") + } + appCode = appcode.Normalize(appCode) + + message := walletmq.WalletOutboxMessage{MessageType: walletmq.MessageTypeWalletOutboxEvent} + query := fmt.Sprintf(` + SELECT app_code, event_id, event_type, transaction_id, command_id, user_id, + asset_type, available_delta, frozen_delta, + COALESCE(CAST(payload AS CHAR), '{}'), created_at_ms + FROM %s + WHERE app_code = ? AND event_id = ? + AND event_type IN (?, ?) AND asset_type <> ?`, r.walletOutboxTable) + err := r.db.QueryRowContext(ctx, query, appCode, eventID, eventWalletBalance, eventVIPActivated, robotCoinAssetType).Scan( + &message.AppCode, + &message.EventID, + &message.EventType, + &message.TransactionID, + &message.CommandID, + &message.UserID, + &message.AssetType, + &message.AvailableDelta, + &message.FrozenDelta, + &message.PayloadJSON, + &message.OccurredAtMS, + ) + if err != nil { + return walletmq.WalletOutboxMessage{}, err + } + // Reuse the live MQ decoder boundary before any external side effect. A + // malformed historical row must stay retryable instead of being sent with a + // subtly different repair-only payload. + encoded, err := walletmq.EncodeWalletOutboxMessage(message) + if err != nil { + return walletmq.WalletOutboxMessage{}, err + } + decoded, err := walletmq.DecodeWalletOutboxMessage(encoded) + if err != nil { + return walletmq.WalletOutboxMessage{}, err + } + if decoded.AppCode != appCode || decoded.EventID != eventID { + return walletmq.WalletOutboxMessage{}, fmt.Errorf("wallet outbox message identity does not match requested key") + } + return decoded, nil +} + func walletBalanceEventFromMessage(message walletmq.WalletOutboxMessage) (WalletBalanceEvent, error) { event := WalletBalanceEvent{ AppCode: appcode.Normalize(message.AppCode), diff --git a/services/notice-service/internal/modules/walletnotice/replay_repository_test.go b/services/notice-service/internal/modules/walletnotice/replay_repository_test.go new file mode 100644 index 00000000..807a5b1c --- /dev/null +++ b/services/notice-service/internal/modules/walletnotice/replay_repository_test.go @@ -0,0 +1,56 @@ +package walletnotice + +import ( + "context" + "regexp" + "testing" + + "github.com/DATA-DOG/go-sqlmock" +) + +func TestLoadWalletOutboxMessageRejectsEmptyAppInsteadOfDefaultingToLalu(t *testing.T) { + db, mock, err := sqlmock.New() + if err != nil { + t.Fatalf("create sqlmock: %v", err) + } + defer db.Close() + repository := &MySQLRepository{db: db, walletOutboxTable: "`hyapp_wallet`.`wallet_outbox`"} + if _, err := repository.LoadWalletOutboxMessage(context.Background(), " ", "evt-1"); err == nil { + t.Fatal("empty app_code must be rejected before owner table access") + } + if err := mock.ExpectationsWereMet(); err != nil { + t.Fatalf("empty key unexpectedly queried wallet owner table: %v", err) + } +} + +func TestLoadWalletOutboxMessageUsesExactImmutablePrivateFact(t *testing.T) { + db, mock, err := sqlmock.New() + if err != nil { + t.Fatalf("create sqlmock: %v", err) + } + defer db.Close() + repository := &MySQLRepository{db: db, walletOutboxTable: "`hyapp_wallet`.`wallet_outbox`"} + mock.ExpectQuery(regexp.QuoteMeta(` + SELECT app_code, event_id, event_type, transaction_id, command_id, user_id, + asset_type, available_delta, frozen_delta, + COALESCE(CAST(payload AS CHAR), '{}'), created_at_ms + FROM `+"`hyapp_wallet`.`wallet_outbox`"+` + WHERE app_code = ? AND event_id = ? + AND event_type IN (?, ?) AND asset_type <> ?`)). + WithArgs("fami", "wallet:evt-1", eventWalletBalance, eventVIPActivated, robotCoinAssetType). + WillReturnRows(sqlmock.NewRows([]string{ + "app_code", "event_id", "event_type", "transaction_id", "command_id", "user_id", + "asset_type", "available_delta", "frozen_delta", "payload", "created_at_ms", + }).AddRow("fami", "wallet:evt-1", eventWalletBalance, "tx-1", "cmd-1", int64(42), "COIN", int64(100), int64(0), `{"balance_after":100}`, int64(1700000000000))) + + message, err := repository.LoadWalletOutboxMessage(context.Background(), " FAMI ", "wallet:evt-1") + if err != nil { + t.Fatalf("load exact wallet owner fact: %v", err) + } + if message.AppCode != "fami" || message.EventID != "wallet:evt-1" || message.UserID != 42 || message.PayloadJSON != `{"balance_after":100}` { + t.Fatalf("wallet owner snapshot mismatch: %+v", message) + } + if err := mock.ExpectationsWereMet(); err != nil { + t.Fatalf("wallet replay query mismatch: %v", err) + } +} diff --git a/services/room-service/internal/room/service/admin_robot_room.go b/services/room-service/internal/room/service/admin_robot_room.go index e234d87a..5158f688 100644 --- a/services/room-service/internal/room/service/admin_robot_room.go +++ b/services/room-service/internal/room/service/admin_robot_room.go @@ -516,12 +516,26 @@ func (s *Service) ensureRobotRoomParticipants(ctx context.Context, config RobotR activeRobots := onlineRobotIDs(config.RobotUserIDs, online) missing := activeTarget - len(activeRobots) if missing > 0 { + // 首次补回缺席机器人也必须从当前 Room Cell 快照选择真实麦位。候选池下标可能 + // 大于 seat_count；同一轮成功补位后立刻占记，避免后续候选复用尚未写回旧快照的麦位。 + occupiedSeats := make(map[int32]bool, len(snapshot.GetMicSeats())) + for _, seat := range snapshot.GetMicSeats() { + if seat.GetUserId() > 0 { + occupiedSeats[seat.GetSeatNo()] = true + } + } for _, userID := range robotRoomFillCandidates(config, online, missing) { // active 机器人房从持久化配置恢复时，旧进程可能已经把机器人 stale 成 left； // 这里只补齐后台配置的活跃人数，不再把整个候选池一次性塞回房间。 - if err := s.joinRobotRoomUser(roomCtx, config, userID, 0, fmt.Sprintf("runtime-join:%d:%d", userID, time.Now().UTC().UnixNano())); err != nil { + seatNo := robotRestoreSeatNo(config, userID, snapshot.GetMicSeats(), occupiedSeats) + if seatNo <= 0 { + logx.Warn(roomCtx, "robot_room_runtime_join_restore_skipped", slog.String("room_id", config.RoomID), slog.Int64("robot_user_id", userID), slog.String("reason", "no_free_seat")) + continue + } + if err := s.joinRobotRoomUser(roomCtx, config, userID, seatNo, fmt.Sprintf("runtime-join:%d:%d", userID, time.Now().UTC().UnixNano())); err != nil { return err } + occupiedSeats[seatNo] = true online[userID] = true activeRobots = append(activeRobots, userID) } @@ -544,13 +558,10 @@ func (s *Service) ensureRobotRoomParticipants(ctx context.Context, config RobotR if userID <= 0 || seated[userID] { continue } - seatNo := preferredRobotSeatNo(config, userID) - if occupiedSeats[seatNo] { - seatNo = firstFreeRobotSeatNo(snapshot.GetMicSeats(), activeTarget) - if seatNo <= 0 { - logx.Warn(roomCtx, "robot_room_runtime_mic_restore_skipped", slog.String("room_id", config.RoomID), slog.Int64("robot_user_id", userID), slog.String("reason", "no_free_seat")) - continue - } + seatNo := robotRestoreSeatNo(config, userID, snapshot.GetMicSeats(), occupiedSeats) + if seatNo <= 0 { + logx.Warn(roomCtx, "robot_room_runtime_mic_restore_skipped", slog.String("room_id", config.RoomID), slog.Int64("robot_user_id", userID), slog.String("reason", "no_free_seat")) + continue } // 虚拟麦位只服务机器人房展示；若并发恢复导致机器人已上麦，Conflict 可视为达成目标。 if _, err := s.RobotVirtualMicUp(roomCtx, RobotMicUpInput{ @@ -568,6 +579,39 @@ func (s *Service) ensureRobotRoomParticipants(ctx context.Context, config RobotR return nil } +// robotRestoreSeatNo 只接受当前 Room Cell 快照里真实存在且空闲的麦位。 +// robot_user_ids 是候选池而不是麦位表：历史配置可包含数十个轮换机器人，因此其下标 +// 可能远大于当前 10/15/20 个麦位；这种 preferred 必须回退到快照中的首个空麦，不能 +// 把不存在的 seat_no 发给 RobotVirtualMicUp 后让每 10 秒恢复扫描永久失败。 +func robotRestoreSeatNo(config RobotRoomConfig, userID int64, seats []*roomv1.SeatState, occupied map[int32]bool) int32 { + preferred := preferredRobotSeatNo(config, userID) + if preferred > 0 && robotSnapshotSeatFree(seats, preferred) && !occupied[preferred] { + return preferred + } + // snapshot 是本轮恢复开始时的只读副本；occupied 还包含本轮刚成功分配、尚未反映到 + // 该副本的麦位。回退时同时检查二者，避免多个待恢复机器人重复选择同一个空麦。 + var firstFree int32 + for _, seat := range seats { + seatNo := seat.GetSeatNo() + if seatNo <= 0 || seat.GetUserId() > 0 || seat.GetLocked() || occupied[seatNo] { + continue + } + if firstFree == 0 || seatNo < firstFree { + firstFree = seatNo + } + } + return firstFree +} + +func robotSnapshotSeatFree(seats []*roomv1.SeatState, seatNo int32) bool { + for _, seat := range seats { + if seat.GetSeatNo() == seatNo { + return seat.GetUserId() == 0 && !seat.GetLocked() + } + } + return false +} + func (s *Service) runRobotRoomRuntime(ctx context.Context, config RobotRoomConfig) { activity := newRobotRoomActivity(config.RobotUserIDs) if snapshot, err := s.currentSnapshot(appcode.WithContext(ctx, config.AppCode), config.RoomID); err == nil { diff --git a/services/room-service/internal/room/service/admin_robot_room_runtime_restore_test.go b/services/room-service/internal/room/service/admin_robot_room_runtime_restore_test.go index 98de44a2..3e1c910b 100644 --- a/services/room-service/internal/room/service/admin_robot_room_runtime_restore_test.go +++ b/services/room-service/internal/room/service/admin_robot_room_runtime_restore_test.go @@ -89,6 +89,101 @@ func TestRobotRoomRuntimeStartRestoresLeftRobots(t *testing.T) { } } +func TestRobotRoomRuntimeStartRestoresHighIndexOwnerToRealSeat(t *testing.T) { + ctx := appcode.WithContext(context.Background(), appcode.Default) + repository := mysqltest.NewRepository(t) + svc := roomservice.New(roomservice.Config{ + NodeID: "node-robot-runtime-high-index-restore-test", + LeaseTTL: 10 * time.Second, + RankLimit: 20, + SnapshotEveryN: 1, + }, router.NewMemoryDirectory(), repository, followTestWallet{}, integration.NewNoopRoomEventPublisher(), integration.NewNoopOutboxPublisher()) + + // 候选池有 12 个机器人而房间只有 10 个麦位；owner 排序后位于第 12 位。 + // bootstrap 会把 active owner 放到真实 seat 1，运行时恢复不能再按候选池下标猜 seat 12。 + candidates := make([]int64, 12) + for index := range candidates { + candidates[index] = int64(2001 + index) + } + ownerID := candidates[len(candidates)-1] + created, err := svc.AdminCreateRobotRoom(ctx, &roomv1.AdminCreateRobotRoomRequest{ + Meta: roomservice.NewRequestMeta("", 9001), + OwnerRobotUserId: ownerID, + CandidateRobotUserIds: candidates, + MinRobotCount: 2, + MaxRobotCount: 2, + SeatCount: 10, + RoomName: "Robot Runtime High Index Restore", + RoomAvatar: testRoomCoverURL, + VisibleRegionId: 686, + GiftRule: &roomv1.AdminRobotRoomGiftRule{ + GiftIds: []string{"normal_gift"}, + LuckyGiftIds: []string{"lucky_gift"}, + NormalGiftIntervalMs: 10000, + LuckyComboMin: 1, + LuckyComboMax: 1, + LuckyPauseMinMs: 5000, + LuckyPauseMaxMs: 5000, + }, + AdminId: 9001, + }) + if err != nil { + t.Fatalf("create high-index-owner robot room failed: %v", err) + } + roomID := created.GetRoom().GetRoomId() + robotIDs := created.GetRoom().GetRobotUserIds() + if len(robotIDs) != len(candidates) || robotIDs[len(robotIDs)-1] != ownerID { + t.Fatalf("test requires owner at preferred seat 12, got owner=%d robots=%v", ownerID, robotIDs) + } + + initial, err := svc.GetRoomSnapshot(ctx, &roomv1.GetRoomSnapshotRequest{ + Meta: &roomv1.RequestMeta{AppCode: appcode.Default, RoomId: roomID}, + RoomId: roomID, + ViewerUserId: ownerID, + }) + if err != nil { + t.Fatalf("initial high-index-owner snapshot failed: %v", err) + } + if _, err := svc.AdminSetRobotRoomStatus(ctx, &roomv1.AdminSetRobotRoomStatusRequest{ + Meta: roomservice.NewRequestMeta(roomID, 9001), + Status: "stopped", + AdminId: 9001, + }); err != nil { + t.Fatalf("stop high-index-owner robot room failed: %v", err) + } + for _, user := range initial.GetRoom().GetOnlineUsers() { + if _, err := svc.LeaveRoom(ctx, &roomv1.LeaveRoomRequest{Meta: roomservice.NewRequestMeta(roomID, user.GetUserId())}); err != nil { + t.Fatalf("leave initial robot %d failed: %v", user.GetUserId(), err) + } + } + + if _, err := svc.AdminSetRobotRoomStatus(ctx, &roomv1.AdminSetRobotRoomStatusRequest{ + Meta: roomservice.NewRequestMeta(roomID, 9001), + Status: "active", + AdminId: 9001, + }); err != nil { + t.Fatalf("restart high-index-owner robot room failed: %v", err) + } + restored, err := svc.GetRoomSnapshot(ctx, &roomv1.GetRoomSnapshotRequest{ + Meta: &roomv1.RequestMeta{AppCode: appcode.Default, RoomId: roomID}, + RoomId: roomID, + ViewerUserId: ownerID, + }) + if err != nil { + t.Fatalf("snapshot after high-index-owner restore failed: %v", err) + } + ownerSeat := int32(0) + for _, seat := range restored.GetRoom().GetMicSeats() { + if seat.GetUserId() == ownerID { + ownerSeat = seat.GetSeatNo() + break + } + } + if ownerSeat <= 0 || ownerSeat > int32(len(restored.GetRoom().GetMicSeats())) { + t.Fatalf("high-index owner must be restored to a real snapshot seat, got seat=%d", ownerSeat) + } +} + func snapshotContainsUser(snapshot *roomv1.RoomSnapshot, userID int64) bool { for _, user := range snapshot.GetOnlineUsers() { if user.GetUserId() == userID { diff --git a/services/room-service/internal/room/service/admin_robot_room_runtime_test.go b/services/room-service/internal/room/service/admin_robot_room_runtime_test.go index 9c1a710e..8588bc90 100644 --- a/services/room-service/internal/room/service/admin_robot_room_runtime_test.go +++ b/services/room-service/internal/room/service/admin_robot_room_runtime_test.go @@ -123,6 +123,51 @@ func TestNormalizeRobotRoomGiftRuleAppliesBehaviorDefaults(t *testing.T) { } } +func TestRobotRestoreSeatNoFallsBackWhenPreferredExceedsSnapshot(t *testing.T) { + robots := make([]int64, 12) + for index := range robots { + robots[index] = int64(1001 + index) + } + seats := make([]*roomv1.SeatState, 10) + for index := range seats { + seats[index] = &roomv1.SeatState{SeatNo: int32(index + 1)} + } + seats[0].UserId = robots[0] + + // 候选池第 12 个机器人历史 preferred=12，但当前房间只有 10 个麦位； + // 恢复必须选择真实存在的首个空麦 2，而不是把 seat=12 交给 Room Cell。 + got := robotRestoreSeatNo( + RobotRoomConfig{RobotUserIDs: robots, SeatCount: 10}, + robots[11], + seats, + map[int32]bool{1: true}, + ) + if got != 2 { + t.Fatalf("out-of-range preferred seat must fall back to first free snapshot seat: got %d want 2", got) + } + + got = robotRestoreSeatNo( + RobotRoomConfig{RobotUserIDs: robots, SeatCount: 10}, + robots[11], + seats, + map[int32]bool{1: true, 2: true}, + ) + if got != 3 { + t.Fatalf("same-pass occupied seat must not be selected twice: got %d want 3", got) + } + + seats[2].Locked = true + got = robotRestoreSeatNo( + RobotRoomConfig{RobotUserIDs: robots, SeatCount: 10}, + robots[11], + seats, + map[int32]bool{1: true, 2: true}, + ) + if got != 4 { + t.Fatalf("locked snapshot seat is not available for restore: got %d want 4", got) + } +} + func TestRobotRoomActivityPrefersOtherReplacement(t *testing.T) { activity := newRobotRoomActivity([]int64{1001, 1002, 1003}) activity.markActive(1001) diff --git a/services/statistics-service/internal/app/query_http.go b/services/statistics-service/internal/app/query_http.go index 032c105c..79d36a84 100644 --- a/services/statistics-service/internal/app/query_http.go +++ b/services/statistics-service/internal/app/query_http.go @@ -11,6 +11,7 @@ import ( "time" "hyapp/pkg/appcode" + "hyapp/pkg/apptracking" mysqlstorage "hyapp/services/statistics-service/internal/storage/mysql" ) @@ -185,13 +186,13 @@ func (s *queryHTTPServer) socialRequirements(w http.ResponseWriter, r *http.Requ endMS = startMS + 24*time.Hour.Milliseconds() } requirements, err := s.repo.QuerySocialRequirements(r.Context(), mysqlstorage.SocialRequirementsQuery{ - AppCode: query.Get("app_code"), - StatTZ: query.Get("stat_tz"), - StartMS: startMS, - EndMS: endMS, - CountryID: parseInt64(query.Get("country_id")), - RegionID: parseInt64(query.Get("region_id")), - RegionIDs: parseInt64CSV(query.Get("region_ids")), + AppCode: query.Get("app_code"), + StatTZ: query.Get("stat_tz"), + StartMS: startMS, + EndMS: endMS, + CountryID: parseInt64(query.Get("country_id")), + RegionID: parseInt64(query.Get("region_id")), + RegionIDs: parseInt64CSV(query.Get("region_ids")), Section: query.Get("section"), UserRole: query.Get("user_role"), PayerType: query.Get("payer_type"), @@ -326,7 +327,7 @@ func (s *queryHTTPServer) appEvents(w http.ResponseWriter, r *http.Request) { writeJSON(w, http.StatusBadRequest, map[string]any{"error": "invalid json"}) return } - if len(body.Events) == 0 || len(body.Events) > 50 { + if !apptracking.ValidBatchSize(len(body.Events)) { writeJSON(w, http.StatusBadRequest, map[string]any{"error": "events batch is invalid"}) return } diff --git a/services/statistics-service/internal/storage/mysql/query_test.go b/services/statistics-service/internal/storage/mysql/query_test.go index 0a4b5ffc..8f9d6c69 100644 --- a/services/statistics-service/internal/storage/mysql/query_test.go +++ b/services/statistics-service/internal/storage/mysql/query_test.go @@ -13,6 +13,7 @@ import ( "github.com/DATA-DOG/go-sqlmock" "hyapp/internal/testutil/mysqlschema" "hyapp/pkg/appcode" + "hyapp/pkg/apptracking" ) func TestGiftDebitConsumptionOwnersDoNotOverlap(t *testing.T) { @@ -627,7 +628,7 @@ func TestConsumeAppTrackingEventsRejectsInvalidProperties(t *testing.T) { properties json.RawMessage }{ {name: "invalid json", properties: json.RawMessage(`{"bad"`)}, - {name: "too large json", properties: json.RawMessage(`"` + strings.Repeat("a", appTrackingMaxPropertiesBytes+1) + `"`)}, + {name: "too large json", properties: json.RawMessage(`"` + strings.Repeat("a", apptracking.MaxPropertiesBytes+1) + `"`)}, } { t.Run(test.name, func(t *testing.T) { _, err := repository.ConsumeAppTrackingEvents(appcode.WithContext(ctx, "lalu"), []AppTrackingEvent{ @@ -640,6 +641,29 @@ func TestConsumeAppTrackingEventsRejectsInvalidProperties(t *testing.T) { } } +func TestNormalizeAppTrackingEventRejects185ByteErrorCodeWithoutTruncation(t *testing.T) { + base := AppTrackingEvent{ + AppCode: "lalu", + EventID: "evt-error-code-limit", + EventName: "pay_failed", + DeviceID: "dev-1", + OccurredAtMS: time.Date(2026, 7, 13, 0, 0, 0, 0, time.UTC).UnixMilli(), + } + base.ErrorCode = strings.Repeat("e", apptracking.MaxErrorCodeBytes) + normalized, _, err := normalizeAppTrackingEvent(context.Background(), base, base.OccurredAtMS) + if err != nil { + t.Fatalf("error_code at shared byte limit must remain valid: %v", err) + } + if normalized.ErrorCode != base.ErrorCode { + t.Fatalf("valid error_code must be stored intact: got=%q want=%q", normalized.ErrorCode, base.ErrorCode) + } + + base.ErrorCode = strings.Repeat("e", 185) + if _, _, err := normalizeAppTrackingEvent(context.Background(), base, base.OccurredAtMS); err == nil { + t.Fatal("185-byte error_code must be rejected instead of truncated or deferred to MySQL") + } +} + func TestConsumeUserRegisteredCountsOnlyNewRegistration(t *testing.T) { ctx := context.Background() _, file, _, _ := runtime.Caller(0) diff --git a/services/statistics-service/internal/storage/mysql/repository.go b/services/statistics-service/internal/storage/mysql/repository.go index ffeaee48..560ad93c 100644 --- a/services/statistics-service/internal/storage/mysql/repository.go +++ b/services/statistics-service/internal/storage/mysql/repository.go @@ -1,7 +1,6 @@ package mysql import ( - "bytes" "context" "database/sql" "encoding/json" @@ -11,6 +10,7 @@ import ( mysqlerr "github.com/go-sql-driver/mysql" "hyapp/pkg/appcode" + "hyapp/pkg/apptracking" "hyapp/pkg/xerr" ) @@ -23,11 +23,6 @@ const ( SourceFinance = "finance" ) -const ( - appTrackingMaxBatchSize = 50 - appTrackingMaxPropertiesBytes = 4096 -) - // Repository owns the statistics read model. Online queries must use aggregate // tables; the optional activity connection only reads lucky-gift day snapshots, // never the high-cardinality draw fact table. @@ -1670,7 +1665,7 @@ func (r *Repository) ConsumeAppTrackingEvents(ctx context.Context, events []AppT } nowMS := time.Now().UTC().UnixMilli() result := AppTrackingReportResult{Accepted: true, Received: len(events), ServerTimeMS: nowMS} - if len(events) == 0 || len(events) > appTrackingMaxBatchSize { + if !apptracking.ValidBatchSize(len(events)) { return AppTrackingReportResult{}, xerr.New(xerr.InvalidArgument, "app tracking events batch is invalid") } tx, err := r.db.BeginTx(ctx, nil) @@ -2329,62 +2324,65 @@ func normalizeAppTrackingEvent(ctx context.Context, event AppTrackingEvent, rece if strings.TrimSpace(event.AppCode) == "" { event.AppCode = appcode.FromContext(ctx) } - event.AppCode = appcode.Normalize(event.AppCode) - event.EventID, err = normalizeAppTrackingText(event.EventID, 160) + event.AppCode, err = normalizeAppTrackingText(appcode.Normalize(event.AppCode), apptracking.MaxAppCodeBytes) if err != nil { return AppTrackingEvent{}, "", err } - event.EventName, err = normalizeAppTrackingText(event.EventName, 96) + event.EventID, err = normalizeAppTrackingText(event.EventID, apptracking.MaxEventIDBytes) + if err != nil { + return AppTrackingEvent{}, "", err + } + event.EventName, err = normalizeAppTrackingText(event.EventName, apptracking.MaxEventNameBytes) if err != nil { return AppTrackingEvent{}, "", err } if event.EventID == "" || event.EventName == "" { return AppTrackingEvent{}, "", xerr.New(xerr.InvalidArgument, "app tracking event_id and event_name are required") } - event.EventType, err = normalizeAppTrackingText(event.EventType, 64) + event.EventType, err = normalizeAppTrackingText(event.EventType, apptracking.MaxEventTypeBytes) if err != nil { return AppTrackingEvent{}, "", err } - event.Screen, err = normalizeAppTrackingText(event.Screen, 128) + event.Screen, err = normalizeAppTrackingText(event.Screen, apptracking.MaxScreenBytes) if err != nil { return AppTrackingEvent{}, "", err } - event.TargetType, err = normalizeAppTrackingText(event.TargetType, 64) + event.TargetType, err = normalizeAppTrackingText(event.TargetType, apptracking.MaxTargetTypeBytes) if err != nil { return AppTrackingEvent{}, "", err } - event.TargetID, err = normalizeAppTrackingText(event.TargetID, 128) + event.TargetID, err = normalizeAppTrackingText(event.TargetID, apptracking.MaxTargetIDBytes) if err != nil { return AppTrackingEvent{}, "", err } - event.DeviceID, err = normalizeAppTrackingText(event.DeviceID, 128) + event.DeviceID, err = normalizeAppTrackingText(event.DeviceID, apptracking.MaxDeviceIDBytes) if err != nil { return AppTrackingEvent{}, "", err } if event.UserID <= 0 && event.DeviceID == "" { return AppTrackingEvent{}, "", xerr.New(xerr.InvalidArgument, "anonymous app tracking event requires device_id") } - event.SessionID, err = normalizeAppTrackingText(event.SessionID, 160) + event.SessionID, err = normalizeAppTrackingText(event.SessionID, apptracking.MaxSessionIDBytes) if err != nil { return AppTrackingEvent{}, "", err } - event.Platform, err = normalizeAppTrackingText(strings.ToLower(event.Platform), 32) + event.Platform, err = normalizeAppTrackingText(strings.ToLower(event.Platform), apptracking.MaxPlatformBytes) if err != nil { return AppTrackingEvent{}, "", err } - event.AppVersion, err = normalizeAppTrackingText(event.AppVersion, 64) + event.AppVersion, err = normalizeAppTrackingText(event.AppVersion, apptracking.MaxAppVersionBytes) if err != nil { return AppTrackingEvent{}, "", err } - event.Language, err = normalizeAppTrackingText(event.Language, 32) + event.Language, err = normalizeAppTrackingText(event.Language, apptracking.MaxLanguageBytes) if err != nil { return AppTrackingEvent{}, "", err } - event.Timezone, err = normalizeAppTrackingText(event.Timezone, 64) + event.Timezone, err = normalizeAppTrackingText(event.Timezone, apptracking.MaxTimezoneBytes) if err != nil { return AppTrackingEvent{}, "", err } - event.ErrorCode, err = normalizeAppTrackingText(event.ErrorCode, 64) + event.ErrorCode, err = normalizeAppTrackingText(event.ErrorCode, apptracking.MaxErrorCodeBytes) if err != nil { return AppTrackingEvent{}, "", err } @@ -2402,21 +2400,21 @@ func normalizeAppTrackingEvent(ctx context.Context, event AppTrackingEvent, rece } func normalizeAppTrackingText(value string, maxBytes int) (string, error) { - value = strings.TrimSpace(value) - if maxBytes > 0 && len(value) > maxBytes { + value, ok := apptracking.NormalizeText(value, maxBytes) + if !ok { return "", xerr.New(xerr.InvalidArgument, "app tracking text field is too long") } return value, nil } func normalizeAppTrackingProperties(raw json.RawMessage) (string, error) { - trimmed := bytes.TrimSpace(raw) - if len(trimmed) == 0 || bytes.Equal(trimmed, []byte("null")) { - return "{}", nil - } - if len(trimmed) > appTrackingMaxPropertiesBytes || !json.Valid(trimmed) { + trimmed, ok := apptracking.NormalizeProperties(raw) + if !ok { return "", xerr.New(xerr.InvalidArgument, "app tracking properties_json is invalid") } + if len(trimmed) == 0 { + return "{}", nil + } return string(trimmed), nil } diff --git a/services/user-service/configs/config.docker.yaml b/services/user-service/configs/config.docker.yaml index 1d8c2a82..f30e1a93 100644 --- a/services/user-service/configs/config.docker.yaml +++ b/services/user-service/configs/config.docker.yaml @@ -50,6 +50,7 @@ cp_intimacy_leaderboard: key_prefix: "user:cp_intimacy_leaderboard" outbox_worker: enabled: true + app_codes: [] poll_interval: "1s" batch_size: 100 publish_timeout: "3s" diff --git a/services/user-service/configs/config.tencent.example.yaml b/services/user-service/configs/config.tencent.example.yaml index 53d052da..d6bf3dc2 100644 --- a/services/user-service/configs/config.tencent.example.yaml +++ b/services/user-service/configs/config.tencent.example.yaml @@ -50,6 +50,8 @@ cp_intimacy_leaderboard: key_prefix: "user:cp_intimacy_leaderboard" outbox_worker: enabled: true + # 空值自动发现全部 App；历史积压回放时先设 ["lalu", "fami"]，核对后再加入 "huwaa"。 + app_codes: [] poll_interval: "1s" batch_size: 100 publish_timeout: "3s" diff --git a/services/user-service/configs/config.yaml b/services/user-service/configs/config.yaml index 7db2e170..c26d979c 100644 --- a/services/user-service/configs/config.yaml +++ b/services/user-service/configs/config.yaml @@ -51,6 +51,7 @@ cp_intimacy_leaderboard: key_prefix: "user:cp_intimacy_leaderboard" outbox_worker: enabled: true + app_codes: [] poll_interval: "1s" batch_size: 100 publish_timeout: "3s" diff --git a/services/user-service/internal/app/app.go b/services/user-service/internal/app/app.go index 716ea596..1a2680dc 100644 --- a/services/user-service/internal/app/app.go +++ b/services/user-service/internal/app/app.go @@ -20,6 +20,7 @@ import ( "hyapp/pkg/healthhttp" "hyapp/pkg/idgen" "hyapp/pkg/logx" + "hyapp/pkg/outboxpartition" "hyapp/pkg/rocketmqx" "hyapp/pkg/roommq" serviceapp "hyapp/pkg/servicekit/app" @@ -66,6 +67,8 @@ type App struct { mqConsumers []*rocketmqx.Consumer // userOutboxProducer 把 user_outbox 事实发布到 RocketMQ，供统计等读模型消费。 userOutboxProducer *rocketmqx.Producer + // userOutboxPartitions 缓存真实 App 列表并提供跨 worker 的公平轮转游标。 + userOutboxPartitions *outboxpartition.Partitions // roomConn 是低频平台治理调用 room-service 的 gRPC 连接。 roomConn *grpc.ClientConn // activityConn 是注册完成后触发注册奖励的 gRPC 连接。 @@ -435,6 +438,7 @@ func New(cfg config.Config) (*App, error) { mysqlRepo: mysqlRepo, mqConsumers: mqConsumers, userOutboxProducer: userOutboxProducer, + userOutboxPartitions: outboxpartition.New(cfg.OutboxWorker.AppCodes, outboxpartition.DefaultDiscoveryRefreshInterval), roomConn: roomConn, activityConn: activityConn, walletConn: walletConn, @@ -539,7 +543,18 @@ func (a *App) runUserOutboxWorker() { } func (a *App) processUserOutboxBatch(workerID string) (int, error) { - records, err := a.mysqlRepo.ClaimPendingUserOutbox(a.workers.Context(), workerID, a.cfg.OutboxWorker.BatchSize) + ctx := a.workers.Context() + appCodes, discoveryErr := a.userOutboxPartitions.Resolve(ctx, a.mysqlRepo.ListUserOutboxAppCodes) + if discoveryErr != nil { + if len(appCodes) == 0 { + return 0, discoveryErr + } + // 发现刷新失败时沿用旧列表；缓存会在五分钟后再试，不阻断既有 App 的事实发布。 + logx.Warn(ctx, "user_outbox_app_discovery_stale", slog.String("worker_id", workerID), slog.String("error", discoveryErr.Error())) + } + records, err := outboxpartition.ClaimFair(ctx, a.userOutboxPartitions.Order(appCodes), a.cfg.OutboxWorker.BatchSize, func(claimCtx context.Context, appCode string, limit int) ([]mysqlstorage.UserOutboxRecord, error) { + return a.mysqlRepo.ClaimPendingUserOutbox(appcode.WithContext(claimCtx, appCode), workerID, limit) + }) if err != nil { return 0, err } diff --git a/services/user-service/internal/config/config.go b/services/user-service/internal/config/config.go index 00397ca2..697dca98 100644 --- a/services/user-service/internal/config/config.go +++ b/services/user-service/internal/config/config.go @@ -10,6 +10,7 @@ import ( "hyapp/pkg/appcode" "hyapp/pkg/configx" "hyapp/pkg/logx" + "hyapp/pkg/outboxpartition" "hyapp/pkg/tencentim" ) @@ -215,7 +216,9 @@ type CPIntimacyLeaderboardConfig struct { // OutboxWorkerConfig 保存 user_outbox MQ 发布策略。 type OutboxWorkerConfig struct { - Enabled bool `yaml:"enabled"` + Enabled bool `yaml:"enabled"` + // AppCodes 是可投递租户 allowlist；空值使用低频数据库发现，非空支持按 App 灰度回放。 + AppCodes []string `yaml:"app_codes"` PollInterval time.Duration `yaml:"poll_interval"` BatchSize int `yaml:"batch_size"` PublishTimeout time.Duration `yaml:"publish_timeout"` @@ -492,6 +495,7 @@ func Load(path string) (Config, error) { if cfg.OutboxWorker.PublishTimeout <= 0 { cfg.OutboxWorker.PublishTimeout = 3 * time.Second } + cfg.OutboxWorker.AppCodes = outboxpartition.Normalize(cfg.OutboxWorker.AppCodes, false) if cfg.OutboxWorker.Enabled && !cfg.RocketMQ.UserOutbox.Enabled { return Config{}, errors.New("outbox_worker requires rocketmq.user_outbox.enabled") } diff --git a/services/user-service/internal/config/outbox_worker_test.go b/services/user-service/internal/config/outbox_worker_test.go new file mode 100644 index 00000000..7196709a --- /dev/null +++ b/services/user-service/internal/config/outbox_worker_test.go @@ -0,0 +1,32 @@ +package config + +import ( + "os" + "path/filepath" + "strings" + "testing" +) + +func TestOutboxWorkerAppCodesNormalizeForStagedReplay(t *testing.T) { + path := filepath.Join(t.TempDir(), "config.yaml") + if err := os.WriteFile(path, []byte("outbox_worker:\n app_codes: [' LALU ', fami, HUWAA, fami]\n"), 0o600); err != nil { + t.Fatalf("write config: %v", err) + } + cfg, err := Load(path) + if err != nil { + t.Fatalf("load staged outbox config: %v", err) + } + if got := strings.Join(cfg.OutboxWorker.AppCodes, ","); got != "lalu,fami,huwaa" { + t.Fatalf("normalized outbox app_codes = %q", got) + } +} + +func TestLocalOutboxWorkerEmptyAppCodesUsesDiscovery(t *testing.T) { + cfg, err := Load("../../configs/config.yaml") + if err != nil { + t.Fatalf("load local config: %v", err) + } + if len(cfg.OutboxWorker.AppCodes) != 0 { + t.Fatalf("local empty app_codes must retain automatic discovery: %v", cfg.OutboxWorker.AppCodes) + } +} diff --git a/services/user-service/internal/storage/mysql/user_outbox.go b/services/user-service/internal/storage/mysql/user_outbox.go index 047eb457..b77e6a9a 100644 --- a/services/user-service/internal/storage/mysql/user_outbox.go +++ b/services/user-service/internal/storage/mysql/user_outbox.go @@ -25,6 +25,29 @@ type UserOutboxRecord struct { LockUntilMS int64 } +// ListUserOutboxAppCodes enumerates real tenant keys from the owner table. +// PRIMARY begins with app_code, and the worker caches this discovery for five +// minutes; normal claims continue to use status/lock indexes within one App. +func (r *Repository) ListUserOutboxAppCodes(ctx context.Context) ([]string, error) { + if r == nil || r.db == nil { + return nil, xerr.New(xerr.Unavailable, "mysql repository is not configured") + } + rows, err := r.db.QueryContext(ctx, `SELECT DISTINCT app_code FROM user_outbox FORCE INDEX (PRIMARY) ORDER BY app_code`) + if err != nil { + return nil, err + } + defer rows.Close() + appCodes := make([]string, 0, 4) + for rows.Next() { + var appCode string + if err := rows.Scan(&appCode); err != nil { + return nil, err + } + appCodes = append(appCodes, appCode) + } + return appCodes, rows.Err() +} + // ClaimPendingUserOutbox locks pending user facts so one worker can publish them. func (r *Repository) ClaimPendingUserOutbox(ctx context.Context, workerID string, batchSize int) ([]UserOutboxRecord, error) { if r == nil || r.db == nil { diff --git a/services/user-service/internal/storage/mysql/user_outbox_app_codes_test.go b/services/user-service/internal/storage/mysql/user_outbox_app_codes_test.go new file mode 100644 index 00000000..02dc9a65 --- /dev/null +++ b/services/user-service/internal/storage/mysql/user_outbox_app_codes_test.go @@ -0,0 +1,30 @@ +package mysql + +import ( + "context" + "regexp" + "testing" + + "github.com/DATA-DOG/go-sqlmock" +) + +func TestListUserOutboxAppCodesForcesPrimaryPrefixIndex(t *testing.T) { + db, mock, err := sqlmock.New() + if err != nil { + t.Fatalf("create sqlmock: %v", err) + } + defer db.Close() + mock.ExpectQuery(regexp.QuoteMeta(`SELECT DISTINCT app_code FROM user_outbox FORCE INDEX (PRIMARY) ORDER BY app_code`)). + WillReturnRows(sqlmock.NewRows([]string{"app_code"}).AddRow("fami").AddRow("huwaa").AddRow("lalu")) + + apps, err := (&Repository{db: db}).ListUserOutboxAppCodes(context.Background()) + if err != nil { + t.Fatalf("list user outbox apps: %v", err) + } + if len(apps) != 3 || apps[0] != "fami" || apps[1] != "huwaa" || apps[2] != "lalu" { + t.Fatalf("user outbox apps mismatch: %v", apps) + } + if err := mock.ExpectationsWereMet(); err != nil { + t.Fatalf("user app discovery query mismatch: %v", err) + } +} diff --git a/services/wallet-service/configs/config.docker.yaml b/services/wallet-service/configs/config.docker.yaml index 88b8d617..5e2abe5d 100644 --- a/services/wallet-service/configs/config.docker.yaml +++ b/services/wallet-service/configs/config.docker.yaml @@ -150,6 +150,7 @@ external_recharge: v5pay_return_url: "https://h5.global-interaction.com/recharge/index.html?env=test" outbox_worker: enabled: true + app_codes: [] poll_interval: "1s" batch_size: 200 concurrency: 4 @@ -159,6 +160,7 @@ outbox_worker: max_backoff: "5m" realtime_outbox_worker: enabled: true + app_codes: [] poll_interval: "500ms" batch_size: 50 concurrency: 2 diff --git a/services/wallet-service/configs/config.tencent.example.yaml b/services/wallet-service/configs/config.tencent.example.yaml index 8efa94f0..2e66796a 100644 --- a/services/wallet-service/configs/config.tencent.example.yaml +++ b/services/wallet-service/configs/config.tencent.example.yaml @@ -150,6 +150,8 @@ external_recharge: v5pay_return_url: "https://h5.global-interaction.com/recharge/index.html" outbox_worker: enabled: true + # 空值自动发现全部 App；历史积压回放时先设 ["lalu", "fami"]，核对后再加入 "huwaa"。 + app_codes: [] poll_interval: "1s" batch_size: 200 concurrency: 8 @@ -159,6 +161,8 @@ outbox_worker: max_backoff: "5m" realtime_outbox_worker: enabled: true + # 实时 lane 必须与普通 lane 使用同一阶段 allowlist，避免同表出现跨 App 抢占差异。 + app_codes: [] poll_interval: "500ms" batch_size: 50 concurrency: 4 diff --git a/services/wallet-service/configs/config.yaml b/services/wallet-service/configs/config.yaml index 0c5eeb43..0442b243 100644 --- a/services/wallet-service/configs/config.yaml +++ b/services/wallet-service/configs/config.yaml @@ -153,6 +153,7 @@ external_recharge_reconcile_worker: batch_size: 50 outbox_worker: enabled: true + app_codes: [] poll_interval: "1s" batch_size: 100 concurrency: 1 @@ -162,6 +163,7 @@ outbox_worker: max_backoff: "5m" realtime_outbox_worker: enabled: true + app_codes: [] poll_interval: "500ms" batch_size: 50 concurrency: 2 diff --git a/services/wallet-service/internal/app/app.go b/services/wallet-service/internal/app/app.go index 1b4bda64..353f4169 100644 --- a/services/wallet-service/internal/app/app.go +++ b/services/wallet-service/internal/app/app.go @@ -13,6 +13,7 @@ import ( "hyapp/pkg/grpchealth" "hyapp/pkg/healthhttp" "hyapp/pkg/logx" + "hyapp/pkg/outboxpartition" "hyapp/pkg/rocketmqx" servicegrpc "hyapp/pkg/servicekit/grpcserver" servicehealth "hyapp/pkg/servicekit/health" @@ -31,18 +32,21 @@ import ( // App 装配 wallet-service gRPC 入口。 type App struct { - server *grpc.Server - listener net.Listener - health *grpchealth.ServingChecker - healthHTTP *healthhttp.Server - mysqlRepo *mysqlstorage.Repository - walletSvc *walletservice.Service - activityConn *grpc.ClientConn - outboxProducer *rocketmqx.Producer - realtimeOutboxProducer *rocketmqx.Producer - projectionConsumer *rocketmqx.Consumer - outboxWorkerCfg config.OutboxWorkerConfig - realtimeOutboxWorkerCfg config.OutboxWorkerConfig + server *grpc.Server + listener net.Listener + health *grpchealth.ServingChecker + healthHTTP *healthhttp.Server + mysqlRepo *mysqlstorage.Repository + walletSvc *walletservice.Service + activityConn *grpc.ClientConn + outboxProducer *rocketmqx.Producer + realtimeOutboxProducer *rocketmqx.Producer + projectionConsumer *rocketmqx.Consumer + outboxWorkerCfg config.OutboxWorkerConfig + realtimeOutboxWorkerCfg config.OutboxWorkerConfig + // 普通与实时 lane 共享同一张表但可独立配置事件类型；各自维护 App 轮转游标，避免一个 lane 的吞吐改变另一个 lane 的公平性。 + outboxPartitions *outboxpartition.Partitions + realtimeOutboxPartitions *outboxpartition.Partitions projectionWorkerCfg config.ProjectionWorkerConfig googlePaidSyncWorkerCfg config.GooglePaidSyncWorkerConfig externalRechargeReconcileWorkerCfg config.ExternalRechargeReconcileWorkerConfig @@ -204,6 +208,8 @@ func New(cfg config.Config) (*App, error) { projectionConsumer: projectionConsumer, outboxWorkerCfg: cfg.OutboxWorker, realtimeOutboxWorkerCfg: cfg.RealtimeOutboxWorker, + outboxPartitions: outboxpartition.New(cfg.OutboxWorker.AppCodes, outboxpartition.DefaultDiscoveryRefreshInterval), + realtimeOutboxPartitions: outboxpartition.New(cfg.RealtimeOutboxWorker.AppCodes, outboxpartition.DefaultDiscoveryRefreshInterval), projectionWorkerCfg: cfg.ProjectionWorker, googlePaidSyncWorkerCfg: googlePaidSyncWorkerCfg, externalRechargeReconcileWorkerCfg: cfg.ExternalRechargeReconcileWorker, diff --git a/services/wallet-service/internal/app/background_workers.go b/services/wallet-service/internal/app/background_workers.go index c304d0d9..5ac97f22 100644 --- a/services/wallet-service/internal/app/background_workers.go +++ b/services/wallet-service/internal/app/background_workers.go @@ -14,10 +14,10 @@ func (a *App) runBackgroundWorkers() { // 普通账务 worker 在实时通道可用时跳过红包 UI 事件，避免两个 worker 抢同一行。 excludedRealtimeTypes = a.realtimeOutboxWorkerCfg.EventTypes } - a.startWalletOutboxWorkers(ctx, "wallet-outbox", a.outboxWorkerCfg, a.outboxProducer, a.walletOutboxTopic, nil, excludedRealtimeTypes) + a.startWalletOutboxWorkers(ctx, "wallet-outbox", a.outboxWorkerCfg, a.outboxPartitions, a.outboxProducer, a.walletOutboxTopic, nil, excludedRealtimeTypes) } if a.realtimeOutboxWorkerCfg.Enabled && a.realtimeOutboxProducer != nil { - a.startWalletOutboxWorkers(ctx, "wallet-realtime-outbox", a.realtimeOutboxWorkerCfg, a.realtimeOutboxProducer, a.realtimeWalletOutboxTopic, a.realtimeOutboxWorkerCfg.EventTypes, nil) + a.startWalletOutboxWorkers(ctx, "wallet-realtime-outbox", a.realtimeOutboxWorkerCfg, a.realtimeOutboxPartitions, a.realtimeOutboxProducer, a.realtimeWalletOutboxTopic, a.realtimeOutboxWorkerCfg.EventTypes, nil) } if a.externalRechargeReconcileWorkerCfg.Enabled { a.startExternalRechargeReconcileWorker(ctx) diff --git a/services/wallet-service/internal/app/outbox_worker.go b/services/wallet-service/internal/app/outbox_worker.go index fe98f8d1..da4893f4 100644 --- a/services/wallet-service/internal/app/outbox_worker.go +++ b/services/wallet-service/internal/app/outbox_worker.go @@ -10,13 +10,14 @@ import ( "hyapp/pkg/appcode" "hyapp/pkg/logx" + "hyapp/pkg/outboxpartition" "hyapp/pkg/rocketmqx" "hyapp/pkg/walletmq" "hyapp/services/wallet-service/internal/config" mysqlstorage "hyapp/services/wallet-service/internal/storage/mysql" ) -func (a *App) startWalletOutboxWorkers(ctx context.Context, workerName string, options config.OutboxWorkerConfig, producer *rocketmqx.Producer, topic string, includeEventTypes []string, excludeEventTypes []string) { +func (a *App) startWalletOutboxWorkers(ctx context.Context, workerName string, options config.OutboxWorkerConfig, partitions *outboxpartition.Partitions, producer *rocketmqx.Producer, topic string, includeEventTypes []string, excludeEventTypes []string) { if options.Concurrency <= 0 { options.Concurrency = 1 } @@ -25,16 +26,16 @@ func (a *App) startWalletOutboxWorkers(ctx context.Context, workerName string, o a.workers.Add(1) go func() { defer a.workers.Done() - a.runWalletOutboxWorker(ctx, workerID, options, producer, topic, includeEventTypes, excludeEventTypes) + a.runWalletOutboxWorker(ctx, workerID, options, partitions, producer, topic, includeEventTypes, excludeEventTypes) }() } } -func (a *App) runWalletOutboxWorker(ctx context.Context, workerID string, options config.OutboxWorkerConfig, producer *rocketmqx.Producer, topic string, includeEventTypes []string, excludeEventTypes []string) { +func (a *App) runWalletOutboxWorker(ctx context.Context, workerID string, options config.OutboxWorkerConfig, partitions *outboxpartition.Partitions, producer *rocketmqx.Producer, topic string, includeEventTypes []string, excludeEventTypes []string) { ticker := time.NewTicker(options.PollInterval) defer ticker.Stop() for { - processed, err := a.processWalletOutboxBatch(ctx, workerID, options, producer, topic, includeEventTypes, excludeEventTypes) + processed, err := a.processWalletOutboxBatch(ctx, workerID, options, partitions, producer, topic, includeEventTypes, excludeEventTypes) if err != nil && !errors.Is(err, context.Canceled) { logx.Error(ctx, "wallet_outbox_publish_failed", err, slog.String("worker_id", workerID)) } @@ -49,18 +50,31 @@ func (a *App) runWalletOutboxWorker(ctx context.Context, workerID string, option } } -func (a *App) processWalletOutboxBatch(ctx context.Context, workerID string, options config.OutboxWorkerConfig, producer *rocketmqx.Producer, topic string, includeEventTypes []string, excludeEventTypes []string) (int, error) { +func (a *App) processWalletOutboxBatch(ctx context.Context, workerID string, options config.OutboxWorkerConfig, partitions *outboxpartition.Partitions, producer *rocketmqx.Producer, topic string, includeEventTypes []string, excludeEventTypes []string) (int, error) { if a.mysqlRepo == nil || producer == nil { return 0, nil } + appCodes, discoveryErr := partitions.Resolve(ctx, a.mysqlRepo.ListWalletOutboxAppCodes) + if discoveryErr != nil { + if len(appCodes) == 0 { + return 0, discoveryErr + } + // 已知 App 继续投递，刷新失败只影响新租户发现；缓存层会延后下一次查询，避免热循环打 MySQL。 + logx.Warn(ctx, "wallet_outbox_app_discovery_stale", slog.String("worker_id", workerID), slog.String("error", discoveryErr.Error())) + } + if len(appCodes) == 0 { + return 0, nil + } processed := 0 for processed < options.BatchSize { // Sync publish is intentionally claimed one record at a time. Claiming an entire // batch with one short lock lets later rows expire while earlier SendSync calls // are still running, so another replica can publish the same tail concurrently. - records, err := a.mysqlRepo.ClaimPendingWalletOutboxFiltered(ctx, workerID, 1, time.Now().UTC().Add(walletOutboxSingleRecordLease(options.PublishTimeout)).UnixMilli(), mysqlstorage.WalletOutboxClaimFilter{ - IncludeEventTypes: includeEventTypes, - ExcludeEventTypes: excludeEventTypes, + records, err := outboxpartition.ClaimFair(ctx, partitions.Order(appCodes), 1, func(claimCtx context.Context, appCode string, limit int) ([]mysqlstorage.WalletOutboxRecord, error) { + return a.mysqlRepo.ClaimPendingWalletOutboxFiltered(appcode.WithContext(claimCtx, appCode), workerID, limit, time.Now().UTC().Add(walletOutboxSingleRecordLease(options.PublishTimeout)).UnixMilli(), mysqlstorage.WalletOutboxClaimFilter{ + IncludeEventTypes: includeEventTypes, + ExcludeEventTypes: excludeEventTypes, + }) }) if err != nil { return processed, err diff --git a/services/wallet-service/internal/config/config.go b/services/wallet-service/internal/config/config.go index 3a327aa2..fdbcf806 100644 --- a/services/wallet-service/internal/config/config.go +++ b/services/wallet-service/internal/config/config.go @@ -9,6 +9,7 @@ import ( "hyapp/pkg/configx" "hyapp/pkg/logx" + "hyapp/pkg/outboxpartition" ) // Config 描述 wallet-service 启动配置。 @@ -102,7 +103,9 @@ type WalletOutboxMQConfig struct { // OutboxWorkerConfig 控制 wallet_outbox pending 事件的 MQ fanout。 type OutboxWorkerConfig struct { - Enabled bool `yaml:"enabled"` + Enabled bool `yaml:"enabled"` + // AppCodes 是可投递租户 allowlist；空值才自动发现，便于积压回放按 App 分阶段放量。 + AppCodes []string `yaml:"app_codes"` PollInterval time.Duration `yaml:"poll_interval"` BatchSize int `yaml:"batch_size"` Concurrency int `yaml:"concurrency"` @@ -820,6 +823,7 @@ func normalizeRocketMQConfig(cfg RocketMQConfig) (RocketMQConfig, error) { } func normalizeOutboxWorkerConfig(cfg OutboxWorkerConfig, defaults OutboxWorkerConfig) OutboxWorkerConfig { + cfg.AppCodes = outboxpartition.Normalize(cfg.AppCodes, false) if cfg.PollInterval <= 0 { cfg.PollInterval = defaults.PollInterval } diff --git a/services/wallet-service/internal/config/config_test.go b/services/wallet-service/internal/config/config_test.go index 26448579..fc00193a 100644 --- a/services/wallet-service/internal/config/config_test.go +++ b/services/wallet-service/internal/config/config_test.go @@ -31,11 +31,21 @@ func TestLoadLocalEnablesOutboxMQ(t *testing.T) { if cfg.OutboxWorker.Concurrency != 1 || cfg.RealtimeOutboxWorker.Concurrency != 2 { t.Fatalf("local outbox worker defaults mismatch: outbox=%+v realtime=%+v", cfg.OutboxWorker, cfg.RealtimeOutboxWorker) } + if len(cfg.OutboxWorker.AppCodes) != 0 || len(cfg.RealtimeOutboxWorker.AppCodes) != 0 { + t.Fatalf("empty app_codes must retain automatic discovery: outbox=%v realtime=%v", cfg.OutboxWorker.AppCodes, cfg.RealtimeOutboxWorker.AppCodes) + } if cfg.BSCUSDT.Enabled || cfg.ExternalRecharge.USDTBEP20Enabled { t.Fatalf("bsc usdt receipt verification must stay disabled by default: bsc=%+v external=%+v", cfg.BSCUSDT, cfg.ExternalRecharge) } } +func TestOutboxWorkerAppCodesNormalizeForStagedReplay(t *testing.T) { + cfg := normalizeOutboxWorkerConfig(OutboxWorkerConfig{AppCodes: []string{" LALU ", "fami", "HUWAA", "fami"}}, Default().OutboxWorker) + if got := strings.Join(cfg.AppCodes, ","); got != "lalu,fami,huwaa" { + t.Fatalf("normalized outbox app_codes = %q", got) + } +} + func TestDefaultWalletEventTagModeIsLegacy(t *testing.T) { if got := Default().RocketMQ.WalletEventTagMode; got != WalletEventTagModeLegacy { t.Fatalf("default wallet event tag mode = %q, want %q", got, WalletEventTagModeLegacy) diff --git a/services/wallet-service/internal/service/wallet/service_test.go b/services/wallet-service/internal/service/wallet/service_test.go index 1000a0c7..d2b3ec7b 100644 --- a/services/wallet-service/internal/service/wallet/service_test.go +++ b/services/wallet-service/internal/service/wallet/service_test.go @@ -2385,6 +2385,45 @@ func TestCreditLuckyGiftRewardIsIdempotent(t *testing.T) { } } +// TestCreditWheelRewardReconciliationReturnsExistingTransaction 验证 Activity 补偿重复提交稳定 draw command 时， +// 钱包先按 command_id + request hash 找回既有交易和原回执，不会再次增加余额或重复生成 outbox。 +func TestCreditWheelRewardReconciliationReturnsExistingTransaction(t *testing.T) { + repository := mysqltest.NewRepository(t) + svc := walletservice.New(repository) + command := ledger.WheelRewardCommand{ + AppCode: "fami", + CommandID: "wheel_reward:wheel_draw_reconcile_1", + TargetUserID: 21102, + Amount: 288, + DrawID: "wheel_draw_reconcile_1", + WheelID: "classic", + SelectedTierID: "classic-coin-288", + VisibleRegionID: 10, + Reason: "wheel_reward", + } + + first, err := svc.CreditWheelReward(context.Background(), command) + if err != nil { + t.Fatalf("CreditWheelReward failed: %v", err) + } + second, err := svc.CreditWheelReward(context.Background(), command) + if err != nil { + t.Fatalf("CreditWheelReward reconciliation retry failed: %v", err) + } + if first.TransactionID == "" || first.TransactionID != second.TransactionID || second.Balance.AvailableAmount != 288 { + t.Fatalf("wheel reward reconciliation receipt mismatch: first=%+v second=%+v", first, second) + } + if got := repository.CountRows("wallet_transactions", "app_code = ? AND command_id = ? AND biz_type = ?", "fami", command.CommandID, "wheel_reward"); got != 1 { + t.Fatalf("wheel reward reconciliation must keep one transaction, got %d", got) + } + if got := repository.CountRows("wallet_outbox", "app_code = ? AND transaction_id = ? AND event_type = ?", "fami", first.TransactionID, "WalletBalanceChanged"); got != 1 { + t.Fatalf("wheel reward reconciliation must keep one balance event, got %d", got) + } + if got := repository.CountRows("wallet_outbox", "app_code = ? AND transaction_id = ? AND event_type = ?", "fami", first.TransactionID, "WalletWheelRewardCredited"); got != 1 { + t.Fatalf("wheel reward reconciliation must keep one reward fact, got %d", got) + } +} + // TestTransferCoinFromSellerMovesDedicatedAssetToPlayerCoin 验证币商专用金币和玩家普通金币不会混账。 func TestTransferCoinFromSellerMovesDedicatedAssetToPlayerCoin(t *testing.T) { repository := mysqltest.NewRepository(t) diff --git a/services/wallet-service/internal/storage/mysql/outbox.go b/services/wallet-service/internal/storage/mysql/outbox.go index c96be517..87d9c79f 100644 --- a/services/wallet-service/internal/storage/mysql/outbox.go +++ b/services/wallet-service/internal/storage/mysql/outbox.go @@ -84,6 +84,29 @@ func (r *Repository) insertWalletOutbox(ctx context.Context, tx *sql.Tx, events return nil } +// ListWalletOutboxAppCodes enumerates the tenant key stored in this owner table. +// PRIMARY starts with app_code, so MySQL can use a loose ordered index scan; +// callers cache the result and never run this discovery query on every poll. +func (r *Repository) ListWalletOutboxAppCodes(ctx context.Context) ([]string, error) { + if r == nil || r.db == nil { + return nil, xerr.New(xerr.Unavailable, "mysql repository is not configured") + } + rows, err := r.db.QueryContext(ctx, `SELECT DISTINCT app_code FROM wallet_outbox FORCE INDEX (PRIMARY) ORDER BY app_code`) + if err != nil { + return nil, err + } + defer rows.Close() + appCodes := make([]string, 0, 4) + for rows.Next() { + var appCode string + if err := rows.Scan(&appCode); err != nil { + return nil, err + } + appCodes = append(appCodes, appCode) + } + return appCodes, rows.Err() +} + // ClaimPendingWalletOutbox atomically claims committed wallet facts for MQ fanout. func (r *Repository) ClaimPendingWalletOutbox(ctx context.Context, workerID string, limit int, lockUntilMS int64) ([]WalletOutboxRecord, error) { return r.ClaimPendingWalletOutboxFiltered(ctx, workerID, limit, lockUntilMS, WalletOutboxClaimFilter{}) diff --git a/services/wallet-service/internal/storage/mysql/outbox_app_codes_test.go b/services/wallet-service/internal/storage/mysql/outbox_app_codes_test.go new file mode 100644 index 00000000..f9796929 --- /dev/null +++ b/services/wallet-service/internal/storage/mysql/outbox_app_codes_test.go @@ -0,0 +1,30 @@ +package mysql + +import ( + "context" + "regexp" + "testing" + + "github.com/DATA-DOG/go-sqlmock" +) + +func TestListWalletOutboxAppCodesForcesPrimaryPrefixIndex(t *testing.T) { + db, mock, err := sqlmock.New() + if err != nil { + t.Fatalf("create sqlmock: %v", err) + } + defer db.Close() + mock.ExpectQuery(regexp.QuoteMeta(`SELECT DISTINCT app_code FROM wallet_outbox FORCE INDEX (PRIMARY) ORDER BY app_code`)). + WillReturnRows(sqlmock.NewRows([]string{"app_code"}).AddRow("fami").AddRow("huwaa").AddRow("lalu")) + + apps, err := (&Repository{db: db}).ListWalletOutboxAppCodes(context.Background()) + if err != nil { + t.Fatalf("list wallet outbox apps: %v", err) + } + if len(apps) != 3 || apps[0] != "fami" || apps[1] != "huwaa" || apps[2] != "lalu" { + t.Fatalf("wallet outbox apps mismatch: %v", apps) + } + if err := mock.ExpectationsWereMet(); err != nil { + t.Fatalf("wallet app discovery query mismatch: %v", err) + } +}