修复多应用异步回放和生产异常

This commit is contained in:
zhx 2026-07-13 22:26:59 +08:00
parent 1d581ff493
commit f0c4df7aa3
67 changed files with 3794 additions and 160 deletions

68
pkg/apptracking/limits.go Normal file
View File

@ -0,0 +1,68 @@
// Package apptracking defines the transport and persistence limits shared by the
// public App tracking endpoint and statistics-service.
package apptracking
import (
"bytes"
"encoding/json"
"strings"
)
const (
// MaxBatchSize keeps one request bounded before gateway fans it into the
// statistics write transaction.
MaxBatchSize = 50
// MaxPropertiesBytes bounds the raw JSON value after surrounding whitespace
// is removed. The value is rejected instead of truncated because truncation
// can change JSON meaning or produce invalid JSON.
MaxPropertiesBytes = 4096
// Text limits are byte limits, not rune limits. They mirror the durable
// app_tracking_events columns and therefore apply identically at both service
// boundaries; an oversized client value must never be deferred to MySQL.
MaxAppCodeBytes = 32
MaxEventIDBytes = 160
MaxEventNameBytes = 96
MaxEventTypeBytes = 64
MaxScreenBytes = 128
MaxTargetTypeBytes = 64
MaxTargetIDBytes = 128
MaxDeviceIDBytes = 128
MaxSessionIDBytes = 160
MaxPlatformBytes = 32
MaxAppVersionBytes = 64
MaxLanguageBytes = 32
MaxTimezoneBytes = 64
MaxErrorCodeBytes = 64
)
// ValidBatchSize rejects both empty reports and reports large enough to hold a
// statistics transaction open for an unbounded period.
func ValidBatchSize(size int) bool {
return size > 0 && size <= MaxBatchSize
}
// NormalizeText removes transport-only surrounding whitespace and checks the
// encoded byte length. Oversized values return false and are never shortened:
// identifiers and error codes are facts whose suffix may be significant.
func NormalizeText(value string, maxBytes int) (string, bool) {
value = strings.TrimSpace(value)
if maxBytes > 0 && len(value) > maxBytes {
return "", false
}
return value, true
}
// NormalizeProperties returns a detached, trimmed JSON value. Empty and JSON
// null both mean no properties; non-empty values must satisfy the same byte and
// syntax contract in gateway and statistics-service.
func NormalizeProperties(raw json.RawMessage) (json.RawMessage, bool) {
trimmed := bytes.TrimSpace(raw)
if len(trimmed) == 0 || bytes.Equal(trimmed, []byte("null")) {
return nil, true
}
if len(trimmed) > MaxPropertiesBytes || !json.Valid(trimmed) {
return nil, false
}
return json.RawMessage(bytes.Clone(trimmed)), true
}

View File

@ -0,0 +1,36 @@
package apptracking
import (
"encoding/json"
"strings"
"testing"
)
func TestNormalizeTextUsesBytesAndNeverTruncates(t *testing.T) {
if got, ok := NormalizeText(" "+strings.Repeat("e", MaxErrorCodeBytes)+" ", MaxErrorCodeBytes); !ok || got != strings.Repeat("e", MaxErrorCodeBytes) {
t.Fatalf("text at byte limit must be trimmed but otherwise preserved: ok=%v got=%q", ok, got)
}
if got, ok := NormalizeText(strings.Repeat("e", 185), MaxErrorCodeBytes); ok || got != "" {
t.Fatalf("oversized error_code must be rejected, never truncated: ok=%v got=%q", ok, got)
}
// 一个汉字编码为三个 UTF-8 bytes这里防止调用方误把字段限制实现成 rune 数。
if got, ok := NormalizeText(strings.Repeat("界", 22), MaxErrorCodeBytes); ok || got != "" {
t.Fatalf("text limits must count UTF-8 bytes: ok=%v got=%q", ok, got)
}
}
func TestNormalizePropertiesAndBatchContract(t *testing.T) {
if !ValidBatchSize(1) || !ValidBatchSize(MaxBatchSize) || ValidBatchSize(0) || ValidBatchSize(MaxBatchSize+1) {
t.Fatalf("batch bounds do not match the shared contract")
}
got, ok := NormalizeProperties(json.RawMessage(` {"screen":"home"} `))
if !ok || string(got) != `{"screen":"home"}` {
t.Fatalf("valid properties mismatch: ok=%v got=%s", ok, got)
}
if got, ok := NormalizeProperties(json.RawMessage(`"` + strings.Repeat("x", MaxPropertiesBytes) + `"`)); ok || got != nil {
t.Fatalf("properties over the encoded byte limit must be rejected: ok=%v bytes=%d", ok, len(got))
}
if got, ok := NormalizeProperties(json.RawMessage(`{"broken"`)); ok || got != nil {
t.Fatalf("invalid properties JSON must be rejected: ok=%v got=%s", ok, got)
}
}

View File

@ -0,0 +1,163 @@
// Package outboxpartition provides the shared tenant scheduling rules used by
// owner-service outbox workers. It does not read or publish business facts;
// repositories retain ownership of indexed claims and delivery state.
package outboxpartition
import (
"context"
"fmt"
"sort"
"strings"
"sync"
"sync/atomic"
"time"
"hyapp/pkg/appcode"
)
const DefaultDiscoveryRefreshInterval = 5 * time.Minute
// Partitions resolves the App partitions visible to one outbox lane and rotates
// the first partition across claims. A configured allowlist is immutable for the
// process lifetime; an empty allowlist falls back to low-frequency DB discovery.
type Partitions struct {
configured []string
refreshInterval time.Duration
mu sync.Mutex
cached []string
hasCache bool
nextRefresh time.Time
cursor atomic.Uint64
}
// New creates a partition scheduler. Operators can stage a backlog safely by
// configuring only the Apps currently allowed to publish; an empty list keeps
// automatic discovery as the compatibility default for existing deployments.
func New(configured []string, refreshInterval time.Duration) *Partitions {
if refreshInterval <= 0 {
refreshInterval = DefaultDiscoveryRefreshInterval
}
return &Partitions{
configured: Normalize(configured, false),
refreshInterval: refreshInterval,
}
}
// Resolve returns configured Apps without touching MySQL. In discovery mode it
// refreshes at most once per interval across all local worker goroutines. A
// failed refresh returns the last successful list alongside the error so the
// caller can log the stale condition without stopping already-known partitions.
func (p *Partitions) Resolve(ctx context.Context, discover func(context.Context) ([]string, error)) ([]string, error) {
if p == nil {
return nil, nil
}
if len(p.configured) > 0 {
return append([]string(nil), p.configured...), nil
}
if discover == nil {
return nil, nil
}
p.mu.Lock()
defer p.mu.Unlock()
now := time.Now().UTC()
if p.hasCache && now.Before(p.nextRefresh) {
return append([]string(nil), p.cached...), nil
}
discovered, err := discover(ctx)
if err != nil {
// Move the next attempt out by the normal interval: a transient database
// error must not turn every hot poll into a DISTINCT query stampede.
p.nextRefresh = now.Add(p.refreshInterval)
if p.hasCache {
return append([]string(nil), p.cached...), err
}
return nil, err
}
p.cached = Normalize(discovered, true)
p.hasCache = true
p.nextRefresh = now.Add(p.refreshInterval)
return append([]string(nil), p.cached...), nil
}
// Order rotates the first App atomically while preserving the remaining cyclic
// order. Multiple worker goroutines therefore share one fair cursor instead of
// independently preferring the first configured tenant.
func (p *Partitions) Order(appCodes []string) []string {
if len(appCodes) == 0 {
return nil
}
if p == nil {
return append([]string(nil), appCodes...)
}
start := int((p.cursor.Add(1) - 1) % uint64(len(appCodes)))
ordered := make([]string, 0, len(appCodes))
for offset := 0; offset < len(appCodes); offset++ {
ordered = append(ordered, appCodes[(start+offset)%len(appCodes)])
}
return ordered
}
// Normalize canonicalizes tenant keys, removes blanks and duplicates, and can
// sort DB-discovered keys for deterministic rotation. Explicit configuration
// preserves operator order so staged rollouts remain readable and predictable.
func Normalize(values []string, sorted bool) []string {
normalized := make([]string, 0, len(values))
seen := make(map[string]struct{}, len(values))
for _, value := range values {
if strings.TrimSpace(value) == "" {
continue
}
value = appcode.Normalize(value)
if _, exists := seen[value]; exists {
continue
}
seen[value] = struct{}{}
normalized = append(normalized, value)
}
if sorted {
sort.Strings(normalized)
}
return normalized
}
// ClaimFair gives every App one bounded first-pass quota, then lets non-empty
// partitions use capacity left by empty queues. Each callback remains App-scoped
// and is expected to perform its own indexed SKIP LOCKED transaction.
func ClaimFair[T any](ctx context.Context, orderedApps []string, limit int, claim func(context.Context, string, int) ([]T, error)) ([]T, error) {
if limit <= 0 || len(orderedApps) == 0 || claim == nil {
return nil, nil
}
quota := (limit + len(orderedApps) - 1) / len(orderedApps)
claimed := make([]T, 0, limit)
for pass := 0; pass < 2 && len(claimed) < limit; pass++ {
for _, app := range orderedApps {
remaining := limit - len(claimed)
if remaining <= 0 {
break
}
appLimit := remaining
if pass == 0 && appLimit > quota {
appLimit = quota
}
records, err := claim(ctx, app, appLimit)
if err != nil {
return nil, err
}
if len(records) > appLimit {
// Silently truncating would strand already-claimed rows until lease
// expiry. Treat repository limit violations as explicit worker errors.
return nil, fmt.Errorf("outbox claim for app %q returned %d records above limit %d", app, len(records), appLimit)
}
claimed = append(claimed, records...)
}
if pass == 0 && len(claimed) == 0 {
// A globally empty first pass must stop here; a second identical pass
// only doubles indexed polling queries and cannot discover new work.
break
}
}
return claimed, nil
}

View File

@ -0,0 +1,96 @@
package outboxpartition
import (
"context"
"errors"
"reflect"
"testing"
"time"
)
func TestConfiguredPartitionsNormalizeAndSkipDiscovery(t *testing.T) {
partitions := New([]string{" LALU ", "fami", "HUWAA", "fami", ""}, time.Hour)
discoverCalls := 0
apps, err := partitions.Resolve(context.Background(), func(context.Context) ([]string, error) {
discoverCalls++
return nil, errors.New("must not run")
})
if err != nil {
t.Fatalf("resolve configured partitions: %v", err)
}
if want := []string{"lalu", "fami", "huwaa"}; !reflect.DeepEqual(apps, want) {
t.Fatalf("configured apps = %v, want %v", apps, want)
}
if discoverCalls != 0 {
t.Fatalf("configured allowlist unexpectedly queried discovery %d times", discoverCalls)
}
}
func TestDiscoveredPartitionsAreCachedAndStaleListSurvivesRefreshFailure(t *testing.T) {
partitions := New(nil, time.Hour)
discoverCalls := 0
discover := func(context.Context) ([]string, error) {
discoverCalls++
if discoverCalls == 1 {
return []string{"huwaa", " LALU ", "fami"}, nil
}
return nil, errors.New("mysql unavailable")
}
first, err := partitions.Resolve(context.Background(), discover)
if err != nil || !reflect.DeepEqual(first, []string{"fami", "huwaa", "lalu"}) {
t.Fatalf("initial discovery apps=%v err=%v", first, err)
}
second, err := partitions.Resolve(context.Background(), discover)
if err != nil || !reflect.DeepEqual(second, first) || discoverCalls != 1 {
t.Fatalf("cached discovery apps=%v err=%v calls=%d", second, err, discoverCalls)
}
// Force only the refresh boundary; production reaches this state after the
// five-minute interval without exposing a public cache mutation API.
partitions.mu.Lock()
partitions.nextRefresh = time.Time{}
partitions.mu.Unlock()
stale, err := partitions.Resolve(context.Background(), discover)
if err == nil || !reflect.DeepEqual(stale, first) {
t.Fatalf("failed refresh must retain stale apps: apps=%v err=%v", stale, err)
}
}
func TestClaimFairRotatesLaluFamiHuwaaAndSkipsEmptyQueue(t *testing.T) {
partitions := New([]string{"lalu", "fami", "huwaa"}, time.Hour)
queues := map[string][]string{
"lalu": {"l1", "l2", "l3"},
"fami": nil,
"huwaa": {"h1", "h2", "h3"},
}
claim := func(_ context.Context, app string, limit int) ([]string, error) {
queue := queues[app]
if len(queue) < limit {
limit = len(queue)
}
result := append([]string(nil), queue[:limit]...)
queues[app] = queue[limit:]
return result, nil
}
apps, err := partitions.Resolve(context.Background(), nil)
if err != nil {
t.Fatalf("resolve configured apps: %v", err)
}
first, err := ClaimFair(context.Background(), partitions.Order(apps), 3, claim)
if err != nil || !reflect.DeepEqual(first, []string{"l1", "h1", "l2"}) {
t.Fatalf("first fair claim records=%v err=%v", first, err)
}
second, err := ClaimFair(context.Background(), partitions.Order(apps), 3, claim)
if err != nil || !reflect.DeepEqual(second, []string{"h2", "l3", "h3"}) {
t.Fatalf("rotated fair claim records=%v err=%v", second, err)
}
emptyCalls := 0
empty, err := ClaimFair[string](context.Background(), partitions.Order(apps), 3, func(context.Context, string, int) ([]string, error) {
emptyCalls++
return nil, nil
})
if err != nil || len(empty) != 0 || emptyCalls != 3 {
t.Fatalf("empty queues records=%v err=%v calls=%d", empty, err, emptyCalls)
}
}

View File

@ -1,6 +1,7 @@
package xerr package xerr
import ( import (
"context"
"errors" "errors"
"google.golang.org/genproto/googleapis/rpc/errdetails" "google.golang.org/genproto/googleapis/rpc/errdetails"
@ -27,6 +28,16 @@ func ToGRPCError(err error) error {
return err return err
} }
// context 终止描述的是调用链生命周期,而不是业务内部故障。仓储层会直接返回
// context.Canceled/DeadlineExceeded若先走兜底 Internal主动取消的并发 sibling
// 会在服务端被误报成 INTERNAL_ERROR并把一次正常的快速失败伪装成数据库事故。
if errors.Is(err, context.Canceled) {
return status.Error(codes.Canceled, context.Canceled.Error())
}
if errors.Is(err, context.DeadlineExceeded) {
return status.Error(codes.DeadlineExceeded, context.DeadlineExceeded.Error())
}
domainErr, ok := As(err) domainErr, ok := As(err)
if !ok { if !ok {
domainErr = &Error{Code: Internal, Message: "internal error"} domainErr = &Error{Code: Internal, Message: "internal error"}

View File

@ -1,6 +1,8 @@
package xerr package xerr
import ( import (
"context"
"fmt"
"go/ast" "go/ast"
"go/parser" "go/parser"
"go/token" "go/token"
@ -29,6 +31,29 @@ func TestToGRPCErrorCarriesReason(t *testing.T) {
} }
} }
// TestToGRPCErrorPreservesContextTermination 防止并发聚合器主动取消 sibling RPC 时,
// database/sql 返回的 context 错误被兜底成 Internal污染告警并掩盖真正的首个失败。
func TestToGRPCErrorPreservesContextTermination(t *testing.T) {
tests := []struct {
name string
err error
code codes.Code
}{
{name: "canceled", err: context.Canceled, code: codes.Canceled},
{name: "wrapped canceled", err: fmt.Errorf("query interrupted: %w", context.Canceled), code: codes.Canceled},
{name: "deadline exceeded", err: context.DeadlineExceeded, code: codes.DeadlineExceeded},
{name: "wrapped deadline exceeded", err: fmt.Errorf("query interrupted: %w", context.DeadlineExceeded), code: codes.DeadlineExceeded},
}
for _, test := range tests {
t.Run(test.name, func(t *testing.T) {
if got := status.Code(ToGRPCError(test.err)); got != test.code {
t.Fatalf("grpc code mismatch: got %s want %s", got, test.code)
}
})
}
}
func TestCatalogCoversDeclaredCodes(t *testing.T) { func TestCatalogCoversDeclaredCodes(t *testing.T) {
declared := declaredCodesFromSource(t) declared := declaredCodesFromSource(t)

View File

@ -0,0 +1,552 @@
// Command audit-wheel-rewards performs a read-only, item-by-item reconciliation
// between unsettled WheelRewardSettlement facts and wallet owner receipts.
package main
import (
"context"
"crypto/sha256"
"database/sql"
"encoding/hex"
"encoding/json"
"errors"
"flag"
"fmt"
"log"
"os"
"sort"
"strconv"
"strings"
"time"
"unicode"
_ "github.com/go-sql-driver/mysql"
"hyapp/pkg/appcode"
"hyapp/services/activity-service/internal/config"
domain "hyapp/services/activity-service/internal/domain/wheel"
)
type outboxRow struct {
AppCode string
OutboxID string
Payload []byte
Status string
RetryCount int
CreatedMS int64
}
type rewardPayload struct {
AppCode string `json:"app_code"`
DrawID string `json:"draw_id"`
DrawIDs []string `json:"draw_ids"`
CommandID string `json:"command_id"`
UserID int64 `json:"user_id"`
WheelID string `json:"wheel_id"`
SelectedTierID string `json:"selected_tier_id"`
RewardType string `json:"reward_type"`
RewardID string `json:"reward_id"`
RewardCount int64 `json:"reward_count"`
RewardCoins int64 `json:"reward_coins"`
MetadataJSON string `json:"metadata_json"`
Rewards []domain.DrawReward `json:"rewards"`
VisibleRegionID int64 `json:"visible_region_id"`
}
type itemAudit struct {
CommandID string `json:"command_id"`
Type string `json:"type"`
State string `json:"state"`
ReceiptID string `json:"receipt_id,omitempty"`
Reason string `json:"reason,omitempty"`
}
type rowAudit struct {
AppCode string `json:"app_code"`
OutboxID string `json:"outbox_id"`
Status string `json:"outbox_status"`
State string `json:"state"`
Items []itemAudit `json:"items"`
}
type auditSummary struct {
Rows int `json:"rows"`
FullyPaid int `json:"fully_paid"`
FullyUnpaid int `json:"fully_unpaid"`
PartiallyPaid int `json:"partially_paid"`
Conflict int `json:"conflict"`
PaidItems int `json:"paid_items"`
UnpaidItems int `json:"unpaid_items"`
ConflictItems int `json:"conflict_items"`
AuditSHA256 string `json:"audit_sha256"`
}
type appCodeFlags []string
func (values *appCodeFlags) String() string {
return strings.Join(*values, ",")
}
func (values *appCodeFlags) Set(value string) error {
value = strings.TrimSpace(value)
if value == "" {
return fmt.Errorf("app-code must not be empty")
}
*values = append(*values, value)
return nil
}
func main() {
configPath := flag.String("config", "services/activity-service/configs/config.yaml", "path to activity-service config")
walletDatabase := flag.String("wallet-database", "hyapp_wallet", "wallet owner database on the same MySQL cluster")
limit := flag.Int("limit", 500, "maximum unsettled rows, 1..500")
details := flag.Bool("details", false, "emit one JSON reconciliation record per outbox row")
var requestedAppCodes appCodeFlags
flag.Var(&requestedAppCodes, "app-code", "required App partition to audit; repeat for multiple Apps, for example --app-code lalu --app-code fami")
flag.Parse()
if *limit <= 0 || *limit > 500 {
log.Fatal("limit must be between 1 and 500")
}
walletDB := normalizedDatabaseName(*walletDatabase)
if walletDB == "" {
log.Fatal("wallet-database contains invalid characters")
}
appCodes, err := normalizeAuditAppCodes(requestedAppCodes)
if err != nil {
log.Fatal(err)
}
cfg, err := config.Load(*configPath)
if err != nil {
log.Fatal(err)
}
db, err := sql.Open("mysql", cfg.MySQLDSN)
if err != nil {
log.Fatal(err)
}
defer func() { _ = db.Close() }()
db.SetMaxOpenConns(2)
db.SetMaxIdleConns(1)
ctx, cancel := context.WithTimeout(context.Background(), 2*time.Minute)
defer cancel()
if err := db.PingContext(ctx); err != nil {
log.Fatal(err)
}
rows, err := loadUnsettledRows(ctx, db, appCodes, *limit)
if err != nil {
log.Fatal(err)
}
audits := make([]rowAudit, 0, len(rows))
for _, row := range rows {
audit, err := auditRow(ctx, db, walletDB, row)
if err != nil {
log.Fatalf("audit app_code=%s outbox_id=%s: %v", row.AppCode, row.OutboxID, err)
}
audits = append(audits, audit)
}
summary := summarize(audits)
if *details {
encoder := json.NewEncoder(os.Stdout)
for _, audit := range audits {
if err := encoder.Encode(audit); err != nil {
log.Fatal(err)
}
}
}
encoded, err := json.Marshal(summary)
if err != nil {
log.Fatal(err)
}
fmt.Println(string(encoded))
}
func loadUnsettledRows(ctx context.Context, db *sql.DB, requestedAppCodes []string, limit int) ([]outboxRow, error) {
if limit <= 0 || limit > 500 {
return nil, fmt.Errorf("limit must be between 1 and 500")
}
appCodes, err := normalizeAuditAppCodes(requestedAppCodes)
if err != nil {
return nil, err
}
// activity_outbox 的线上索引都以 app_code 开头。每个 App 拆成 retry/lock 两个等值分区查询,
// 既能强制使用对应索引,也避免 status OR 让优化器退化为全表扫描。每个分支只取 global remaining
// 合并出该 App 最早的一页后再进入下一 App因此返回量严格不超过 limit数据库返回行数也被约束在约 2*limit 内。
result := make([]outboxRow, 0, limit)
for _, appCodeValue := range appCodes {
remaining := limit - len(result)
if remaining <= 0 {
break
}
branches := []struct {
query string
args []any
}{
{
query: unsettledRetryQuery(),
args: []any{
appCodeValue,
domain.EventTypeWheelRewardSettlement,
domain.OutboxStatusPending,
domain.OutboxStatusRetryable,
remaining,
},
},
{
query: unsettledDeliveringQuery(),
args: []any{
appCodeValue,
domain.EventTypeWheelRewardSettlement,
domain.OutboxStatusDelivering,
remaining,
},
},
}
appCandidates := make([]outboxRow, 0, remaining)
seen := make(map[string]struct{}, remaining)
for _, branch := range branches {
rows, err := loadUnsettledBranch(ctx, db, branch.query, branch.args...)
if err != nil {
return nil, fmt.Errorf("load app_code=%s: %w", appCodeValue, err)
}
for _, row := range rows {
key := row.AppCode + "\x00" + row.OutboxID
if _, exists := seen[key]; exists {
continue
}
seen[key] = struct{}{}
appCandidates = append(appCandidates, row)
}
}
sort.Slice(appCandidates, func(i, j int) bool {
if appCandidates[i].CreatedMS != appCandidates[j].CreatedMS {
return appCandidates[i].CreatedMS < appCandidates[j].CreatedMS
}
if appCandidates[i].OutboxID != appCandidates[j].OutboxID {
return appCandidates[i].OutboxID < appCandidates[j].OutboxID
}
return appCandidates[i].Status < appCandidates[j].Status
})
if len(appCandidates) > remaining {
appCandidates = appCandidates[:remaining]
}
result = append(result, appCandidates...)
}
return result, nil
}
func unsettledRetryQuery() string {
return `
SELECT app_code, outbox_id, payload, status, retry_count, created_at_ms
FROM activity_outbox FORCE INDEX (idx_activity_outbox_event_retry)
WHERE app_code = ? AND event_type = ? AND status IN (?, ?)
ORDER BY created_at_ms, outbox_id
LIMIT ?`
}
func unsettledDeliveringQuery() string {
return `
SELECT app_code, outbox_id, payload, status, retry_count, created_at_ms
FROM activity_outbox FORCE INDEX (idx_activity_outbox_event_lock)
WHERE app_code = ? AND event_type = ? AND status = ?
ORDER BY created_at_ms, outbox_id
LIMIT ?`
}
func loadUnsettledBranch(ctx context.Context, db *sql.DB, query string, args ...any) ([]outboxRow, error) {
rows, err := db.QueryContext(ctx, query, args...)
if err != nil {
return nil, err
}
defer rows.Close()
result := make([]outboxRow, 0)
for rows.Next() {
var row outboxRow
if err := rows.Scan(&row.AppCode, &row.OutboxID, &row.Payload, &row.Status, &row.RetryCount, &row.CreatedMS); err != nil {
return nil, err
}
result = append(result, row)
}
return result, rows.Err()
}
func normalizeAuditAppCodes(values []string) ([]string, error) {
if len(values) == 0 {
return nil, fmt.Errorf("at least one --app-code is required")
}
const maxAuditAppCodes = 16
seen := make(map[string]struct{}, len(values))
result := make([]string, 0, len(values))
for _, value := range values {
if strings.TrimSpace(value) == "" {
return nil, fmt.Errorf("app-code must not be empty")
}
value = appcode.Normalize(value)
if _, exists := seen[value]; exists {
continue
}
seen[value] = struct{}{}
result = append(result, value)
}
if len(result) == 0 {
return nil, fmt.Errorf("at least one --app-code is required")
}
if len(result) > maxAuditAppCodes {
return nil, fmt.Errorf("at most %d app-code values are allowed", maxAuditAppCodes)
}
sort.Strings(result)
return result, nil
}
func auditRow(ctx context.Context, db *sql.DB, walletDB string, row outboxRow) (rowAudit, error) {
var payload rewardPayload
if err := json.Unmarshal(row.Payload, &payload); err != nil {
return rowAudit{}, fmt.Errorf("decode immutable payload: %w", err)
}
row.AppCode = appcode.Normalize(row.AppCode)
if payload.AppCode != "" && appcode.Normalize(payload.AppCode) != row.AppCode {
return rowAudit{}, fmt.Errorf("payload app_code does not match outbox partition")
}
payload.DrawID = strings.TrimSpace(payload.DrawID)
payload.WheelID = strings.TrimSpace(payload.WheelID)
if payload.DrawID == "" || payload.WheelID == "" || payload.UserID <= 0 {
return rowAudit{}, fmt.Errorf("payload identity is incomplete")
}
rewards := payload.Rewards
if len(rewards) == 0 {
rewards = []domain.DrawReward{{
SelectedTierID: payload.SelectedTierID,
RewardType: payload.RewardType,
RewardID: payload.RewardID,
RewardCount: payload.RewardCount,
RewardCoins: payload.RewardCoins,
MetadataJSON: payload.MetadataJSON,
}}
}
if len(rewards) == 0 {
return rowAudit{}, fmt.Errorf("payload reward snapshot is empty")
}
audit := rowAudit{AppCode: row.AppCode, OutboxID: row.OutboxID, Status: row.Status, Items: make([]itemAudit, 0, len(rewards))}
for index, reward := range rewards {
reward.RewardType = strings.ToLower(strings.TrimSpace(reward.RewardType))
var item itemAudit
var err error
switch reward.RewardType {
case domain.RewardTypeCoin:
item, err = auditCoin(ctx, db, walletDB, row.AppCode, payload, reward, index, len(rewards))
case domain.RewardTypeGift, domain.RewardTypeProp, domain.RewardTypeDress:
item, err = auditResource(ctx, db, walletDB, row.AppCode, payload, reward, index, len(rewards))
default:
item = itemAudit{Type: reward.RewardType, State: "conflict", Reason: "unsupported reward type"}
}
if err != nil {
return rowAudit{}, err
}
audit.Items = append(audit.Items, item)
}
audit.State = aggregateRowState(audit.Items)
return audit, nil
}
func auditCoin(ctx context.Context, db *sql.DB, walletDB string, app string, payload rewardPayload, reward domain.DrawReward, index int, total int) (itemAudit, error) {
commandID := rewardCommandID("wheel_reward", payload.DrawID, index, total)
item := itemAudit{CommandID: commandID, Type: domain.RewardTypeCoin, State: "unpaid"}
query := coinAuditQuery(walletDB)
var transactionID, bizType, statusValue, metadataJSON string
err := db.QueryRowContext(ctx, query, app, commandID).Scan(&transactionID, &bizType, &statusValue, &metadataJSON)
if err != nil {
// database/sql 和 driver/observer 包装都可能给 ErrNoRows 增加上下文;审计必须把这种稳定语义归为 unpaid而不是中止整批核账。
if errors.Is(err, sql.ErrNoRows) {
return item, nil
}
return itemAudit{}, err
}
metadata := map[string]any{}
if err := json.Unmarshal([]byte(metadataJSON), &metadata); err != nil {
item.State, item.Reason = "conflict", "wallet metadata is invalid"
return item, nil
}
// wallet_transactions 和 resource_grants 共用 succeeded 终态completed 从未是当前账本状态,不能把真实已发金币误判为冲突。
if bizType != "wheel_reward" || statusValue != "succeeded" || jsonInt64(metadata, "target_user_id") != payload.UserID || jsonInt64(metadata, "amount") != reward.RewardCoins || jsonString(metadata, "draw_id") != payload.DrawID || jsonString(metadata, "wheel_id") != payload.WheelID || jsonString(metadata, "selected_tier_id") != strings.TrimSpace(reward.SelectedTierID) || jsonString(metadata, "reason") != "wheel_reward" || jsonInt64(metadata, "visible_region_id") != payload.VisibleRegionID {
item.State, item.Reason = "conflict", "existing wallet transaction does not match immutable reward snapshot"
return item, nil
}
item.State = "paid"
item.ReceiptID = transactionID
return item, nil
}
func coinAuditQuery(walletDB string) string {
return fmt.Sprintf(`
SELECT transaction_id, biz_type, status, COALESCE(CAST(metadata_json AS CHAR), '{}')
FROM %s.wallet_transactions
WHERE app_code = ? AND command_id = ?`, walletDB)
}
func auditResource(ctx context.Context, db *sql.DB, walletDB string, app string, payload rewardPayload, reward domain.DrawReward, index int, total int) (itemAudit, error) {
commandID := rewardCommandID("wheel_resource", payload.DrawID, index, total)
item := itemAudit{CommandID: commandID, Type: reward.RewardType, State: "unpaid"}
resourceID := rewardResourceID(reward)
quantity := reward.RewardCount
if quantity <= 0 {
quantity = 1
}
durationMS := metadataInt64(reward.MetadataJSON, "duration_ms", "durationMs", "validity_ms", "validityMs")
query := resourceAuditQuery(walletDB)
var grantID, statusValue, grantSource, subjectType, subjectID, reason string
var targetUserID, operatorUserID, actualQuantity, actualDurationMS, actualResourceID int64
err := db.QueryRowContext(ctx, query, app, commandID).Scan(&grantID, &statusValue, &targetUserID, &grantSource, &subjectType, &subjectID, &reason, &operatorUserID, &actualQuantity, &actualDurationMS, &actualResourceID)
if err != nil {
if errors.Is(err, sql.ErrNoRows) {
return item, nil
}
return itemAudit{}, err
}
if resourceID <= 0 || statusValue != "succeeded" || targetUserID != payload.UserID || grantSource != "wheel_reward" || subjectType != "resource" || subjectID != strconv.FormatInt(resourceID, 10) || reason != "wheel_reward" || operatorUserID != payload.UserID || actualQuantity != quantity || actualDurationMS != durationMS || actualResourceID != resourceID {
item.State, item.Reason = "conflict", "existing resource grant does not match immutable reward snapshot"
return item, nil
}
item.State = "paid"
item.ReceiptID = grantID
return item, nil
}
func resourceAuditQuery(walletDB string) string {
// 每个 SELECT 列只出现一次且与 Scan 顺序一一对应;显式逐列排版避免核账命令在生产才暴露漏逗号或重复列。
return fmt.Sprintf(`
SELECT rg.grant_id,
rg.status,
rg.target_user_id,
rg.grant_source,
rg.grant_subject_type,
rg.grant_subject_id,
rg.reason,
rg.operator_user_id,
COALESCE(rgi.quantity, 0),
COALESCE(rgi.duration_ms, 0),
COALESCE(rgi.resource_id, 0)
FROM %s.resource_grants rg
LEFT JOIN %s.resource_grant_items rgi
ON rgi.app_code = rg.app_code AND rgi.grant_id = rg.grant_id
WHERE rg.app_code = ? AND rg.command_id = ?
ORDER BY rgi.grant_item_id
LIMIT 1`, walletDB, walletDB)
}
func aggregateRowState(items []itemAudit) string {
paid, unpaid, conflict := 0, 0, 0
for _, item := range items {
switch item.State {
case "paid":
paid++
case "unpaid":
unpaid++
default:
conflict++
}
}
if conflict > 0 {
return "conflict"
}
if paid == len(items) {
return "fully_paid"
}
if unpaid == len(items) {
return "fully_unpaid"
}
return "partially_paid"
}
func summarize(audits []rowAudit) auditSummary {
summary := auditSummary{Rows: len(audits)}
canonical := make([]string, 0, len(audits))
for _, audit := range audits {
switch audit.State {
case "fully_paid":
summary.FullyPaid++
case "fully_unpaid":
summary.FullyUnpaid++
case "partially_paid":
summary.PartiallyPaid++
default:
summary.Conflict++
}
for _, item := range audit.Items {
switch item.State {
case "paid":
summary.PaidItems++
case "unpaid":
summary.UnpaidItems++
default:
summary.ConflictItems++
}
}
encoded, _ := json.Marshal(audit)
canonical = append(canonical, string(encoded))
}
sort.Strings(canonical)
sum := sha256.Sum256([]byte(strings.Join(canonical, "\n")))
summary.AuditSHA256 = hex.EncodeToString(sum[:])
return summary
}
func rewardCommandID(prefix string, drawID string, index int, total int) string {
if total <= 1 {
return prefix + ":" + drawID
}
return fmt.Sprintf("%s:%s:%d", prefix, drawID, index+1)
}
func rewardResourceID(reward domain.DrawReward) int64 {
if value := metadataInt64(reward.MetadataJSON, "resource_id", "resourceId"); value > 0 {
return value
}
value, _ := strconv.ParseInt(strings.TrimSpace(reward.RewardID), 10, 64)
return value
}
func metadataInt64(raw string, keys ...string) int64 {
metadata := map[string]any{}
if err := json.Unmarshal([]byte(strings.TrimSpace(raw)), &metadata); err != nil {
return 0
}
for _, key := range keys {
if value := jsonInt64(metadata, key); value > 0 {
return value
}
}
return 0
}
func jsonInt64(values map[string]any, key string) int64 {
switch value := values[key].(type) {
case float64:
return int64(value)
case json.Number:
parsed, _ := value.Int64()
return parsed
case string:
parsed, _ := strconv.ParseInt(strings.TrimSpace(value), 10, 64)
return parsed
default:
return 0
}
}
func jsonString(values map[string]any, key string) string {
value, _ := values[key].(string)
return strings.TrimSpace(value)
}
func normalizedDatabaseName(value string) string {
value = strings.TrimSpace(value)
if value == "" {
return ""
}
for _, char := range value {
if char == '_' || unicode.IsDigit(char) || unicode.IsLetter(char) {
continue
}
return ""
}
return value
}

View File

@ -0,0 +1,306 @@
package main
import (
"context"
"database/sql"
"encoding/json"
"fmt"
"regexp"
"testing"
"github.com/DATA-DOG/go-sqlmock"
domain "hyapp/services/activity-service/internal/domain/wheel"
)
func TestAuditRowCoinStates(t *testing.T) {
tests := []struct {
name string
prepare func(sqlmock.Sqlmock, rewardPayload)
wantState string
wantItem string
}{
{
name: "fully paid",
prepare: func(mock sqlmock.Sqlmock, payload rewardPayload) {
metadata, _ := json.Marshal(map[string]any{
"target_user_id": payload.UserID,
"amount": int64(200),
"draw_id": payload.DrawID,
"wheel_id": payload.WheelID,
"selected_tier_id": "coin-200",
"reason": "wheel_reward",
"visible_region_id": payload.VisibleRegionID,
})
mock.ExpectQuery(regexp.QuoteMeta(coinAuditQuery("hyapp_wallet"))).
WithArgs("fami", "wheel_reward:draw-1").
WillReturnRows(coinAuditRows().AddRow("wtx-1", "wheel_reward", "succeeded", string(metadata)))
},
wantState: "fully_paid",
wantItem: "paid",
},
{
name: "fully unpaid uses errors Is",
prepare: func(mock sqlmock.Sqlmock, _ rewardPayload) {
mock.ExpectQuery(regexp.QuoteMeta(coinAuditQuery("hyapp_wallet"))).
WithArgs("fami", "wheel_reward:draw-1").
WillReturnError(fmt.Errorf("wrapped wallet miss: %w", sql.ErrNoRows))
},
wantState: "fully_unpaid",
wantItem: "unpaid",
},
{
name: "conflict on immutable amount",
prepare: func(mock sqlmock.Sqlmock, payload rewardPayload) {
metadata, _ := json.Marshal(map[string]any{
"target_user_id": payload.UserID,
"amount": int64(201),
"draw_id": payload.DrawID,
"wheel_id": payload.WheelID,
"selected_tier_id": "coin-200",
"reason": "wheel_reward",
"visible_region_id": payload.VisibleRegionID,
})
mock.ExpectQuery(regexp.QuoteMeta(coinAuditQuery("hyapp_wallet"))).
WithArgs("fami", "wheel_reward:draw-1").
WillReturnRows(coinAuditRows().AddRow("wtx-conflict", "wheel_reward", "succeeded", string(metadata)))
},
wantState: "conflict",
wantItem: "conflict",
},
{
name: "conflict on biz type",
prepare: func(mock sqlmock.Sqlmock, payload rewardPayload) {
metadata := matchingCoinMetadata(payload)
mock.ExpectQuery(regexp.QuoteMeta(coinAuditQuery("hyapp_wallet"))).
WithArgs("fami", "wheel_reward:draw-1").
WillReturnRows(coinAuditRows().AddRow("wtx-wrong-biz", "task_reward", "succeeded", metadata))
},
wantState: "conflict",
wantItem: "conflict",
},
{
name: "conflict on visible region",
prepare: func(mock sqlmock.Sqlmock, payload rewardPayload) {
metadata := matchingCoinMetadata(payload)
var values map[string]any
_ = json.Unmarshal([]byte(metadata), &values)
values["visible_region_id"] = int64(99)
mismatch, _ := json.Marshal(values)
mock.ExpectQuery(regexp.QuoteMeta(coinAuditQuery("hyapp_wallet"))).
WithArgs("fami", "wheel_reward:draw-1").
WillReturnRows(coinAuditRows().AddRow("wtx-wrong-region", "wheel_reward", "succeeded", string(mismatch)))
},
wantState: "conflict",
wantItem: "conflict",
},
}
for _, test := range tests {
t.Run(test.name, func(t *testing.T) {
db, mock, err := sqlmock.New()
if err != nil {
t.Fatalf("create sqlmock: %v", err)
}
defer db.Close()
payload := rewardPayload{
AppCode: "fami",
DrawID: "draw-1",
UserID: 1001,
WheelID: "classic",
VisibleRegionID: 9,
Rewards: []domain.DrawReward{{SelectedTierID: "coin-200", RewardType: domain.RewardTypeCoin, RewardCoins: 200}},
}
test.prepare(mock, payload)
audit, err := auditRow(context.Background(), db, "hyapp_wallet", auditOutboxRow(t, payload))
if err != nil {
t.Fatalf("auditRow returned error: %v", err)
}
if audit.State != test.wantState || len(audit.Items) != 1 || audit.Items[0].State != test.wantItem {
t.Fatalf("unexpected audit: %+v", audit)
}
if err := mock.ExpectationsWereMet(); err != nil {
t.Fatalf("unmet SQL expectations: %v", err)
}
})
}
}
func TestAuditRowResourceStates(t *testing.T) {
tests := []struct {
name string
prepare func(sqlmock.Sqlmock)
wantState string
wantItem string
}{
{
name: "fully paid",
prepare: func(mock sqlmock.Sqlmock) {
mock.ExpectQuery(regexp.QuoteMeta(resourceAuditQuery("hyapp_wallet"))).
WithArgs("fami", "wheel_resource:draw-1").
WillReturnRows(resourceAuditRows().AddRow("rgr-1", "succeeded", int64(1001), "wheel_reward", "resource", "326", "wheel_reward", int64(1001), int64(2), int64(5000), int64(326)))
},
wantState: "fully_paid",
wantItem: "paid",
},
{
name: "fully unpaid uses errors Is",
prepare: func(mock sqlmock.Sqlmock) {
mock.ExpectQuery(regexp.QuoteMeta(resourceAuditQuery("hyapp_wallet"))).
WithArgs("fami", "wheel_resource:draw-1").
WillReturnError(fmt.Errorf("wrapped resource miss: %w", sql.ErrNoRows))
},
wantState: "fully_unpaid",
wantItem: "unpaid",
},
{
name: "conflict rejects legacy active status",
prepare: func(mock sqlmock.Sqlmock) {
mock.ExpectQuery(regexp.QuoteMeta(resourceAuditQuery("hyapp_wallet"))).
WithArgs("fami", "wheel_resource:draw-1").
WillReturnRows(resourceAuditRows().AddRow("rgr-conflict", "active", int64(1001), "wheel_reward", "resource", "326", "wheel_reward", int64(1001), int64(2), int64(5000), int64(326)))
},
wantState: "conflict",
wantItem: "conflict",
},
}
for _, test := range tests {
t.Run(test.name, func(t *testing.T) {
db, mock, err := sqlmock.New()
if err != nil {
t.Fatalf("create sqlmock: %v", err)
}
defer db.Close()
payload := rewardPayload{
AppCode: "fami",
DrawID: "draw-1",
UserID: 1001,
WheelID: "classic",
Rewards: []domain.DrawReward{{
SelectedTierID: "gift-326",
RewardType: domain.RewardTypeGift,
RewardID: "326",
RewardCount: 2,
MetadataJSON: `{"resource_id":326,"duration_ms":5000}`,
}},
}
test.prepare(mock)
audit, err := auditRow(context.Background(), db, "hyapp_wallet", auditOutboxRow(t, payload))
if err != nil {
t.Fatalf("auditRow returned error: %v", err)
}
if audit.State != test.wantState || len(audit.Items) != 1 || audit.Items[0].State != test.wantItem {
t.Fatalf("unexpected audit: %+v", audit)
}
if err := mock.ExpectationsWereMet(); err != nil {
t.Fatalf("unmet SQL expectations: %v", err)
}
})
}
}
func TestLoadUnsettledRowsUsesAppScopedIndexesAndGlobalLimit(t *testing.T) {
db, mock, err := sqlmock.New()
if err != nil {
t.Fatalf("create sqlmock: %v", err)
}
defer db.Close()
const limit = 3
columns := []string{"app_code", "outbox_id", "payload", "status", "retry_count", "created_at_ms"}
// normalizeAuditAppCodes sorts and deduplicates, so SQL order is deterministic regardless of flag order.
mock.ExpectQuery(regexp.QuoteMeta(unsettledRetryQuery())).
WithArgs("fami", domain.EventTypeWheelRewardSettlement, domain.OutboxStatusPending, domain.OutboxStatusRetryable, limit).
WillReturnRows(sqlmock.NewRows(columns).
AddRow("fami", "fami-pending", []byte(`{}`), domain.OutboxStatusPending, 0, int64(20)))
mock.ExpectQuery(regexp.QuoteMeta(unsettledDeliveringQuery())).
WithArgs("fami", domain.EventTypeWheelRewardSettlement, domain.OutboxStatusDelivering, limit).
WillReturnRows(sqlmock.NewRows(columns).
AddRow("fami", "fami-delivering", []byte(`{}`), domain.OutboxStatusDelivering, 1, int64(10)))
mock.ExpectQuery(regexp.QuoteMeta(unsettledRetryQuery())).
WithArgs("lalu", domain.EventTypeWheelRewardSettlement, domain.OutboxStatusPending, domain.OutboxStatusRetryable, 1).
WillReturnRows(sqlmock.NewRows(columns).
AddRow("lalu", "lalu-oldest", []byte(`{}`), domain.OutboxStatusRetryable, 2, int64(5)))
mock.ExpectQuery(regexp.QuoteMeta(unsettledDeliveringQuery())).
WithArgs("lalu", domain.EventTypeWheelRewardSettlement, domain.OutboxStatusDelivering, 1).
WillReturnRows(sqlmock.NewRows(columns).
AddRow("lalu", "lalu-delivering", []byte(`{}`), domain.OutboxStatusDelivering, 1, int64(15)))
rows, err := loadUnsettledRows(context.Background(), db, []string{"lalu", "fami", "fami"}, limit)
if err != nil {
t.Fatalf("loadUnsettledRows returned error: %v", err)
}
if len(rows) != limit {
t.Fatalf("global limit mismatch: got=%d want=%d rows=%+v", len(rows), limit, rows)
}
wantIDs := []string{"fami-delivering", "fami-pending", "lalu-oldest"}
for index, wantID := range wantIDs {
if rows[index].OutboxID != wantID {
t.Fatalf("deterministic order[%d]=%q want=%q rows=%+v", index, rows[index].OutboxID, wantID, rows)
}
}
if err := mock.ExpectationsWereMet(); err != nil {
t.Fatalf("unmet SQL expectations: %v", err)
}
}
func TestNormalizeAuditAppCodesRequiresRepeatedExplicitScope(t *testing.T) {
if _, err := normalizeAuditAppCodes(nil); err == nil {
t.Fatal("missing --app-code must be rejected before opening or querying MySQL")
}
values, err := normalizeAuditAppCodes([]string{" LALU ", "fami", "lalu", "huwaa"})
if err != nil {
t.Fatalf("normalizeAuditAppCodes returned error: %v", err)
}
want := []string{"fami", "huwaa", "lalu"}
if len(values) != len(want) {
t.Fatalf("normalized app count=%d want=%d values=%+v", len(values), len(want), values)
}
for index := range want {
if values[index] != want[index] {
t.Fatalf("normalized app[%d]=%q want=%q", index, values[index], want[index])
}
}
}
func auditOutboxRow(t *testing.T, payload rewardPayload) outboxRow {
t.Helper()
body, err := json.Marshal(payload)
if err != nil {
t.Fatalf("marshal payload: %v", err)
}
return outboxRow{AppCode: payload.AppCode, OutboxID: "wheel_reward_" + payload.DrawID, Payload: body, Status: domain.OutboxStatusPending}
}
func resourceAuditRows() *sqlmock.Rows {
return sqlmock.NewRows([]string{
"grant_id",
"status",
"target_user_id",
"grant_source",
"grant_subject_type",
"grant_subject_id",
"reason",
"operator_user_id",
"quantity",
"duration_ms",
"resource_id",
})
}
func coinAuditRows() *sqlmock.Rows {
return sqlmock.NewRows([]string{"transaction_id", "biz_type", "status", "metadata_json"})
}
func matchingCoinMetadata(payload rewardPayload) string {
body, _ := json.Marshal(map[string]any{
"target_user_id": payload.UserID,
"amount": int64(200),
"draw_id": payload.DrawID,
"wheel_id": payload.WheelID,
"selected_tier_id": "coin-200",
"reason": "wheel_reward",
"visible_region_id": payload.VisibleRegionID,
})
return string(body)
}

View File

@ -30,6 +30,13 @@ user_leaderboard_worker:
redis_password: "" redis_password: ""
redis_db: 0 redis_db: 0
key_prefix: "activity:user_leaderboard" key_prefix: "activity:user_leaderboard"
wheel_reward_worker:
enabled: true
app_codes: ["lalu", "fami", "huwaa"]
poll_interval: "1s"
batch_size: 50
lock_ttl: "30s"
max_retry: 16
message_action_confirm_worker: message_action_confirm_worker:
enabled: true enabled: true
outbox_poll_interval: "1s" outbox_poll_interval: "1s"

View File

@ -30,6 +30,13 @@ user_leaderboard_worker:
redis_password: "TENCENT_REDIS_PASSWORD" redis_password: "TENCENT_REDIS_PASSWORD"
redis_db: 0 redis_db: 0
key_prefix: "activity:user_leaderboard" key_prefix: "activity:user_leaderboard"
wheel_reward_worker:
enabled: true
app_codes: ["lalu", "fami", "huwaa"]
poll_interval: "1s"
batch_size: 50
lock_ttl: "30s"
max_retry: 16
message_action_confirm_worker: message_action_confirm_worker:
enabled: true enabled: true
outbox_poll_interval: "1s" outbox_poll_interval: "1s"

View File

@ -30,6 +30,13 @@ user_leaderboard_worker:
redis_password: "" redis_password: ""
redis_db: 0 redis_db: 0
key_prefix: "activity:user_leaderboard" key_prefix: "activity:user_leaderboard"
wheel_reward_worker:
enabled: true
app_codes: ["lalu", "fami", "huwaa"]
poll_interval: "1s"
batch_size: 50
lock_ttl: "30s"
max_retry: 16
message_action_confirm_worker: message_action_confirm_worker:
enabled: true enabled: true
outbox_poll_interval: "1s" outbox_poll_interval: "1s"

View File

@ -21,6 +21,7 @@ import (
firstrechargeservice "hyapp/services/activity-service/internal/service/firstrecharge" firstrechargeservice "hyapp/services/activity-service/internal/service/firstrecharge"
inviteactivityservice "hyapp/services/activity-service/internal/service/inviteactivity" inviteactivityservice "hyapp/services/activity-service/internal/service/inviteactivity"
messageservice "hyapp/services/activity-service/internal/service/message" messageservice "hyapp/services/activity-service/internal/service/message"
wheelservice "hyapp/services/activity-service/internal/service/wheel"
mysqlstorage "hyapp/services/activity-service/internal/storage/mysql" mysqlstorage "hyapp/services/activity-service/internal/storage/mysql"
) )
@ -37,10 +38,13 @@ type App struct {
cumulativeRecharge *cumulativerechargeservice.Service cumulativeRecharge *cumulativerechargeservice.Service
inviteActivityReward *inviteactivityservice.Service inviteActivityReward *inviteactivityservice.Service
actionConfirm *messageservice.ActionConfirmService actionConfirm *messageservice.ActionConfirmService
wheel *wheelservice.Service
broadcastWorkerEnabled bool broadcastWorkerEnabled bool
messageActionWorkerEnabled bool messageActionWorkerEnabled bool
wheelRewardWorkerEnabled bool
workerNodeID string workerNodeID string
messageActionWorkerOptions config.MessageActionConfirmWorkerConfig messageActionWorkerOptions config.MessageActionConfirmWorkerConfig
wheelRewardWorkerOptions config.WheelRewardWorkerConfig
messageActionOutboxTopic string messageActionOutboxTopic string
userLeaderboardRedisClose func() error userLeaderboardRedisClose func() error
mqConsumers []*rocketmqx.Consumer mqConsumers []*rocketmqx.Consumer
@ -133,10 +137,13 @@ func New(cfg config.Config) (*App, error) {
cumulativeRecharge: services.cumulativeRecharge, cumulativeRecharge: services.cumulativeRecharge,
inviteActivityReward: services.inviteActivityReward, inviteActivityReward: services.inviteActivityReward,
actionConfirm: services.actionConfirm, actionConfirm: services.actionConfirm,
wheel: services.wheel,
broadcastWorkerEnabled: cfg.Broadcast.Enabled, broadcastWorkerEnabled: cfg.Broadcast.Enabled,
messageActionWorkerEnabled: cfg.MessageActionConfirmWorker.Enabled, messageActionWorkerEnabled: cfg.MessageActionConfirmWorker.Enabled,
wheelRewardWorkerEnabled: cfg.WheelRewardWorker.Enabled,
workerNodeID: cfg.NodeID, workerNodeID: cfg.NodeID,
messageActionWorkerOptions: cfg.MessageActionConfirmWorker, messageActionWorkerOptions: cfg.MessageActionConfirmWorker,
wheelRewardWorkerOptions: cfg.WheelRewardWorker,
messageActionOutboxTopic: cfg.RocketMQ.MessageActionOutbox.Topic, messageActionOutboxTopic: cfg.RocketMQ.MessageActionOutbox.Topic,
userLeaderboardRedisClose: userLeaderboardRedisClose, userLeaderboardRedisClose: userLeaderboardRedisClose,
mqConsumers: mqConsumers, mqConsumers: mqConsumers,
@ -166,6 +173,18 @@ func (a *App) Run() error {
a.runMessageActionOutboxWorker(ctx) a.runMessageActionOutboxWorker(ctx)
}) })
} }
if a.wheelRewardWorkerEnabled && a.wheel != nil && a.workers != nil {
a.workers.Go(func(ctx context.Context) {
a.wheel.RunRewardWorker(ctx, wheelservice.RewardWorkerOptions{
WorkerID: a.workerNodeID + ":wheel-reward",
AppCodes: a.wheelRewardWorkerOptions.AppCodes,
PollInterval: a.wheelRewardWorkerOptions.PollInterval,
BatchSize: a.wheelRewardWorkerOptions.BatchSize,
LockTTL: a.wheelRewardWorkerOptions.LockTTL,
MaxRetry: a.wheelRewardWorkerOptions.MaxRetry,
})
})
}
err := a.server.Serve(a.listener) err := a.server.Serve(a.listener)
if a.workers != nil { if a.workers != nil {
a.workers.StopAndWait() a.workers.StopAndWait()

View File

@ -40,6 +40,8 @@ type Config struct {
TaskEventWorker TaskEventWorkerConfig `yaml:"task_event_worker"` TaskEventWorker TaskEventWorkerConfig `yaml:"task_event_worker"`
// UserLeaderboardWorker 控制用户送礼榜单对 wallet_outbox 送礼事实的 Redis 投影。 // UserLeaderboardWorker 控制用户送礼榜单对 wallet_outbox 送礼事实的 Redis 投影。
UserLeaderboardWorker UserLeaderboardWorkerConfig `yaml:"user_leaderboard_worker"` UserLeaderboardWorker UserLeaderboardWorkerConfig `yaml:"user_leaderboard_worker"`
// WheelRewardWorker 补偿抽奖事务已经确定、但同步钱包发奖未完成的 WheelRewardSettlement outbox。
WheelRewardWorker WheelRewardWorkerConfig `yaml:"wheel_reward_worker"`
// TencentIM 是 activity-service 发送全局/区域播报群消息的 REST 配置。 // TencentIM 是 activity-service 发送全局/区域播报群消息的 REST 配置。
TencentIM TencentIMConfig `yaml:"tencent_im"` TencentIM TencentIMConfig `yaml:"tencent_im"`
// Broadcast 控制 IM 播报群 reconciler、outbox worker 和贵重礼物阈值。 // Broadcast 控制 IM 播报群 reconciler、outbox worker 和贵重礼物阈值。
@ -120,6 +122,17 @@ type UserLeaderboardWorkerConfig struct {
KeyPrefix string `yaml:"key_prefix"` KeyPrefix string `yaml:"key_prefix"`
} }
// WheelRewardWorkerConfig 控制转盘发奖 outbox 的租户分区、锁和重试边界。
// AppCodes 必须显式覆盖线上业务 Appworker 每轮逐 App claim禁止空 context 默认只处理 lalu。
type WheelRewardWorkerConfig struct {
Enabled bool `yaml:"enabled"`
AppCodes []string `yaml:"app_codes"`
PollInterval time.Duration `yaml:"poll_interval"`
BatchSize int `yaml:"batch_size"`
LockTTL time.Duration `yaml:"lock_ttl"`
MaxRetry int `yaml:"max_retry"`
}
// TencentIMConfig 描述 activity-service 调用腾讯云 IM REST API 的配置。 // TencentIMConfig 描述 activity-service 调用腾讯云 IM REST API 的配置。
type TencentIMConfig struct { type TencentIMConfig struct {
Enabled bool `yaml:"enabled"` Enabled bool `yaml:"enabled"`
@ -266,6 +279,14 @@ func Default() Config {
UserLeaderboardWorker: UserLeaderboardWorkerConfig{ UserLeaderboardWorker: UserLeaderboardWorkerConfig{
Enabled: false, Enabled: false,
}, },
WheelRewardWorker: WheelRewardWorkerConfig{
Enabled: true,
AppCodes: []string{"lalu", "fami", "huwaa"},
PollInterval: time.Second,
BatchSize: 50,
LockTTL: 30 * time.Second,
MaxRetry: 16,
},
MessageActionConfirmWorker: MessageActionConfirmWorkerConfig{ MessageActionConfirmWorker: MessageActionConfirmWorkerConfig{
Enabled: true, Enabled: true,
OutboxPollInterval: time.Second, OutboxPollInterval: time.Second,
@ -447,6 +468,22 @@ func Load(path string) (Config, error) {
} }
cfg.UserLeaderboardWorker.RedisAddr = strings.TrimSpace(cfg.UserLeaderboardWorker.RedisAddr) cfg.UserLeaderboardWorker.RedisAddr = strings.TrimSpace(cfg.UserLeaderboardWorker.RedisAddr)
cfg.UserLeaderboardWorker.KeyPrefix = strings.TrimSpace(cfg.UserLeaderboardWorker.KeyPrefix) cfg.UserLeaderboardWorker.KeyPrefix = strings.TrimSpace(cfg.UserLeaderboardWorker.KeyPrefix)
cfg.WheelRewardWorker.AppCodes = normalizeAppCodes(cfg.WheelRewardWorker.AppCodes)
if len(cfg.WheelRewardWorker.AppCodes) == 0 {
cfg.WheelRewardWorker.AppCodes = append([]string(nil), Default().WheelRewardWorker.AppCodes...)
}
if cfg.WheelRewardWorker.PollInterval <= 0 {
cfg.WheelRewardWorker.PollInterval = time.Second
}
if cfg.WheelRewardWorker.BatchSize <= 0 {
cfg.WheelRewardWorker.BatchSize = 50
}
if cfg.WheelRewardWorker.LockTTL <= 0 {
cfg.WheelRewardWorker.LockTTL = 30 * time.Second
}
if cfg.WheelRewardWorker.MaxRetry <= 0 {
cfg.WheelRewardWorker.MaxRetry = 16
}
rocketMQ, err := normalizeRocketMQConfig(cfg.RocketMQ) rocketMQ, err := normalizeRocketMQConfig(cfg.RocketMQ)
if err != nil { if err != nil {
return Config{}, err return Config{}, err

View File

@ -27,6 +27,9 @@ func TestLoadLocalEnablesOutboxMQConsumers(t *testing.T) {
if cfg.UserLeaderboardWorker.RedisAddr != "127.0.0.1:13379" || cfg.UserLeaderboardWorker.KeyPrefix != "activity:user_leaderboard" { if cfg.UserLeaderboardWorker.RedisAddr != "127.0.0.1:13379" || cfg.UserLeaderboardWorker.KeyPrefix != "activity:user_leaderboard" {
t.Fatalf("local user leaderboard worker redis config mismatch: %+v", cfg.UserLeaderboardWorker) t.Fatalf("local user leaderboard worker redis config mismatch: %+v", cfg.UserLeaderboardWorker)
} }
if !cfg.WheelRewardWorker.Enabled || len(cfg.WheelRewardWorker.AppCodes) != 3 || cfg.WheelRewardWorker.AppCodes[1] != "fami" || cfg.WheelRewardWorker.BatchSize <= 0 || cfg.WheelRewardWorker.LockTTL <= 0 {
t.Fatalf("local wheel reward compensation must cover every app with bounded claim settings: %+v", cfg.WheelRewardWorker)
}
if cfg.RocketMQ.WalletOutbox.InviteActivityConsumerGroup == "" || cfg.RocketMQ.WalletOutbox.UserLeaderboardConsumerGroup == "" || cfg.RocketMQ.WalletOutbox.VIPRebateNoticeConsumerGroup == "" || cfg.RocketMQ.UserOutbox.InviteActivityConsumerGroup == "" { if cfg.RocketMQ.WalletOutbox.InviteActivityConsumerGroup == "" || cfg.RocketMQ.WalletOutbox.UserLeaderboardConsumerGroup == "" || cfg.RocketMQ.WalletOutbox.VIPRebateNoticeConsumerGroup == "" || cfg.RocketMQ.UserOutbox.InviteActivityConsumerGroup == "" {
t.Fatalf("local config must configure invite activity consumer groups: wallet=%+v user=%+v", cfg.RocketMQ.WalletOutbox, cfg.RocketMQ.UserOutbox) t.Fatalf("local config must configure invite activity consumer groups: wallet=%+v user=%+v", cfg.RocketMQ.WalletOutbox, cfg.RocketMQ.UserOutbox)
} }
@ -58,6 +61,9 @@ func TestLoadTencentExampleEnablesRoomOutboxMQ(t *testing.T) {
if !cfg.UserLeaderboardWorker.Enabled || cfg.UserLeaderboardWorker.RedisAddr == "" || cfg.UserLeaderboardWorker.RedisPassword == "" || cfg.RocketMQ.WalletOutbox.UserLeaderboardConsumerGroup == "" { if !cfg.UserLeaderboardWorker.Enabled || cfg.UserLeaderboardWorker.RedisAddr == "" || cfg.UserLeaderboardWorker.RedisPassword == "" || cfg.RocketMQ.WalletOutbox.UserLeaderboardConsumerGroup == "" {
t.Fatalf("tencent example must configure user leaderboard worker: worker=%+v wallet=%+v", cfg.UserLeaderboardWorker, cfg.RocketMQ.WalletOutbox) t.Fatalf("tencent example must configure user leaderboard worker: worker=%+v wallet=%+v", cfg.UserLeaderboardWorker, cfg.RocketMQ.WalletOutbox)
} }
if !cfg.WheelRewardWorker.Enabled || len(cfg.WheelRewardWorker.AppCodes) != 3 || cfg.WheelRewardWorker.AppCodes[2] != "huwaa" || cfg.WheelRewardWorker.MaxRetry != 16 {
t.Fatalf("tencent example must configure multi-app wheel reward compensation: %+v", cfg.WheelRewardWorker)
}
} }
func TestLoadRejectsLuckyGiftConsumerWithoutBroadcastDelivery(t *testing.T) { func TestLoadRejectsLuckyGiftConsumerWithoutBroadcastDelivery(t *testing.T) {

View File

@ -13,6 +13,12 @@ const (
RewardTypeDress = "dress" RewardTypeDress = "dress"
EventTypeWheelRewardSettlement = "WheelRewardSettlement" EventTypeWheelRewardSettlement = "WheelRewardSettlement"
OutboxStatusPending = "pending"
OutboxStatusDelivering = "delivering"
OutboxStatusRetryable = "retryable"
OutboxStatusDelivered = "delivered"
OutboxStatusFailed = "failed"
) )
// RuleConfig 是转盘配置的不可变版本快照。 // RuleConfig 是转盘配置的不可变版本快照。
@ -113,3 +119,19 @@ type DrawQuery struct {
Page int32 Page int32
PageSize int32 PageSize int32
} }
// RewardOutboxRecord 是转盘发奖补偿 worker 的持久化工作项。
// Payload 始终保留抽奖事务写入时的不可变奖品快照;补偿时不能回读当前奖档配置,否则运营改配置后会改变历史应发权益。
type RewardOutboxRecord struct {
AppCode string
OutboxID string
Payload []byte
Status string
RetryCount int
NextRetryAtMS int64
LockedBy string
LockUntilMS int64
LastError string
CreatedAtMS int64
UpdatedAtMS int64
}

View File

@ -221,6 +221,11 @@ func (s *Service) HandleRoomEvent(ctx context.Context, envelope *roomeventsv1.Ev
} }
eventCtx := appcode.WithContext(ctx, envelope.GetAppCode()) eventCtx := appcode.WithContext(ctx, envelope.GetAppCode())
ownerUserID, err := s.fetchRoomOwner(eventCtx, roomID) ownerUserID, err := s.fetchRoomOwner(eventCtx, roomID)
if xerr.IsCode(err, xerr.NotFound) {
// RoomGiftSent 已经是 room-service 提交过的历史事实;若消费时房间已被删除,当前 owner 已不可恢复,
// 该事件对“开业房主流水”只能收敛为 skipped。这里只吞精确 NotFound网络、超时和内部错误仍返回 MQ 重试,避免把临时依赖故障误判成业务终态。
return 0, nil
}
if err != nil || ownerUserID <= 0 { if err != nil || ownerUserID <= 0 {
return 0, err return 0, err
} }

View File

@ -6,6 +6,8 @@ import (
"time" "time"
"google.golang.org/grpc" "google.golang.org/grpc"
"google.golang.org/grpc/codes"
"google.golang.org/grpc/status"
"google.golang.org/protobuf/proto" "google.golang.org/protobuf/proto"
roomeventsv1 "hyapp.local/api/proto/events/room/v1" roomeventsv1 "hyapp.local/api/proto/events/room/v1"
roomv1 "hyapp.local/api/proto/room/v1" roomv1 "hyapp.local/api/proto/room/v1"
@ -109,6 +111,46 @@ func TestHandleRoomEventScoresApplicantRoomOwner(t *testing.T) {
} }
} }
func TestHandleRoomEventTreatsDeletedRoomAsTerminalSkip(t *testing.T) {
repo := &agencyOpeningFakeRepo{consumeStatus: domain.EventStatusConsumed}
room := &agencyOpeningFakeRoomClient{err: status.Error(codes.NotFound, "room not found")}
svc := New(repo, nil, nil, room)
body, err := proto.Marshal(&roomeventsv1.RoomGiftSent{SenderUserId: 9, TargetUserId: 88, GiftValue: 900})
if err != nil {
t.Fatalf("marshal gift: %v", err)
}
count, err := svc.HandleRoomEvent(context.Background(), &roomeventsv1.EventEnvelope{
EventId: "deleted-room-gift", RoomId: "deleted-room", EventType: "RoomGiftSent", AppCode: "lalu", Body: body,
})
if err != nil || count != 0 {
t.Fatalf("deleted room must be terminal skipped, count=%d err=%v", count, err)
}
if repo.consumed.EventID != "" {
t.Fatalf("deleted room must not create agency turnover: %+v", repo.consumed)
}
}
func TestHandleRoomEventKeepsTransientRoomFailureRetryable(t *testing.T) {
repo := &agencyOpeningFakeRepo{consumeStatus: domain.EventStatusConsumed}
room := &agencyOpeningFakeRoomClient{err: status.Error(codes.Unavailable, "room service unavailable")}
svc := New(repo, nil, nil, room)
body, err := proto.Marshal(&roomeventsv1.RoomGiftSent{SenderUserId: 9, TargetUserId: 88, GiftValue: 900})
if err != nil {
t.Fatalf("marshal gift: %v", err)
}
count, err := svc.HandleRoomEvent(context.Background(), &roomeventsv1.EventEnvelope{
EventId: "transient-room-gift", RoomId: "room-1", EventType: "RoomGiftSent", AppCode: "lalu", Body: body,
})
if err == nil || status.Code(err) != codes.Unavailable || count != 0 {
t.Fatalf("transient room failure must remain retryable, count=%d err=%v", count, err)
}
if repo.consumed.EventID != "" {
t.Fatalf("transient room failure must not create partial projection: %+v", repo.consumed)
}
}
type agencyOpeningFakeRepo struct { type agencyOpeningFakeRepo struct {
currentCycle domain.Cycle currentCycle domain.Cycle
hasCurrent bool hasCurrent bool
@ -179,10 +221,14 @@ func (s agencyOpeningFakeAgencySource) ResolveOwnerAgency(context.Context, int64
type agencyOpeningFakeRoomClient struct { type agencyOpeningFakeRoomClient struct {
ownerUserID int64 ownerUserID int64
roomID string roomID string
err error
} }
func (c *agencyOpeningFakeRoomClient) AdminGetRoom(_ context.Context, req *roomv1.AdminGetRoomRequest, _ ...grpc.CallOption) (*roomv1.AdminGetRoomResponse, error) { func (c *agencyOpeningFakeRoomClient) AdminGetRoom(_ context.Context, req *roomv1.AdminGetRoomRequest, _ ...grpc.CallOption) (*roomv1.AdminGetRoomResponse, error) {
c.roomID = req.GetRoomId() c.roomID = req.GetRoomId()
if c.err != nil {
return nil, c.err
}
return &roomv1.AdminGetRoomResponse{ return &roomv1.AdminGetRoomResponse{
Room: &roomv1.AdminRoomListItem{OwnerUserId: c.ownerUserID}, Room: &roomv1.AdminRoomListItem{OwnerUserId: c.ownerUserID},
}, nil }, nil

View File

@ -2,6 +2,7 @@ package wheel
import ( import (
"context" "context"
"crypto/sha256"
"encoding/json" "encoding/json"
"fmt" "fmt"
"log/slog" "log/slog"
@ -24,6 +25,9 @@ type Repository interface {
ListWheelDraws(ctx context.Context, query domain.DrawQuery) ([]domain.DrawResult, int64, error) ListWheelDraws(ctx context.Context, query domain.DrawQuery) ([]domain.DrawResult, int64, error)
GetWheelDrawSummary(ctx context.Context, query domain.DrawQuery) (domain.DrawSummary, error) GetWheelDrawSummary(ctx context.Context, query domain.DrawQuery) (domain.DrawSummary, error)
MarkWheelDrawsGranted(ctx context.Context, appCode string, drawIDs []string, transactionID string, nowMS int64) error MarkWheelDrawsGranted(ctx context.Context, appCode string, drawIDs []string, transactionID string, nowMS int64) error
ClaimWheelRewardOutbox(ctx context.Context, workerID string, nowMS int64, lockUntilMS int64, limit int, maxRetry int) ([]domain.RewardOutboxRecord, error)
MarkWheelRewardOutboxDelivered(ctx context.Context, outboxID string, workerID string, drawIDs []string, transactionID string, nowMS int64) error
MarkWheelRewardOutboxFailed(ctx context.Context, outboxID string, workerID string, nowMS int64, nextRetryAtMS int64, maxRetry int, lastError string) error
} }
type WalletClient interface { type WalletClient interface {
@ -128,26 +132,41 @@ func (s *Service) fulfillWheelRewardFastPath(ctx context.Context, cmd domain.Dra
if result.RewardStatus == domain.StatusGranted { if result.RewardStatus == domain.StatusGranted {
return result return result
} }
if s.wallet == nil { fulfilled, err := s.fulfillWheelReward(ctx, cmd, result)
logx.Warn(ctx, "wheel_reward_wallet_not_configured", slog.String("draw_id", result.DrawID)) if err != nil {
// 抽奖结果已经在事务内确认,发放失败时不能回滚抽奖事实;保留 pending/outbox补偿 worker 会使用相同 command_id 继续核账和补发。
logx.Error(ctx, "wheel_reward_fast_path_failed", err,
slog.String("draw_id", result.DrawID),
slog.String("command_id", result.CommandID),
)
return result return result
} }
result = fulfilled
drawIDs := []string{result.DrawID}
if result.DrawID == "" {
drawIDs = append([]string(nil), result.DrawIDs...)
}
if err := s.repository.MarkWheelDrawsGranted(appcode.WithContext(context.WithoutCancel(ctx), appcode.FromContext(ctx)), appcode.FromContext(ctx), drawIDs, result.WalletTransactionID, s.now().UTC().UnixMilli()); err != nil {
logx.Error(ctx, "wheel_reward_fast_path_mark_granted_failed", err, slog.String("draw_id", result.DrawID), slog.String("wallet_transaction_id", result.WalletTransactionID))
}
return result
}
// fulfillWheelReward 调用 wallet owner 的幂等账务命令完成一组奖品发放。
// CreditWheelReward/GrantResource 在钱包事务内都会先按稳定 command_id 和 request hash 查询既有交易或权益;因此 worker 在超时、进程崩溃或标记 delivered 失败后重放时,先得到原收据而不是再次入账。
func (s *Service) fulfillWheelReward(ctx context.Context, cmd domain.DrawCommand, result domain.DrawResult) (domain.DrawResult, error) {
if s.wallet == nil {
return result, xerr.New(xerr.Unavailable, "wheel reward wallet client is not configured")
}
rewards := wheelResultRewards(result) rewards := wheelResultRewards(result)
if len(rewards) == 0 { if len(rewards) == 0 {
return result return result, xerr.New(xerr.InvalidArgument, "wheel reward snapshot is empty")
} }
receipts := make([]string, 0, len(rewards)) receipts := make([]string, 0, len(rewards))
for index, reward := range rewards { for index, reward := range rewards {
receipt, coinBalance, err := s.fulfillWheelRewardItem(ctx, cmd, result, reward, index, len(rewards)) receipt, coinBalance, err := s.fulfillWheelRewardItem(ctx, cmd, result, reward, index, len(rewards))
if err != nil { if err != nil {
// 抽奖结果已经在事务内确认,发放失败时不能回滚抽奖事实;保留 pending/outbox补偿 worker 或重试可以用同一 command_id 幂等补发。 return result, fmt.Errorf("fulfill reward item %d type=%s tier=%s: %w", index+1, reward.RewardType, reward.SelectedTierID, err)
logx.Error(ctx, "wheel_reward_fast_path_failed", err,
slog.String("draw_id", result.DrawID),
slog.String("command_id", result.CommandID),
slog.String("reward_type", reward.RewardType),
slog.String("selected_tier_id", reward.SelectedTierID),
)
return result
} }
if receipt != "" { if receipt != "" {
receipts = append(receipts, receipt) receipts = append(receipts, receipt)
@ -157,15 +176,20 @@ func (s *Service) fulfillWheelRewardFastPath(ctx context.Context, cmd domain.Dra
} }
} }
result.RewardStatus = domain.StatusGranted result.RewardStatus = domain.StatusGranted
result.WalletTransactionID = strings.Join(receipts, ",") result.WalletTransactionID = wheelRewardReceiptReference(receipts)
drawIDs := []string{result.DrawID} return result, nil
if result.DrawID == "" {
drawIDs = append([]string(nil), result.DrawIDs...)
} }
if err := s.repository.MarkWheelDrawsGranted(appcode.WithContext(context.WithoutCancel(ctx), appcode.FromContext(ctx)), appcode.FromContext(ctx), drawIDs, result.WalletTransactionID, s.now().UTC().UnixMilli()); err != nil {
logx.Error(ctx, "wheel_reward_fast_path_mark_granted_failed", err, slog.String("draw_id", result.DrawID), slog.String("wallet_transaction_id", result.WalletTransactionID)) func wheelRewardReceiptReference(receipts []string) string {
joined := strings.Join(receipts, ",")
const rewardTransactionIDMaxLength = 128
if len(joined) <= rewardTransactionIDMaxLength {
return joined
} }
return result // wheel_draw_records 的历史字段是 VARCHAR(128),而钱包 transaction/grant ID 各自约 68 字符;
// 多奖品直接逗号拼接会让钱包已经全部发放后仍无法标记 granted。超长集合保存稳定摘要逐项原收据仍由钱包 owner 按派生 command_id 可审计查询。
sum := sha256.Sum256([]byte(joined))
return fmt.Sprintf("batch:%d:%x", len(receipts), sum)
} }
func (s *Service) fulfillWheelRewardItem(ctx context.Context, cmd domain.DrawCommand, result domain.DrawResult, reward domain.DrawReward, index, total int) (string, int64, error) { func (s *Service) fulfillWheelRewardItem(ctx context.Context, cmd domain.DrawCommand, result domain.DrawResult, reward domain.DrawReward, index, total int) (string, int64, error) {

View File

@ -2,6 +2,8 @@ package wheel
import ( import (
"context" "context"
"errors"
"strings"
"testing" "testing"
"time" "time"
@ -96,11 +98,35 @@ func TestDrawFulfillsBatchCoinAndResourceRewards(t *testing.T) {
if result.CoinBalanceAfter != 9900 { if result.CoinBalanceAfter != 9900 {
t.Fatalf("coin balance mismatch: %d", result.CoinBalanceAfter) t.Fatalf("coin balance mismatch: %d", result.CoinBalanceAfter)
} }
if len(result.WalletTransactionID) > 128 {
t.Fatalf("batch reward receipt reference exceeds wheel_draw_records boundary: %d", len(result.WalletTransactionID))
}
}
func TestWheelRewardReceiptReferenceCompactsMultiRewardReceipts(t *testing.T) {
receipts := []string{
"wtx_aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa",
"rgr_bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb",
}
first := wheelRewardReceiptReference(receipts)
second := wheelRewardReceiptReference(receipts)
if first != second || !strings.HasPrefix(first, "batch:2:") || len(first) > 128 {
t.Fatalf("receipt reference must be stable and bounded: first=%q second=%q", first, second)
}
} }
type fakeWheelRepository struct { type fakeWheelRepository struct {
executeResult domain.DrawResult executeResult domain.DrawResult
markGrantedDrawIDs []string markGrantedDrawIDs []string
claimRecords []domain.RewardOutboxRecord
claimApps []string
deliveredOutboxID string
deliveredWorkerID string
deliveredDrawIDs []string
deliveredTransactionID string
deliverErr error
failedOutboxID string
failedReason string
} }
func (r *fakeWheelRepository) PublishWheelRuleConfig(context.Context, domain.RuleConfig, int64) (domain.RuleConfig, error) { func (r *fakeWheelRepository) PublishWheelRuleConfig(context.Context, domain.RuleConfig, int64) (domain.RuleConfig, error) {
@ -128,6 +154,27 @@ func (r *fakeWheelRepository) MarkWheelDrawsGranted(_ context.Context, _ string,
return nil return nil
} }
func (r *fakeWheelRepository) ClaimWheelRewardOutbox(ctx context.Context, _ string, _ int64, _ int64, _ int, _ int) ([]domain.RewardOutboxRecord, error) {
r.claimApps = append(r.claimApps, appcode.FromContext(ctx))
records := append([]domain.RewardOutboxRecord(nil), r.claimRecords...)
r.claimRecords = nil
return records, nil
}
func (r *fakeWheelRepository) MarkWheelRewardOutboxDelivered(_ context.Context, outboxID string, workerID string, drawIDs []string, transactionID string, _ int64) error {
r.deliveredOutboxID = outboxID
r.deliveredWorkerID = workerID
r.deliveredDrawIDs = append([]string(nil), drawIDs...)
r.deliveredTransactionID = transactionID
return r.deliverErr
}
func (r *fakeWheelRepository) MarkWheelRewardOutboxFailed(_ context.Context, outboxID string, _ string, _ int64, _ int64, _ int, lastError string) error {
r.failedOutboxID = outboxID
r.failedReason = lastError
return nil
}
type fakeWheelWallet struct { type fakeWheelWallet struct {
coinCredits []*walletv1.CreditWheelRewardRequest coinCredits []*walletv1.CreditWheelRewardRequest
resourceGrants []*walletv1.GrantResourceRequest resourceGrants []*walletv1.GrantResourceRequest
@ -146,3 +193,85 @@ func (w *fakeWheelWallet) GrantResource(_ context.Context, req *walletv1.GrantRe
w.resourceGrants = append(w.resourceGrants, req) w.resourceGrants = append(w.resourceGrants, req)
return &walletv1.ResourceGrantResponse{Grant: &walletv1.ResourceGrant{GrantId: "grant-" + req.GetCommandId()}}, nil return &walletv1.ResourceGrantResponse{Grant: &walletv1.ResourceGrant{GrantId: "grant-" + req.GetCommandId()}}, nil
} }
func TestProcessPendingRewardsReplaysStableWalletCommandAndMarksDelivered(t *testing.T) {
repository := &fakeWheelRepository{claimRecords: []domain.RewardOutboxRecord{{
AppCode: "fami",
OutboxID: "wheel_reward_wheel_draw_replay",
// 线上历史 payload 的 rewards 使用 Go 默认字段名;补偿解码必须兼容该不可变快照,不能改读当前奖档。
Payload: []byte(`{"app_code":"fami","draw_id":"wheel_draw_replay","draw_ids":["wheel_draw_replay"],"command_id":"draw-command","user_id":1001,"wheel_id":"classic","rewards":[{"SelectedTierID":"tier-1","RewardType":"coin","RewardCoins":200}],"visible_region_id":9}`),
}}}
wallet := &fakeWheelWallet{coinBalanceAfter: 1200}
service := New(repository, WithWallet(wallet))
service.SetClock(func() time.Time { return time.UnixMilli(1700000000000).UTC() })
result, err := service.ProcessPendingRewards(context.Background(), RewardWorkerOptions{
WorkerID: "activity-1:wheel-reward",
AppCodes: []string{"fami"},
})
if err != nil {
t.Fatalf("ProcessPendingRewards returned error: %v", err)
}
if result.Claimed != 1 || result.Delivered != 1 || result.Failed != 0 {
t.Fatalf("unexpected batch result: %+v", result)
}
if len(wallet.coinCredits) != 1 {
t.Fatalf("wallet credit count = %d, want 1", len(wallet.coinCredits))
}
request := wallet.coinCredits[0]
if request.GetCommandId() != "wheel_reward:wheel_draw_replay" || request.GetAppCode() != "fami" || request.GetTargetUserId() != 1001 || request.GetAmount() != 200 {
t.Fatalf("wallet reconciliation command mismatch: %+v", request)
}
if repository.deliveredOutboxID != "wheel_reward_wheel_draw_replay" || repository.deliveredWorkerID != "activity-1:wheel-reward" || repository.deliveredTransactionID != "tx-wheel_reward:wheel_draw_replay" {
t.Fatalf("delivered marker mismatch: %+v", repository)
}
if len(repository.deliveredDrawIDs) != 1 || repository.deliveredDrawIDs[0] != "wheel_draw_replay" {
t.Fatalf("delivered draw ids mismatch: %+v", repository.deliveredDrawIDs)
}
if len(repository.claimApps) != 1 || repository.claimApps[0] != "fami" {
t.Fatalf("claim must use explicit app partition, got %+v", repository.claimApps)
}
}
func TestProcessPendingRewardsRejectsCrossAppPayloadBeforeWallet(t *testing.T) {
repository := &fakeWheelRepository{claimRecords: []domain.RewardOutboxRecord{{
AppCode: "fami",
OutboxID: "wheel_reward_cross_app",
Payload: []byte(`{"app_code":"huwaa","draw_id":"cross_app","command_id":"cmd","user_id":1001,"wheel_id":"classic","reward_type":"coin","reward_coins":200,"selected_tier_id":"tier-1"}`),
}}}
wallet := &fakeWheelWallet{}
service := New(repository, WithWallet(wallet))
service.SetClock(func() time.Time { return time.UnixMilli(1700000000000).UTC() })
result, err := service.ProcessPendingRewards(context.Background(), RewardWorkerOptions{WorkerID: "worker", AppCodes: []string{"fami"}})
if err == nil || result.Claimed != 1 || result.Failed != 1 || result.Delivered != 0 {
t.Fatalf("cross-app payload must fail closed, result=%+v err=%v", result, err)
}
if len(wallet.coinCredits) != 0 || len(wallet.resourceGrants) != 0 {
t.Fatalf("cross-app payload must not reach wallet: coin=%d resource=%d", len(wallet.coinCredits), len(wallet.resourceGrants))
}
if repository.failedOutboxID != "wheel_reward_cross_app" || repository.failedReason == "" {
t.Fatalf("failed outbox marker missing: %+v", repository)
}
}
func TestProcessPendingRewardsLeavesCompletedWalletCommandForLeaseRecoveryWhenDeliveryMarkFails(t *testing.T) {
repository := &fakeWheelRepository{
deliverErr: errors.New("mysql unavailable"),
claimRecords: []domain.RewardOutboxRecord{{
AppCode: "lalu",
OutboxID: "wheel_reward_mark_failure",
Payload: []byte(`{"app_code":"lalu","draw_id":"mark_failure","command_id":"cmd","user_id":1001,"wheel_id":"classic","reward_type":"coin","reward_coins":20,"selected_tier_id":"tier-1"}`),
}},
}
wallet := &fakeWheelWallet{}
service := New(repository, WithWallet(wallet))
result, err := service.ProcessPendingRewards(context.Background(), RewardWorkerOptions{WorkerID: "worker", AppCodes: []string{"lalu"}})
if err == nil || result.Failed != 1 || len(wallet.coinCredits) != 1 {
t.Fatalf("delivery mark failure mismatch: result=%+v wallet=%d err=%v", result, len(wallet.coinCredits), err)
}
if repository.failedOutboxID != "" {
t.Fatalf("lost lease/delivery failure must stay delivering for safe idempotent reclaim, got failed marker %q", repository.failedOutboxID)
}
}

View File

@ -0,0 +1,218 @@
package wheel
import (
"context"
"encoding/json"
"errors"
"fmt"
"log/slog"
"strings"
"time"
"hyapp/pkg/appcode"
"hyapp/pkg/logx"
"hyapp/pkg/xerr"
domain "hyapp/services/activity-service/internal/domain/wheel"
)
// RewardWorkerOptions 定义补偿 worker 的租户、锁和重试边界。
type RewardWorkerOptions struct {
WorkerID string
AppCodes []string
PollInterval time.Duration
BatchSize int
LockTTL time.Duration
MaxRetry int
}
// RewardWorkerBatchResult 只统计本轮 claim 后的终态;失败项已经释放为 retryable/failed不会静默丢失。
type RewardWorkerBatchResult struct {
Claimed int
Delivered int
Failed int
}
type wheelRewardOutboxPayload struct {
AppCode string `json:"app_code"`
DrawID string `json:"draw_id"`
DrawIDs []string `json:"draw_ids"`
CommandID string `json:"command_id"`
UserID int64 `json:"user_id"`
WheelID string `json:"wheel_id"`
SelectedTierID string `json:"selected_tier_id"`
RewardType string `json:"reward_type"`
RewardID string `json:"reward_id"`
RewardCount int64 `json:"reward_count"`
RewardCoins int64 `json:"reward_coins"`
Rewards []domain.DrawReward `json:"rewards"`
VisibleRegionID int64 `json:"visible_region_id"`
}
// RunRewardWorker 持续逐 App 分区补偿 WheelRewardSettlement。
// AppCode 必须写进 context 后再 claim 和调用钱包,避免后台根 context 被 appcode 默认值悄悄收敛成 lalu。
func (s *Service) RunRewardWorker(ctx context.Context, options RewardWorkerOptions) {
options = normalizeRewardWorkerOptions(options)
ticker := time.NewTicker(options.PollInterval)
defer ticker.Stop()
for {
if _, err := s.ProcessPendingRewards(ctx, options); err != nil && ctx.Err() == nil {
logx.Error(ctx, "wheel_reward_outbox_batch_failed", err, slog.String("worker_id", options.WorkerID))
}
select {
case <-ctx.Done():
return
case <-ticker.C:
}
}
}
// ProcessPendingRewards 执行一轮有界补偿。
// 钱包 RPC 使用 draw_id 派生的稳定 command_id钱包 owner 会在同一事务内先查既有交易/权益及 request hash命中则返回原收据未命中才创建因此 activity 只在收齐所有收据后标 delivered。
func (s *Service) ProcessPendingRewards(ctx context.Context, options RewardWorkerOptions) (RewardWorkerBatchResult, error) {
if err := s.requireRepository(); err != nil {
return RewardWorkerBatchResult{}, err
}
options = normalizeRewardWorkerOptions(options)
var result RewardWorkerBatchResult
var batchErr error
for _, appCodeValue := range options.AppCodes {
if ctx.Err() != nil {
return result, errors.Join(batchErr, ctx.Err())
}
appCtx := appcode.WithContext(ctx, appCodeValue)
nowMS := s.now().UTC().UnixMilli()
records, err := s.repository.ClaimWheelRewardOutbox(appCtx, options.WorkerID, nowMS, nowMS+options.LockTTL.Milliseconds(), options.BatchSize, options.MaxRetry)
if err != nil {
batchErr = errors.Join(batchErr, fmt.Errorf("claim app=%s: %w", appCodeValue, err))
continue
}
result.Claimed += len(records)
for _, record := range records {
if err := s.processRewardOutboxRecord(appCtx, record, options); err != nil {
result.Failed++
batchErr = errors.Join(batchErr, fmt.Errorf("process app=%s outbox=%s: %w", appCodeValue, record.OutboxID, err))
continue
}
result.Delivered++
}
}
return result, batchErr
}
func (s *Service) processRewardOutboxRecord(ctx context.Context, record domain.RewardOutboxRecord, options RewardWorkerOptions) error {
payload, err := decodeWheelRewardOutboxPayload(record)
if err == nil {
var fulfilled domain.DrawResult
fulfilled, err = s.fulfillWheelReward(ctx, domain.DrawCommand{
CommandID: payload.CommandID,
WheelID: payload.WheelID,
UserID: payload.UserID,
VisibleRegionID: payload.VisibleRegionID,
}, domain.DrawResult{
DrawID: payload.DrawID,
DrawIDs: payload.DrawIDs,
CommandID: payload.CommandID,
WheelID: payload.WheelID,
SelectedTierID: payload.SelectedTierID,
RewardType: payload.RewardType,
RewardID: payload.RewardID,
RewardCount: payload.RewardCount,
RewardCoins: payload.RewardCoins,
RewardStatus: domain.StatusPending,
Rewards: payload.Rewards,
})
if err == nil {
drawIDs := payload.DrawIDs
if len(drawIDs) == 0 {
drawIDs = []string{payload.DrawID}
}
// draw、统计和 outbox 的最终提交带 worker fencing token若 lease 已被接管,这里会回滚而不是覆盖新 worker。
return s.repository.MarkWheelRewardOutboxDelivered(ctx, record.OutboxID, options.WorkerID, drawIDs, fulfilled.WalletTransactionID, s.now().UTC().UnixMilli())
}
}
nowMS := s.now().UTC().UnixMilli()
nextRetryAtMS := nowMS + wheelRewardRetryDelay(record.RetryCount+1).Milliseconds()
markErr := s.repository.MarkWheelRewardOutboxFailed(ctx, record.OutboxID, options.WorkerID, nowMS, nextRetryAtMS, options.MaxRetry, err.Error())
if markErr != nil {
return errors.Join(err, fmt.Errorf("mark wheel reward retryable: %w", markErr))
}
return err
}
func decodeWheelRewardOutboxPayload(record domain.RewardOutboxRecord) (wheelRewardOutboxPayload, error) {
var payload wheelRewardOutboxPayload
if err := json.Unmarshal(record.Payload, &payload); err != nil {
return payload, fmt.Errorf("decode wheel reward payload: %w", err)
}
recordApp := appcode.Normalize(record.AppCode)
payloadApp := appcode.Normalize(payload.AppCode)
if payload.AppCode != "" && payloadApp != recordApp {
return payload, xerr.New(xerr.Conflict, "wheel reward payload app_code does not match outbox partition")
}
payload.AppCode = recordApp
payload.DrawID = strings.TrimSpace(payload.DrawID)
payload.CommandID = strings.TrimSpace(payload.CommandID)
payload.WheelID = strings.TrimSpace(payload.WheelID)
if payload.DrawID == "" || payload.CommandID == "" || payload.WheelID == "" || payload.UserID <= 0 {
return payload, xerr.New(xerr.InvalidArgument, "wheel reward payload identity is incomplete")
}
if len(payload.DrawIDs) == 0 {
payload.DrawIDs = []string{payload.DrawID}
}
for index := range payload.DrawIDs {
payload.DrawIDs[index] = strings.TrimSpace(payload.DrawIDs[index])
if payload.DrawIDs[index] == "" {
return payload, xerr.New(xerr.InvalidArgument, "wheel reward payload contains empty draw_id")
}
}
return payload, nil
}
func normalizeRewardWorkerOptions(options RewardWorkerOptions) RewardWorkerOptions {
options.WorkerID = strings.TrimSpace(options.WorkerID)
if options.WorkerID == "" {
options.WorkerID = "activity-wheel-reward"
}
seen := make(map[string]struct{}, len(options.AppCodes))
appCodes := make([]string, 0, len(options.AppCodes))
for _, value := range options.AppCodes {
value = appcode.Normalize(value)
if _, exists := seen[value]; exists {
continue
}
seen[value] = struct{}{}
appCodes = append(appCodes, value)
}
if len(appCodes) == 0 {
appCodes = []string{"lalu", "fami", "huwaa"}
}
options.AppCodes = appCodes
if options.PollInterval <= 0 {
options.PollInterval = time.Second
}
if options.BatchSize <= 0 {
options.BatchSize = 50
}
if options.LockTTL <= 0 {
options.LockTTL = 30 * time.Second
}
if options.MaxRetry <= 0 {
options.MaxRetry = 16
}
return options
}
func wheelRewardRetryDelay(attempt int) time.Duration {
if attempt < 1 {
attempt = 1
}
if attempt > 7 {
attempt = 7
}
delay := time.Second * time.Duration(1<<(attempt-1))
if delay > time.Minute {
return time.Minute
}
return delay
}

View File

@ -0,0 +1,212 @@
package mysql
import (
"context"
"strings"
"hyapp/pkg/appcode"
"hyapp/pkg/xerr"
domain "hyapp/services/activity-service/internal/domain/wheel"
)
// ClaimWheelRewardOutbox 抢占一个 App 分区内可补偿的转盘发奖事实。
// 行锁和 SKIP LOCKED 允许双节点并行;过期 delivering 可被接管,而仍持有效 lease 的记录不会被第二个节点重复处理。
func (r *Repository) ClaimWheelRewardOutbox(ctx context.Context, workerID string, nowMS int64, lockUntilMS int64, limit int, maxRetry int) ([]domain.RewardOutboxRecord, error) {
if r == nil || r.db == nil {
return nil, xerr.New(xerr.Unavailable, "mysql repository is not configured")
}
workerID = strings.TrimSpace(workerID)
if workerID == "" {
return nil, xerr.New(xerr.InvalidArgument, "wheel reward worker_id is required")
}
if limit <= 0 {
limit = 50
}
if maxRetry <= 0 {
maxRetry = 16
}
tx, err := r.db.BeginTx(ctx, nil)
if err != nil {
return nil, err
}
defer func() { _ = tx.Rollback() }()
appCode := appcode.FromContext(ctx)
rows, err := tx.QueryContext(ctx, `
SELECT app_code, outbox_id, payload, status, retry_count, next_retry_at_ms,
locked_by, lock_until_ms, last_error, created_at_ms, updated_at_ms
FROM activity_outbox
WHERE app_code = ?
AND event_type = ?
AND retry_count < ?
AND (
(status IN (?, ?) AND next_retry_at_ms <= ?)
OR (status = ? AND lock_until_ms <= ?)
)
ORDER BY created_at_ms, outbox_id
LIMIT ?
FOR UPDATE SKIP LOCKED`,
appCode,
domain.EventTypeWheelRewardSettlement,
maxRetry,
domain.OutboxStatusPending,
domain.OutboxStatusRetryable,
nowMS,
domain.OutboxStatusDelivering,
nowMS,
limit,
)
if err != nil {
return nil, err
}
records := make([]domain.RewardOutboxRecord, 0, limit)
for rows.Next() {
var record domain.RewardOutboxRecord
if err := rows.Scan(
&record.AppCode,
&record.OutboxID,
&record.Payload,
&record.Status,
&record.RetryCount,
&record.NextRetryAtMS,
&record.LockedBy,
&record.LockUntilMS,
&record.LastError,
&record.CreatedAtMS,
&record.UpdatedAtMS,
); err != nil {
_ = rows.Close()
return nil, err
}
records = append(records, record)
}
if err := rows.Close(); err != nil {
return nil, err
}
if err := rows.Err(); err != nil {
return nil, err
}
for index := range records {
result, err := tx.ExecContext(ctx, `
UPDATE activity_outbox
SET status = ?, locked_by = ?, lock_until_ms = ?, updated_at_ms = ?
WHERE app_code = ? AND outbox_id = ?`,
domain.OutboxStatusDelivering,
workerID,
lockUntilMS,
nowMS,
records[index].AppCode,
records[index].OutboxID,
)
if err != nil {
return nil, err
}
if affected, err := result.RowsAffected(); err != nil || affected != 1 {
if err != nil {
return nil, err
}
return nil, xerr.New(xerr.Conflict, "wheel reward outbox claim lost")
}
records[index].Status = domain.OutboxStatusDelivering
records[index].LockedBy = workerID
records[index].LockUntilMS = lockUntilMS
}
if err := tx.Commit(); err != nil {
return nil, err
}
return records, nil
}
// MarkWheelRewardOutboxDelivered 把钱包返回的既有或新建收据与 draw 状态、统计和 outbox 在一个 MySQL 事务中提交。
// locked_by 是 fencing tokenworker lease 过期并被接管后,旧 worker 即使晚返回也不能覆盖新 owner 的处理结果。
func (r *Repository) MarkWheelRewardOutboxDelivered(ctx context.Context, outboxID string, workerID string, drawIDs []string, transactionID string, nowMS int64) error {
if r == nil || r.db == nil {
return xerr.New(xerr.Unavailable, "mysql repository is not configured")
}
outboxID = strings.TrimSpace(outboxID)
workerID = strings.TrimSpace(workerID)
if outboxID == "" || workerID == "" {
return xerr.New(xerr.InvalidArgument, "wheel reward outbox delivery identity is incomplete")
}
tx, err := r.db.BeginTx(ctx, nil)
if err != nil {
return err
}
defer func() { _ = tx.Rollback() }()
appCode := appcode.FromContext(ctx)
if err := markWheelDrawsGrantedTx(ctx, tx, appCode, drawIDs, transactionID, nowMS, true); err != nil {
return err
}
result, err := tx.ExecContext(ctx, `
UPDATE activity_outbox
SET status = ?, locked_by = '', lock_until_ms = 0, last_error = '', updated_at_ms = ?
WHERE app_code = ? AND outbox_id = ? AND event_type = ? AND status = ? AND locked_by = ?`,
domain.OutboxStatusDelivered,
nowMS,
appCode,
outboxID,
domain.EventTypeWheelRewardSettlement,
domain.OutboxStatusDelivering,
workerID,
)
if err != nil {
return err
}
if affected, err := result.RowsAffected(); err != nil || affected != 1 {
if err != nil {
return err
}
return xerr.New(xerr.Conflict, "wheel reward outbox delivery lock lost")
}
return tx.Commit()
}
// MarkWheelRewardOutboxFailed 释放失败 lease并以有界重试次数决定 retryable 或 failed。
func (r *Repository) MarkWheelRewardOutboxFailed(ctx context.Context, outboxID string, workerID string, nowMS int64, nextRetryAtMS int64, maxRetry int, lastError string) error {
if r == nil || r.db == nil {
return xerr.New(xerr.Unavailable, "mysql repository is not configured")
}
if maxRetry <= 0 {
maxRetry = 16
}
result, err := r.db.ExecContext(ctx, `
UPDATE activity_outbox
SET status = CASE WHEN retry_count + 1 >= ? THEN ? ELSE ? END,
next_retry_at_ms = CASE WHEN retry_count + 1 >= ? THEN 0 ELSE ? END,
retry_count = retry_count + 1,
locked_by = '',
lock_until_ms = 0,
last_error = ?,
updated_at_ms = ?
WHERE app_code = ? AND outbox_id = ? AND event_type = ? AND status = ? AND locked_by = ?`,
maxRetry,
domain.OutboxStatusFailed,
domain.OutboxStatusRetryable,
maxRetry,
nextRetryAtMS,
truncateWheelOutboxError(lastError),
nowMS,
appcode.FromContext(ctx),
strings.TrimSpace(outboxID),
domain.EventTypeWheelRewardSettlement,
domain.OutboxStatusDelivering,
strings.TrimSpace(workerID),
)
if err != nil {
return err
}
if affected, err := result.RowsAffected(); err != nil || affected != 1 {
if err != nil {
return err
}
return xerr.New(xerr.Conflict, "wheel reward outbox failure lock lost")
}
return nil
}
func truncateWheelOutboxError(message string) string {
message = strings.TrimSpace(message)
if len(message) <= 512 {
return message
}
return message[:512]
}

View File

@ -0,0 +1,79 @@
package mysql
import (
"context"
"testing"
"hyapp/internal/testutil/mysqlschema"
"hyapp/pkg/appcode"
domain "hyapp/services/activity-service/internal/domain/wheel"
)
func TestWheelRewardOutboxClaimAndDeliveryAreLeaseFencedAndAtomic(t *testing.T) {
schema := mysqlschema.New(t, mysqlschema.Config{
EnvVar: "ACTIVITY_SERVICE_MYSQL_TEST_DSN",
InitDBPath: mysqlschema.InitDBPath(t, mysqlschema.CallerFile(t, 1), "..", "..", "..", "deploy", "mysql", "initdb", "001_activity_service.sql"),
DatabasePrefix: "hyapp_activity_wheel_outbox_test",
})
repository, err := Open(context.Background(), schema.DSN)
if err != nil {
t.Fatalf("open repository: %v", err)
}
t.Cleanup(func() { _ = repository.Close() })
ctx := appcode.WithContext(context.Background(), "fami")
const (
drawID = "wheel_draw_outbox_test"
outboxID = "wheel_reward_wheel_draw_outbox_test"
)
if _, err := schema.DB.ExecContext(ctx, `
INSERT INTO wheel_draw_stats (
app_code, wheel_id, total_draws, unique_users, total_spent_coins, total_rtp_value_coins,
pending_draws, granted_draws, failed_draws, created_at_ms, updated_at_ms
) VALUES ('fami', 'classic', 1, 1, 100, 20, 1, 0, 0, 1000, 1000)`); err != nil {
t.Fatalf("seed wheel stats: %v", err)
}
if _, err := schema.DB.ExecContext(ctx, `
INSERT INTO wheel_draw_records (
app_code, draw_id, command_id, user_id, device_id, visible_region_id, wheel_id, coin_spent,
rule_version, rtp_window_index, selected_tier_id, reward_type, reward_id, reward_count,
reward_coins, rtp_value_coins, candidate_tiers_json, rtp_snapshot_json, prize_metadata_json,
reward_status, paid_at_ms, created_at_ms, updated_at_ms
) VALUES ('fami', ?, 'draw-command', 1001, 'device', 9, 'classic', 100,
1, 1, 'tier-1', 'coin', '', 1, 20, 20, '{}', '{}', '{}', 'pending', 1000, 1000, 1000)`, drawID); err != nil {
t.Fatalf("seed wheel draw: %v", err)
}
if _, err := schema.DB.ExecContext(ctx, `
INSERT INTO activity_outbox (
app_code, outbox_id, event_type, payload, status, retry_count, next_retry_at_ms,
locked_by, lock_until_ms, last_error, created_at_ms, updated_at_ms
) VALUES ('fami', ?, ?, '{}', 'pending', 0, 1000, '', 0, '', 1000, 1000)`, outboxID, domain.EventTypeWheelRewardSettlement); err != nil {
t.Fatalf("seed wheel outbox: %v", err)
}
records, err := repository.ClaimWheelRewardOutbox(ctx, "worker-1", 1000, 31000, 10, 16)
if err != nil || len(records) != 1 || records[0].LockedBy != "worker-1" || records[0].Status != domain.OutboxStatusDelivering {
t.Fatalf("claim mismatch: records=%+v err=%v", records, err)
}
secondClaim, err := repository.ClaimWheelRewardOutbox(ctx, "worker-2", 2000, 32000, 10, 16)
if err != nil || len(secondClaim) != 0 {
t.Fatalf("active lease must block second worker: records=%+v err=%v", secondClaim, err)
}
if err := repository.MarkWheelRewardOutboxDelivered(ctx, outboxID, "worker-1", []string{drawID}, "wtx-existing", 3000); err != nil {
t.Fatalf("mark delivered: %v", err)
}
var drawStatus, receiptRef, outboxStatus, lockedBy string
var pending, granted int64
if err := schema.DB.QueryRowContext(ctx, `SELECT reward_status, reward_transaction_id FROM wheel_draw_records WHERE app_code = 'fami' AND draw_id = ?`, drawID).Scan(&drawStatus, &receiptRef); err != nil {
t.Fatalf("read draw state: %v", err)
}
if err := schema.DB.QueryRowContext(ctx, `SELECT pending_draws, granted_draws FROM wheel_draw_stats WHERE app_code = 'fami' AND wheel_id = 'classic'`).Scan(&pending, &granted); err != nil {
t.Fatalf("read wheel stats: %v", err)
}
if err := schema.DB.QueryRowContext(ctx, `SELECT status, locked_by FROM activity_outbox WHERE app_code = 'fami' AND outbox_id = ?`, outboxID).Scan(&outboxStatus, &lockedBy); err != nil {
t.Fatalf("read outbox state: %v", err)
}
if drawStatus != domain.StatusGranted || receiptRef != "wtx-existing" || pending != 0 || granted != 1 || outboxStatus != domain.OutboxStatusDelivered || lockedBy != "" {
t.Fatalf("delivery state mismatch: draw=%s receipt=%s pending=%d granted=%d outbox=%s lock=%q", drawStatus, receiptRef, pending, granted, outboxStatus, lockedBy)
}
}

View File

@ -317,11 +317,38 @@ func (r *Repository) MarkWheelDrawsGranted(ctx context.Context, appCode string,
return err return err
} }
defer func() { _ = tx.Rollback() }() defer func() { _ = tx.Rollback() }()
if err := markWheelDrawsGrantedTx(ctx, tx, appCode, drawIDs, transactionID, nowMS, false); err != nil {
return err
}
for _, drawID := range drawIDs { for _, drawID := range drawIDs {
drawID = strings.TrimSpace(drawID) drawID = strings.TrimSpace(drawID)
if drawID == "" { if drawID == "" {
continue continue
} }
if _, err := tx.ExecContext(ctx, `
UPDATE activity_outbox
SET status = 'delivered', locked_by = '', lock_until_ms = 0, last_error = '', updated_at_ms = ?
WHERE app_code = ? AND outbox_id = ?`,
nowMS, appCode, "wheel_reward_"+drawID,
); err != nil {
return err
}
}
return tx.Commit()
}
// markWheelDrawsGrantedTx 原子搬移 draw 和汇总状态requireDraw 用于补偿 worker 防止损坏 payload 把 outbox 错标 delivered。
func markWheelDrawsGrantedTx(ctx context.Context, tx *sql.Tx, appCode string, drawIDs []string, transactionID string, nowMS int64, requireDraw bool) error {
seen := make(map[string]struct{}, len(drawIDs))
for _, drawID := range drawIDs {
drawID = strings.TrimSpace(drawID)
if drawID == "" {
continue
}
if _, exists := seen[drawID]; exists {
continue
}
seen[drawID] = struct{}{}
var wheelID, oldStatus string var wheelID, oldStatus string
if err := tx.QueryRowContext(ctx, ` if err := tx.QueryRowContext(ctx, `
SELECT wheel_id, reward_status SELECT wheel_id, reward_status
@ -331,6 +358,9 @@ func (r *Repository) MarkWheelDrawsGranted(ctx context.Context, appCode string,
appCode, drawID, appCode, drawID,
).Scan(&wheelID, &oldStatus); err != nil { ).Scan(&wheelID, &oldStatus); err != nil {
if errors.Is(err, sql.ErrNoRows) { if errors.Is(err, sql.ErrNoRows) {
if requireDraw {
return xerr.New(xerr.NotFound, "wheel draw record not found")
}
continue continue
} }
return err return err
@ -356,16 +386,11 @@ func (r *Repository) MarkWheelDrawsGranted(ctx context.Context, appCode string,
return err return err
} }
} }
if _, err := tx.ExecContext(ctx, `
UPDATE activity_outbox
SET status = 'delivered', locked_by = '', lock_until_ms = 0, last_error = '', updated_at_ms = ?
WHERE app_code = ? AND outbox_id = ?`,
nowMS, appCode, "wheel_reward_"+drawID,
); err != nil {
return err
} }
if requireDraw && len(seen) == 0 {
return xerr.New(xerr.InvalidArgument, "wheel draw_ids are empty")
} }
return tx.Commit() return nil
} }
func (r *Repository) executeSingleWheelDraw(cmd domain.DrawCommand, commandID string, unitSpent int64, config domain.RuleConfig, window *wheelRTPWindow, pool *wheelPool, nowMS int64) (domain.DrawResult, int64, error) { func (r *Repository) executeSingleWheelDraw(cmd domain.DrawCommand, commandID string, unitSpent int64, config domain.RuleConfig, window *wheelRTPWindow, pool *wheelPool, nowMS int64) (domain.DrawResult, int64, error) {

View File

@ -34,6 +34,7 @@ rocketmq:
producer_group: "hyapp-game-outbox-producer" producer_group: "hyapp-game-outbox-producer"
outbox_worker: outbox_worker:
enabled: true enabled: true
app_codes: []
poll_interval: "1s" poll_interval: "1s"
batch_size: 100 batch_size: 100
publish_timeout: "3s" publish_timeout: "3s"

View File

@ -34,6 +34,8 @@ rocketmq:
producer_group: "hyapp-game-outbox-producer" producer_group: "hyapp-game-outbox-producer"
outbox_worker: outbox_worker:
enabled: true enabled: true
# 空值自动发现全部 App历史积压回放时先设 ["lalu", "fami"],核对后再加入 "huwaa"。
app_codes: []
poll_interval: "60s" poll_interval: "60s"
batch_size: 10 batch_size: 10
publish_timeout: "3s" publish_timeout: "3s"

View File

@ -34,6 +34,7 @@ rocketmq:
producer_group: "hyapp-game-outbox-producer" producer_group: "hyapp-game-outbox-producer"
outbox_worker: outbox_worker:
enabled: true enabled: true
app_codes: []
poll_interval: "1s" poll_interval: "1s"
batch_size: 100 batch_size: 100
publish_timeout: "3s" publish_timeout: "3s"

View File

@ -11,10 +11,12 @@ import (
"google.golang.org/grpc" "google.golang.org/grpc"
"google.golang.org/grpc/credentials/insecure" "google.golang.org/grpc/credentials/insecure"
gamev1 "hyapp.local/api/proto/game/v1" gamev1 "hyapp.local/api/proto/game/v1"
"hyapp/pkg/appcode"
"hyapp/pkg/gamemq" "hyapp/pkg/gamemq"
"hyapp/pkg/grpchealth" "hyapp/pkg/grpchealth"
"hyapp/pkg/healthhttp" "hyapp/pkg/healthhttp"
"hyapp/pkg/logx" "hyapp/pkg/logx"
"hyapp/pkg/outboxpartition"
"hyapp/pkg/rocketmqx" "hyapp/pkg/rocketmqx"
servicegrpc "hyapp/pkg/servicekit/grpcserver" servicegrpc "hyapp/pkg/servicekit/grpcserver"
servicehealth "hyapp/pkg/servicekit/health" servicehealth "hyapp/pkg/servicekit/health"
@ -40,6 +42,8 @@ type App struct {
activityConn *grpc.ClientConn activityConn *grpc.ClientConn
robotConn *grpc.ClientConn robotConn *grpc.ClientConn
outboxProducer *rocketmqx.Producer outboxProducer *rocketmqx.Producer
// outboxPartitions 缓存真实 App 列表并让双实例从不同轮转起点公平 claim。
outboxPartitions *outboxpartition.Partitions
outboxCtx context.Context outboxCtx context.Context
outboxCancel context.CancelFunc outboxCancel context.CancelFunc
outboxWG sync.WaitGroup outboxWG sync.WaitGroup
@ -154,7 +158,7 @@ func New(cfg config.Config) (*App, error) {
} }
outboxCtx, outboxCancel := context.WithCancel(context.Background()) outboxCtx, outboxCancel := context.WithCancel(context.Background())
return &App{server: server, listener: listener, health: health, healthHTTP: healthHTTP, repo: repo, walletConn: walletConn, userConn: userConn, activityConn: activityConn, robotConn: robotConn, outboxProducer: outboxProducer, outboxCtx: outboxCtx, outboxCancel: outboxCancel, cfg: cfg}, nil return &App{server: server, listener: listener, health: health, healthHTTP: healthHTTP, repo: repo, walletConn: walletConn, userConn: userConn, activityConn: activityConn, robotConn: robotConn, outboxProducer: outboxProducer, outboxPartitions: outboxpartition.New(cfg.OutboxWorker.AppCodes, outboxpartition.DefaultDiscoveryRefreshInterval), outboxCtx: outboxCtx, outboxCancel: outboxCancel, cfg: cfg}, nil
} }
func (a *App) Run() error { func (a *App) Run() error {
@ -229,7 +233,17 @@ func (a *App) runOutboxWorker() {
func (a *App) processOutboxBatch() (int, error) { func (a *App) processOutboxBatch() (int, error) {
workerID := "game-outbox-" + a.cfg.NodeID workerID := "game-outbox-" + a.cfg.NodeID
records, err := a.repo.ClaimPendingGameOutbox(a.outboxCtx, workerID, a.cfg.OutboxWorker.BatchSize) appCodes, discoveryErr := a.outboxPartitions.Resolve(a.outboxCtx, a.repo.ListGameOutboxAppCodes)
if discoveryErr != nil {
if len(appCodes) == 0 {
return 0, discoveryErr
}
// 已缓存 App 继续发布;低频刷新失败不应把整个 game outbox worker 停掉。
logx.Warn(a.outboxCtx, "game_outbox_app_discovery_stale", slog.String("worker_id", workerID), slog.String("error", discoveryErr.Error()))
}
records, err := outboxpartition.ClaimFair(a.outboxCtx, a.outboxPartitions.Order(appCodes), a.cfg.OutboxWorker.BatchSize, func(claimCtx context.Context, appCode string, limit int) ([]mysqlstorage.GameOutboxRecord, error) {
return a.repo.ClaimPendingGameOutbox(appcode.WithContext(claimCtx, appCode), workerID, limit)
})
if err != nil { if err != nil {
return 0, err return 0, err
} }

View File

@ -9,6 +9,7 @@ import (
"hyapp/pkg/configx" "hyapp/pkg/configx"
"hyapp/pkg/logx" "hyapp/pkg/logx"
"hyapp/pkg/outboxpartition"
"hyapp/pkg/tencentim" "hyapp/pkg/tencentim"
) )
@ -96,6 +97,8 @@ type GameOutboxConfig struct {
type OutboxWorkerConfig struct { type OutboxWorkerConfig struct {
Enabled bool `yaml:"enabled"` Enabled bool `yaml:"enabled"`
// AppCodes 是可投递租户 allowlist空值自动发现非空用于分阶段回放历史积压。
AppCodes []string `yaml:"app_codes"`
PollInterval time.Duration `yaml:"poll_interval"` PollInterval time.Duration `yaml:"poll_interval"`
BatchSize int `yaml:"batch_size"` BatchSize int `yaml:"batch_size"`
PublishTimeout time.Duration `yaml:"publish_timeout"` PublishTimeout time.Duration `yaml:"publish_timeout"`
@ -210,6 +213,7 @@ func (cfg *Config) Normalize() error {
if cfg.OutboxWorker.PublishTimeout <= 0 { if cfg.OutboxWorker.PublishTimeout <= 0 {
cfg.OutboxWorker.PublishTimeout = 3 * time.Second cfg.OutboxWorker.PublishTimeout = 3 * time.Second
} }
cfg.OutboxWorker.AppCodes = outboxpartition.Normalize(cfg.OutboxWorker.AppCodes, false)
if cfg.OutboxWorker.Enabled && !cfg.RocketMQ.GameOutbox.Enabled { if cfg.OutboxWorker.Enabled && !cfg.RocketMQ.GameOutbox.Enabled {
return errors.New("outbox_worker requires rocketmq.game_outbox.enabled") return errors.New("outbox_worker requires rocketmq.game_outbox.enabled")
} }

View File

@ -2,6 +2,27 @@ package config
import "testing" import "testing"
func TestOutboxWorkerAppCodesNormalizeForStagedReplay(t *testing.T) {
cfg := Default()
cfg.OutboxWorker.AppCodes = []string{" LALU ", "fami", "HUWAA", "fami"}
if err := cfg.Normalize(); err != nil {
t.Fatalf("normalize staged outbox config: %v", err)
}
if len(cfg.OutboxWorker.AppCodes) != 3 || cfg.OutboxWorker.AppCodes[0] != "lalu" || cfg.OutboxWorker.AppCodes[1] != "fami" || cfg.OutboxWorker.AppCodes[2] != "huwaa" {
t.Fatalf("normalized outbox app_codes = %v", cfg.OutboxWorker.AppCodes)
}
}
func TestLocalOutboxWorkerEmptyAppCodesUsesDiscovery(t *testing.T) {
cfg, err := Load("../../configs/config.yaml")
if err != nil {
t.Fatalf("load local config: %v", err)
}
if len(cfg.OutboxWorker.AppCodes) != 0 {
t.Fatalf("local empty app_codes must retain automatic discovery: %v", cfg.OutboxWorker.AppCodes)
}
}
func TestLoadConfigsKeepActivityServiceAddr(t *testing.T) { func TestLoadConfigsKeepActivityServiceAddr(t *testing.T) {
tests := []struct { tests := []struct {
name string name string

View File

@ -0,0 +1,30 @@
package mysql
import (
"context"
"regexp"
"testing"
"github.com/DATA-DOG/go-sqlmock"
)
func TestListGameOutboxAppCodesForcesPrimaryPrefixIndex(t *testing.T) {
db, mock, err := sqlmock.New()
if err != nil {
t.Fatalf("create sqlmock: %v", err)
}
defer db.Close()
mock.ExpectQuery(regexp.QuoteMeta(`SELECT DISTINCT app_code FROM game_outbox FORCE INDEX (PRIMARY) ORDER BY app_code`)).
WillReturnRows(sqlmock.NewRows([]string{"app_code"}).AddRow("fami").AddRow("huwaa").AddRow("lalu"))
apps, err := (&Repository{db: db}).ListGameOutboxAppCodes(context.Background())
if err != nil {
t.Fatalf("list game outbox apps: %v", err)
}
if len(apps) != 3 || apps[0] != "fami" || apps[1] != "huwaa" || apps[2] != "lalu" {
t.Fatalf("game outbox apps mismatch: %v", apps)
}
if err := mock.ExpectationsWereMet(); err != nil {
t.Fatalf("game app discovery query mismatch: %v", err)
}
}

View File

@ -1152,6 +1152,29 @@ type GameOutboxRecord struct {
LockUntilMS int64 LockUntilMS int64
} }
// ListGameOutboxAppCodes enumerates real tenant keys from the game owner table.
// PRIMARY is ordered by (app_code,event_id), allowing cached discovery without
// weakening the app_code prefix used by every SKIP LOCKED claim transaction.
func (r *Repository) ListGameOutboxAppCodes(ctx context.Context) ([]string, error) {
if r == nil || r.db == nil {
return nil, xerr.New(xerr.Unavailable, "mysql repository is not configured")
}
rows, err := r.db.QueryContext(ctx, `SELECT DISTINCT app_code FROM game_outbox FORCE INDEX (PRIMARY) ORDER BY app_code`)
if err != nil {
return nil, err
}
defer rows.Close()
appCodes := make([]string, 0, 4)
for rows.Next() {
var appCode string
if err := rows.Scan(&appCode); err != nil {
return nil, err
}
appCodes = append(appCodes, appCode)
}
return appCodes, rows.Err()
}
// ClaimPendingGameOutbox locks pending game facts so one worker publishes them. // ClaimPendingGameOutbox locks pending game facts so one worker publishes them.
func (r *Repository) ClaimPendingGameOutbox(ctx context.Context, workerID string, batchSize int) ([]GameOutboxRecord, error) { func (r *Repository) ClaimPendingGameOutbox(ctx context.Context, workerID string, batchSize int) ([]GameOutboxRecord, error) {
if r == nil || r.db == nil { if r == nil || r.db == nil {
@ -1356,23 +1379,34 @@ func (r *Repository) queryUnlockedLevelEventClaimKeys(ctx context.Context, tx *s
} }
func (r *Repository) queryExpiredLevelEventClaimKeys(ctx context.Context, tx *sql.Tx, appCode string, nowMS int64, batchSize int) ([]levelEventClaimKey, error) { func (r *Repository) queryExpiredLevelEventClaimKeys(ctx context.Context, tx *sql.Tx, appCode string, nowMS int64, batchSize int) ([]levelEventClaimKey, error) {
// 过期锁接管仍使用 retry/lock 前缀缩小范围,但只锁主键,避免大 JSON 字段进入排序和慢日志。 // 过期接管只处理 runningpending/failed 已由 unlocked path 负责。拆开 legacy lock=0
return queryLevelEventClaimKeysByStatus(batchSize, func(status string) ([]levelEventClaimKey, error) { // 与正常过期锁可继续利用 (app_code,status,locked_until_ms,...) 前缀,同时明确
// 排除未来锁,避免并发 cron 抢走仍由其它 worker 处理的事实。
query := func(lockCondition string, args ...any) ([]levelEventClaimKey, error) {
queryArgs := []any{appCode, "running"}
queryArgs = append(queryArgs, args...)
queryArgs = append(queryArgs, nowMS, batchSize)
rows, err := tx.QueryContext(ctx, ` rows, err := tx.QueryContext(ctx, `
SELECT event_id, created_at_ms SELECT event_id, created_at_ms
FROM game_level_event_outbox FORCE INDEX (idx_game_level_event_claim) FROM game_level_event_outbox FORCE INDEX (idx_game_level_event_claim)
WHERE app_code = ? AND status = ? AND next_retry_at_ms <= ? WHERE app_code = ? AND status = ? AND `+lockCondition+` AND next_retry_at_ms <= ?
AND locked_until_ms > 0 AND locked_until_ms <= ?
ORDER BY created_at_ms ASC, event_id ASC ORDER BY created_at_ms ASC, event_id ASC
LIMIT ? FOR UPDATE SKIP LOCKED`, LIMIT ? FOR UPDATE SKIP LOCKED`, queryArgs...)
appCode, status, nowMS, nowMS, batchSize,
)
if err != nil { if err != nil {
return nil, err return nil, err
} }
defer rows.Close() defer rows.Close()
return scanLevelEventClaimKeys(rows) return scanLevelEventClaimKeys(rows)
}) }
legacyUnlocked, err := query("locked_until_ms = 0")
if err != nil {
return nil, err
}
expired, err := query("locked_until_ms > 0 AND locked_until_ms <= ?", nowMS)
if err != nil {
return nil, err
}
return mergeLevelEventClaimKeys(batchSize, legacyUnlocked, expired), nil
} }
func queryLevelEventClaimKeysByStatus(batchSize int, query func(status string) ([]levelEventClaimKey, error)) ([]levelEventClaimKey, error) { func queryLevelEventClaimKeysByStatus(batchSize int, query func(status string) ([]levelEventClaimKey, error)) ([]levelEventClaimKey, error) {

View File

@ -483,6 +483,65 @@ func TestClaimPendingLevelEventsPreservesCreatedOrderAcrossStatuses(t *testing.T
} }
} }
func TestClaimPendingLevelEventsReclaimsLegacyAndExpiredRunningLocks(t *testing.T) {
caller := mysqlschema.CallerFile(t, 1)
schema := mysqlschema.New(t, mysqlschema.Config{
InitDBPath: mysqlschema.InitDBPath(t, caller, "../../../deploy/mysql/initdb/001_game_service.sql"),
DatabasePrefix: "hy_game_test",
})
ctx := appcode.WithContext(context.Background(), "lalu")
repo, err := Open(ctx, schema.DSN)
if err != nil {
t.Fatalf("open repository failed: %v", err)
}
t.Cleanup(func() { _ = repo.Close() })
insertRunning := func(eventID string, lockedUntilMS int64, createdAtMS int64) {
t.Helper()
if _, err := repo.db.ExecContext(ctx, `
INSERT INTO game_level_event_outbox (
app_code, event_id, order_id, user_id, game_id, provider_order_id,
provider_round_id, coin_amount, wallet_transaction_id, payload_json,
status, attempt_count, next_retry_at_ms, locked_by, locked_until_ms,
failure_reason, occurred_at_ms, created_at_ms, updated_at_ms
) VALUES ('lalu', ?, ?, 42, 'game_1', ?, '', 100, 'wallet_tx_1', '{}',
'running', 1, 0, 'old-worker', ?, '', ?, ?, ?)`,
eventID, "order_"+eventID, "provider_"+eventID, lockedUntilMS, createdAtMS, createdAtMS, createdAtMS,
); err != nil {
t.Fatalf("insert running level event %s failed: %v", eventID, err)
}
}
insertRunning("legacy_zero_lock", 0, 1000)
insertRunning("expired_lock", 2500, 1100)
insertRunning("future_lock", 4000, 1200)
events, err := repo.ClaimPendingLevelEvents(ctx, "replacement-worker", 3000, 30*time.Second, 10)
if err != nil {
t.Fatalf("claim expired running events failed: %v", err)
}
if len(events) != 2 || events[0].EventID != "legacy_zero_lock" || events[1].EventID != "expired_lock" {
t.Fatalf("reclaimed running events mismatch: %+v", events)
}
for _, event := range events {
if event.Status != "running" {
t.Fatalf("reclaimed event must expose its pre-claim running state: %+v", event)
}
}
var lockedBy string
var lockedUntilMS int64
if err := repo.db.QueryRowContext(ctx, `
SELECT locked_by, locked_until_ms FROM game_level_event_outbox
WHERE app_code = 'lalu' AND event_id = 'future_lock'`,
).Scan(&lockedBy, &lockedUntilMS); err != nil {
t.Fatalf("query future running lock failed: %v", err)
}
if lockedBy != "old-worker" || lockedUntilMS != 4000 {
t.Fatalf("future lock was stolen: locked_by=%q locked_until_ms=%d", lockedBy, lockedUntilMS)
}
}
func TestLevelEventClaimKeyQueryUsesOrderIndexWithoutFilesort(t *testing.T) { func TestLevelEventClaimKeyQueryUsesOrderIndexWithoutFilesort(t *testing.T) {
caller := mysqlschema.CallerFile(t, 1) caller := mysqlschema.CallerFile(t, 1)
schema := mysqlschema.New(t, mysqlschema.Config{ schema := mysqlschema.New(t, mysqlschema.Config{

View File

@ -1,7 +1,6 @@
package appapi package appapi
import ( import (
"bytes"
"encoding/json" "encoding/json"
"net/http" "net/http"
"strings" "strings"
@ -9,16 +8,12 @@ import (
userv1 "hyapp.local/api/proto/user/v1" userv1 "hyapp.local/api/proto/user/v1"
"hyapp/pkg/appcode" "hyapp/pkg/appcode"
"hyapp/pkg/apptracking"
"hyapp/services/gateway-service/internal/auth" "hyapp/services/gateway-service/internal/auth"
"hyapp/services/gateway-service/internal/client" "hyapp/services/gateway-service/internal/client"
"hyapp/services/gateway-service/internal/transport/http/httpkit" "hyapp/services/gateway-service/internal/transport/http/httpkit"
) )
const (
appEventMaxBatchSize = 50
appEventMaxPropertiesBytes = 4096
)
type appEventsRequest struct { type appEventsRequest struct {
Events []appEventRequest `json:"events"` Events []appEventRequest `json:"events"`
} }
@ -44,26 +39,17 @@ type appEventRequest struct {
} }
func (h *Handler) reportAppEvents(writer http.ResponseWriter, request *http.Request) { func (h *Handler) reportAppEvents(writer http.ResponseWriter, request *http.Request) {
if h.statisticsClient == nil {
// App 埋点事实由 statistics-service 持久化gateway 未注入写入边界时不能假成功,否则客户端会丢掉重试机会。
httpkit.WriteError(writer, request, http.StatusBadGateway, httpkit.CodeUpstreamError, "upstream service error")
return
}
var body appEventsRequest var body appEventsRequest
if !httpkit.Decode(writer, request, &body) { if !httpkit.Decode(writer, request, &body) {
return return
} }
if len(body.Events) == 0 || len(body.Events) > appEventMaxBatchSize { if !apptracking.ValidBatchSize(len(body.Events)) {
httpkit.WriteError(writer, request, http.StatusBadRequest, httpkit.CodeInvalidArgument, "invalid argument") httpkit.WriteError(writer, request, http.StatusBadRequest, httpkit.CodeInvalidArgument, "invalid argument")
return return
} }
nowMS := time.Now().UTC().UnixMilli() nowMS := time.Now().UTC().UnixMilli()
userID := auth.UserIDFromContext(request.Context()) userID := auth.UserIDFromContext(request.Context())
countryID, regionID, ok := h.appEventUserDimensions(writer, request, userID)
if !ok {
return
}
headerDeviceID := httpkit.FirstHeader(request, "X-Device-ID", "X-HY-Device-ID") headerDeviceID := httpkit.FirstHeader(request, "X-Device-ID", "X-HY-Device-ID")
headerPlatform := httpkit.FirstHeader(request, "X-App-Platform", "X-Platform") headerPlatform := httpkit.FirstHeader(request, "X-App-Platform", "X-Platform")
headerAppVersion := httpkit.FirstHeader(request, "X-App-Version", "X-Client-Version") headerAppVersion := httpkit.FirstHeader(request, "X-App-Version", "X-Client-Version")
@ -75,8 +61,6 @@ func (h *Handler) reportAppEvents(writer http.ResponseWriter, request *http.Requ
event, valid := appTrackingEventFromRequest(item, appTrackingEventDefaults{ event, valid := appTrackingEventFromRequest(item, appTrackingEventDefaults{
appCode: appcode.FromContext(request.Context()), appCode: appcode.FromContext(request.Context()),
userID: userID, userID: userID,
countryID: countryID,
regionID: regionID,
deviceID: headerDeviceID, deviceID: headerDeviceID,
sessionID: auth.SessionIDFromContext(request.Context()), sessionID: auth.SessionIDFromContext(request.Context()),
platform: headerPlatform, platform: headerPlatform,
@ -91,6 +75,21 @@ func (h *Handler) reportAppEvents(writer http.ResponseWriter, request *http.Requ
} }
events = append(events, event) events = append(events, event)
} }
if h.statisticsClient == nil {
// App 埋点事实由 statistics-service 持久化gateway 未注入写入边界时不能假成功,否则客户端会丢掉重试机会。
httpkit.WriteError(writer, request, http.StatusBadGateway, httpkit.CodeUpstreamError, "upstream service error")
return
}
// 先完成客户端可控字段的全部长度校验,再读取 user-service。这样超限
// 请求稳定返回 400不会制造无意义的下游调用更不会被包装成 502。
countryID, regionID, ok := h.appEventUserDimensions(writer, request, userID)
if !ok {
return
}
for index := range events {
events[index].CountryID = countryID
events[index].RegionID = regionID
}
result, err := h.statisticsClient.ReportAppTrackingEvents(request.Context(), events) result, err := h.statisticsClient.ReportAppTrackingEvents(request.Context(), events)
if err != nil { if err != nil {
@ -128,8 +127,6 @@ func (h *Handler) appEventUserDimensions(writer http.ResponseWriter, request *ht
type appTrackingEventDefaults struct { type appTrackingEventDefaults struct {
appCode string appCode string
userID int64 userID int64
countryID int64
regionID int64
deviceID string deviceID string
sessionID string sessionID string
platform string platform string
@ -140,16 +137,7 @@ type appTrackingEventDefaults struct {
} }
func appTrackingEventFromRequest(item appEventRequest, defaults appTrackingEventDefaults) (client.AppTrackingEvent, bool) { func appTrackingEventFromRequest(item appEventRequest, defaults appTrackingEventDefaults) (client.AppTrackingEvent, bool) {
item.EventID = strings.TrimSpace(item.EventID)
item.EventName = strings.TrimSpace(item.EventName)
if item.EventID == "" || item.EventName == "" {
return client.AppTrackingEvent{}, false
}
deviceID := firstTrimmed(item.DeviceID, defaults.deviceID) deviceID := firstTrimmed(item.DeviceID, defaults.deviceID)
if defaults.userID <= 0 && deviceID == "" {
// 匿名埋点没有服务端 user_id只能依赖稳定 device_id 做排障和后续粗粒度去重。
return client.AppTrackingEvent{}, false
}
properties, ok := normalizeAppEventProperties(item.Properties) properties, ok := normalizeAppEventProperties(item.Properties)
if !ok { if !ok {
return client.AppTrackingEvent{}, false return client.AppTrackingEvent{}, false
@ -158,14 +146,14 @@ func appTrackingEventFromRequest(item appEventRequest, defaults appTrackingEvent
if occurredAtMS <= 0 { if occurredAtMS <= 0 {
occurredAtMS = defaults.receivedAtMS occurredAtMS = defaults.receivedAtMS
} }
return client.AppTrackingEvent{ event := client.AppTrackingEvent{
AppCode: defaults.appCode, AppCode: defaults.appCode,
EventID: item.EventID, EventID: item.EventID,
EventName: item.EventName, EventName: item.EventName,
EventType: strings.TrimSpace(item.EventType), EventType: item.EventType,
Screen: strings.TrimSpace(item.Screen), Screen: item.Screen,
TargetType: strings.TrimSpace(item.TargetType), TargetType: item.TargetType,
TargetID: strings.TrimSpace(item.TargetID), TargetID: item.TargetID,
UserID: defaults.userID, UserID: defaults.userID,
DeviceID: deviceID, DeviceID: deviceID,
SessionID: firstTrimmed(defaults.sessionID, item.SessionID), SessionID: firstTrimmed(defaults.sessionID, item.SessionID),
@ -173,25 +161,60 @@ func appTrackingEventFromRequest(item appEventRequest, defaults appTrackingEvent
AppVersion: firstTrimmed(item.AppVersion, defaults.appVersion), AppVersion: firstTrimmed(item.AppVersion, defaults.appVersion),
Language: firstTrimmed(item.Language, defaults.language), Language: firstTrimmed(item.Language, defaults.language),
Timezone: firstTrimmed(item.Timezone, defaults.timezone), Timezone: firstTrimmed(item.Timezone, defaults.timezone),
CountryID: defaults.countryID,
RegionID: defaults.regionID,
DurationMS: item.DurationMS, DurationMS: item.DurationMS,
Success: item.Success, Success: item.Success,
ErrorCode: strings.TrimSpace(item.ErrorCode), ErrorCode: item.ErrorCode,
Properties: properties, Properties: properties,
OccurredAtMS: occurredAtMS, OccurredAtMS: occurredAtMS,
}, true }
if !normalizeAppTrackingEventText(&event) || event.EventID == "" || event.EventName == "" {
return client.AppTrackingEvent{}, false
}
if defaults.userID <= 0 && event.DeviceID == "" {
// 匿名埋点没有服务端 user_id只能依赖稳定 device_id 做排障和后续粗粒度去重。
return client.AppTrackingEvent{}, false
}
return event, true
} }
func normalizeAppEventProperties(raw json.RawMessage) (json.RawMessage, bool) { func normalizeAppEventProperties(raw json.RawMessage) (json.RawMessage, bool) {
trimmed := bytes.TrimSpace(raw) return apptracking.NormalizeProperties(raw)
if len(trimmed) == 0 || bytes.Equal(trimmed, []byte("null")) {
return nil, true
} }
if len(trimmed) > appEventMaxPropertiesBytes || !json.Valid(trimmed) {
return nil, false func normalizeAppTrackingEventText(event *client.AppTrackingEvent) bool {
if event == nil {
return false
} }
return json.RawMessage(trimmed), true // Every field uses the same byte limit as statistics persistence. Returning
// false is deliberate: identifiers and diagnostic error codes must be
// accepted intact or rejected, never silently truncated into another fact.
fields := []struct {
value *string
maxBytes int
}{
{value: &event.AppCode, maxBytes: apptracking.MaxAppCodeBytes},
{value: &event.EventID, maxBytes: apptracking.MaxEventIDBytes},
{value: &event.EventName, maxBytes: apptracking.MaxEventNameBytes},
{value: &event.EventType, maxBytes: apptracking.MaxEventTypeBytes},
{value: &event.Screen, maxBytes: apptracking.MaxScreenBytes},
{value: &event.TargetType, maxBytes: apptracking.MaxTargetTypeBytes},
{value: &event.TargetID, maxBytes: apptracking.MaxTargetIDBytes},
{value: &event.DeviceID, maxBytes: apptracking.MaxDeviceIDBytes},
{value: &event.SessionID, maxBytes: apptracking.MaxSessionIDBytes},
{value: &event.Platform, maxBytes: apptracking.MaxPlatformBytes},
{value: &event.AppVersion, maxBytes: apptracking.MaxAppVersionBytes},
{value: &event.Language, maxBytes: apptracking.MaxLanguageBytes},
{value: &event.Timezone, maxBytes: apptracking.MaxTimezoneBytes},
{value: &event.ErrorCode, maxBytes: apptracking.MaxErrorCodeBytes},
}
for _, field := range fields {
normalized, ok := apptracking.NormalizeText(*field.value, field.maxBytes)
if !ok {
return false
}
*field.value = normalized
}
return true
} }
func appEventRequestLanguage(request *http.Request) string { func appEventRequestLanguage(request *http.Request) string {

View File

@ -4625,6 +4625,36 @@ func TestReportAppEventsRejectsInvalidRequests(t *testing.T) {
} }
} }
func TestReportAppEventsRejects185ByteErrorCodeBeforeDownstream(t *testing.T) {
statisticsClient := &fakeStatisticsClient{}
handler := NewHandlerWithClients(&fakeRoomClient{}, nil, nil, &fakeUserProfileClient{})
handler.SetStatisticsClient(statisticsClient)
router := handler.Routes(auth.NewVerifier("secret"))
body, err := json.Marshal(map[string]any{
"events": []map[string]any{{
"event_id": "evt-error-code-too-long",
"event_name": "pay_failed",
"device_id": "dev-1",
"error_code": strings.Repeat("e", 185),
}},
})
if err != nil {
t.Fatalf("marshal app events request: %v", err)
}
request := httptest.NewRequest(http.MethodPost, "/api/v1/app/events", bytes.NewReader(body))
request.Header.Set("X-Request-ID", "req-app-events-error-code-limit")
recorder := httptest.NewRecorder()
router.ServeHTTP(recorder, request)
// Regression: statistics-service used to reject this after forwarding and
// gateway translated that downstream 400 into a misleading 502.
assertEnvelope(t, recorder, http.StatusBadRequest, httpkit.CodeInvalidArgument, "req-app-events-error-code-limit")
if len(statisticsClient.lastAppEvents) != 0 {
t.Fatalf("oversized error_code must not reach statistics-service: %+v", statisticsClient.lastAppEvents)
}
}
func TestReportAppEventsUpstreamFailure(t *testing.T) { func TestReportAppEventsUpstreamFailure(t *testing.T) {
statisticsClient := &fakeStatisticsClient{appErr: errors.New("statistics unavailable")} statisticsClient := &fakeStatisticsClient{appErr: errors.New("statistics unavailable")}
handler := NewHandlerWithClients(&fakeRoomClient{}, nil, nil, &fakeUserProfileClient{}) handler := NewHandlerWithClients(&fakeRoomClient{}, nil, nil, &fakeUserProfileClient{})

View File

@ -79,6 +79,12 @@ func logRoomGiftPanelStage(ctx context.Context, requestID string, roomID string,
slog.Bool("degraded", degraded), slog.Bool("degraded", degraded),
} }
switch { switch {
case roomGiftPanelStageCanceled(err):
// 首个必要阶段失败后 handler 会取消仍在运行的 sibling RPC这些 Canceled
// 只是请求内并发已被收敛,不是新的下游故障。保留 INFO 轨迹用于 request_id
// 关联,但不能为同一个主错误再制造多条 ERROR 告警。
attrs = append(attrs, slog.String("error", err.Error()), slog.Bool("canceled", true))
logx.Info(logCtx, "gateway_room_gift_panel_stage", attrs...)
case err != nil && degraded: case err != nil && degraded:
attrs = append(attrs, slog.String("error", err.Error())) attrs = append(attrs, slog.String("error", err.Error()))
logx.Warn(logCtx, "gateway_room_gift_panel_stage", attrs...) logx.Warn(logCtx, "gateway_room_gift_panel_stage", attrs...)
@ -91,6 +97,10 @@ func logRoomGiftPanelStage(ctx context.Context, requestID string, roomID string,
} }
} }
func roomGiftPanelStageCanceled(err error) bool {
return err != nil && (errors.Is(err, context.Canceled) || status.Code(err) == codes.Canceled)
}
// serveRoomGiftPanel 在单个明确预算内聚合 owner service 读模型。 // serveRoomGiftPanel 在单个明确预算内聚合 owner service 读模型。
// 第一阶段只并发互不依赖的 RPC依赖 visible_region_id 的礼物配置在快照返回后启动,避免猜测区域。 // 第一阶段只并发互不依赖的 RPC依赖 visible_region_id 的礼物配置在快照返回后启动,避免猜测区域。
func (h *Handler) serveRoomGiftPanel(writer http.ResponseWriter, request *http.Request) { func (h *Handler) serveRoomGiftPanel(writer http.ResponseWriter, request *http.Request) {

View File

@ -4,6 +4,7 @@ import (
"context" "context"
"encoding/json" "encoding/json"
"errors" "errors"
"fmt"
"net/http" "net/http"
"net/http/httptest" "net/http/httptest"
"sync/atomic" "sync/atomic"
@ -18,6 +19,9 @@ import (
"hyapp/services/gateway-service/internal/auth" "hyapp/services/gateway-service/internal/auth"
gatewayclient "hyapp/services/gateway-service/internal/client" gatewayclient "hyapp/services/gateway-service/internal/client"
"hyapp/services/gateway-service/internal/transport/http/httpkit" "hyapp/services/gateway-service/internal/transport/http/httpkit"
"google.golang.org/grpc/codes"
"google.golang.org/grpc/status"
) )
type giftPanelTestRoomQueryClient struct { type giftPanelTestRoomQueryClient struct {
@ -164,6 +168,21 @@ func TestRoomGiftPanelProductionBudgetsStayBelowFlutterTimeout(t *testing.T) {
} }
} }
func TestRoomGiftPanelStageCanceledRecognizesLocalAndGRPCCancellation(t *testing.T) {
for _, err := range []error{
context.Canceled,
fmt.Errorf("query interrupted: %w", context.Canceled),
status.Error(codes.Canceled, "context canceled"),
} {
if !roomGiftPanelStageCanceled(err) {
t.Fatalf("expected cancellation for %v", err)
}
}
if roomGiftPanelStageCanceled(status.Error(codes.DeadlineExceeded, "deadline exceeded")) {
t.Fatal("deadline exhaustion is a real latency failure and must remain error-level")
}
}
func TestRoomGiftPanelRunsIndependentStagesConcurrentlyAndSkipsUnusedEnrichment(t *testing.T) { func TestRoomGiftPanelRunsIndependentStagesConcurrentlyAndSkipsUnusedEnrichment(t *testing.T) {
phaseOne := newGiftPanelStageGate() phaseOne := newGiftPanelStageGate()
phaseTwo := newGiftPanelStageGate() phaseTwo := newGiftPanelStageGate()

View File

@ -0,0 +1,505 @@
// Command replay-deliveries replays an explicitly selected set of expired
// notice delivery leases from immutable owner outbox facts.
package main
import (
"context"
"database/sql"
"encoding/json"
"errors"
"flag"
"fmt"
"log"
"os"
"sort"
"strings"
"time"
roomeventsv1 "hyapp.local/api/proto/events/room/v1"
"hyapp/pkg/appcode"
"hyapp/pkg/tencentim"
"hyapp/pkg/walletmq"
"hyapp/services/notice-service/internal/config"
"hyapp/services/notice-service/internal/modules/roomnotice"
"hyapp/services/notice-service/internal/modules/walletnotice"
mysqlplatform "hyapp/services/notice-service/internal/platform/mysql"
)
const (
walletSource = "wallet_outbox"
roomSource = "room_outbox"
c2cChannel = "tencent_im_c2c"
)
type repeatedFlag []string
func (values *repeatedFlag) String() string { return strings.Join(*values, ",") }
func (values *repeatedFlag) Set(value string) error {
value = strings.TrimSpace(value)
if value == "" {
return errors.New("flag value cannot be empty")
}
*values = append(*values, value)
return nil
}
type deliverySelector struct {
SourceName string
AppCode string
EventID string
}
func (selector deliverySelector) String() string {
return selector.SourceName + ":" + selector.AppCode + ":" + selector.EventID
}
type deliverySelectorFlag []deliverySelector
func (values *deliverySelectorFlag) String() string {
parts := make([]string, 0, len(*values))
for _, selector := range *values {
parts = append(parts, selector.String())
}
return strings.Join(parts, ",")
}
func (values *deliverySelectorFlag) Set(value string) error {
parts := strings.SplitN(strings.TrimSpace(value), ":", 3)
if len(parts) != 3 {
return errors.New("delivery must use source_name:app_code:event_id")
}
selector := deliverySelector{
SourceName: strings.TrimSpace(parts[0]),
AppCode: strings.TrimSpace(parts[1]),
EventID: strings.TrimSpace(parts[2]),
}
if selector.SourceName != walletSource && selector.SourceName != roomSource {
return fmt.Errorf("unsupported delivery source %q", selector.SourceName)
}
if selector.AppCode == "" || selector.EventID == "" {
return errors.New("delivery app_code and event_id are required")
}
selector.AppCode = appcode.Normalize(selector.AppCode)
*values = append(*values, selector)
return nil
}
type deliveryCandidate struct {
Selector string `json:"selector"`
SourceName string `json:"source_name"`
AppCode string `json:"app_code"`
EventID string `json:"event_id"`
Status string `json:"status"`
RetryCount int `json:"retry_count"`
UpdatedAtMS int64 `json:"updated_at_ms"`
}
type replayItem struct {
Candidate deliveryCandidate
WalletMessage *walletmq.WalletOutboxMessage
RoomEnvelope *roomeventsv1.EventEnvelope
}
func main() {
configPath := flag.String("config", "services/notice-service/configs/config.yaml", "path to notice-service config")
execute := flag.Bool("execute", false, "publish selected due deliveries; default is a read-only preview")
limit := flag.Int("limit", 100, "maximum candidate count, 1..500")
var eventIDs repeatedFlag
var appCodes repeatedFlag
var deliveries deliverySelectorFlag
flag.Var(&eventIDs, "event-id", "preview filter for a source event ID; repeat for multiple events")
flag.Var(&appCodes, "app-code", "required App partition for indexed preview; repeat for multiple Apps")
flag.Var(&deliveries, "delivery", "exact source_name:app_code:event_id selector copied from preview; repeat for multiple deliveries")
flag.Parse()
if *limit <= 0 || *limit > 500 {
log.Fatal("limit must be between 1 and 500")
}
if len(deliveries) > 0 && (len(eventIDs) > 0 || len(appCodes) > 0) {
log.Fatal("--delivery cannot be combined with --event-id or --app-code")
}
if *execute && len(deliveries) == 0 {
// A write run is pinned to full primary-key selectors copied from preview;
// separate App/event filters can form an accidental cross product.
log.Fatal("--execute requires at least one exact --delivery from a preview")
}
if len(deliveries) > *limit {
log.Fatal("exact delivery count exceeds --limit")
}
if len(deliveries) == 0 && len(appCodes) == 0 {
// notice_delivery_events indexes start with source_name,app_code. Refusing
// an unpartitioned preview prevents a LIMIT query from scanning a 20+ GiB table.
log.Fatal("preview requires at least one --app-code or exact --delivery")
}
cfg, err := config.Load(*configPath)
if err != nil {
log.Fatal(err)
}
ctx, cancel := context.WithTimeout(context.Background(), 2*time.Minute)
defer cancel()
store, err := mysqlplatform.Open(ctx, cfg.MySQLDSN)
if err != nil {
log.Fatal(err)
}
defer func() { _ = store.Close() }()
nowMS := time.Now().UnixMilli()
var candidates []deliveryCandidate
if len(deliveries) > 0 {
candidates, err = loadExactDueDeliveries(ctx, store.DB, deliveries, nowMS)
} else {
candidates, err = listDueDeliveries(ctx, store.DB, eventIDs, appCodes, *limit, nowMS)
}
if err != nil {
log.Fatal(err)
}
if !*execute {
encoder := json.NewEncoder(os.Stdout)
for _, candidate := range candidates {
if err := encoder.Encode(candidate); err != nil {
log.Fatal(err)
}
}
fmt.Fprintf(os.Stderr, "previewed %d due delivery rows; no writes performed\n", len(candidates))
return
}
if !cfg.TencentIM.Enabled {
log.Fatal("tencent_im.enabled is required for replay")
}
walletRepo, err := walletnotice.NewMySQLRepository(store.DB, cfg.WalletDatabase)
if err != nil {
log.Fatal(err)
}
roomRepo, err := roomnotice.NewMySQLRepository(store.DB, cfg.RoomDatabase)
if err != nil {
log.Fatal(err)
}
items, err := loadReplayItems(ctx, candidates, walletRepo, roomRepo)
if err != nil {
// Preload every immutable owner fact before creating an IM client or
// publishing the first item. One missing/corrupt fact therefore aborts the
// exact set without producing a partial operator repair.
log.Fatal(err)
}
publisher, err := tencentim.NewRESTClient(cfg.TencentIM.RESTConfig())
if err != nil {
log.Fatal(err)
}
walletService := walletnotice.New(walletnotice.Config{NodeID: cfg.NodeID}, walletRepo, publisher)
roomService := roomnotice.New(roomnotice.Config{NodeID: cfg.NodeID}, roomRepo, publisher)
walletOptions := walletWorkerOptions(cfg)
roomOptions := roomWorkerOptions(cfg)
failures := 0
for _, item := range items {
candidate := item.Candidate
candidateCtx := appcode.WithContext(ctx, candidate.AppCode)
var replayErr error
switch candidate.SourceName {
case walletSource:
if item.WalletMessage == nil {
replayErr = errors.New("wallet owner snapshot is missing")
} else {
_, replayErr = walletService.ProcessWalletOutboxMessage(candidateCtx, *item.WalletMessage, walletOptions)
}
case roomSource:
if item.RoomEnvelope == nil {
replayErr = errors.New("room owner snapshot is missing")
} else {
_, replayErr = roomService.ProcessRoomOutboxEnvelope(candidateCtx, item.RoomEnvelope, roomOptions)
}
default:
replayErr = fmt.Errorf("unsupported source %q", candidate.SourceName)
}
if replayErr == nil {
replayErr = requireDelivered(ctx, store.DB, candidate)
}
if replayErr != nil {
failures++
fmt.Fprintf(os.Stderr, "replay failed source=%s app_code=%s event_id=%s: %v\n", candidate.SourceName, candidate.AppCode, candidate.EventID, replayErr)
continue
}
fmt.Printf("delivered source=%s app_code=%s event_id=%s\n", candidate.SourceName, candidate.AppCode, candidate.EventID)
}
if failures > 0 {
log.Fatalf("replay finished with %d failures out of %d selected rows", failures, len(candidates))
}
fmt.Printf("replay delivered %d selected rows\n", len(candidates))
}
func loadReplayItems(ctx context.Context, candidates []deliveryCandidate, walletRepo *walletnotice.MySQLRepository, roomRepo *roomnotice.MySQLRepository) ([]replayItem, error) {
items := make([]replayItem, 0, len(candidates))
for _, candidate := range candidates {
candidateCtx := appcode.WithContext(ctx, candidate.AppCode)
item := replayItem{Candidate: candidate}
switch candidate.SourceName {
case walletSource:
if walletRepo == nil {
return nil, errors.New("wallet owner repository is not configured")
}
message, err := walletRepo.LoadWalletOutboxMessage(candidateCtx, candidate.AppCode, candidate.EventID)
if err != nil {
return nil, fmt.Errorf("load owner snapshot source=%s app_code=%s event_id=%s: %w", candidate.SourceName, candidate.AppCode, candidate.EventID, err)
}
item.WalletMessage = &message
case roomSource:
if roomRepo == nil {
return nil, errors.New("room owner repository is not configured")
}
envelope, err := roomRepo.LoadRoomOutboxEnvelope(candidateCtx, candidate.AppCode, candidate.EventID)
if err != nil {
return nil, fmt.Errorf("load owner snapshot source=%s app_code=%s event_id=%s: %w", candidate.SourceName, candidate.AppCode, candidate.EventID, err)
}
item.RoomEnvelope = envelope
default:
return nil, fmt.Errorf("unsupported source %q", candidate.SourceName)
}
items = append(items, item)
}
return items, nil
}
func listDueDeliveries(ctx context.Context, db *sql.DB, eventIDs []string, appCodes []string, limit int, nowMS int64) ([]deliveryCandidate, error) {
if db == nil {
return nil, errors.New("notice database is not configured")
}
if limit <= 0 || limit > 500 {
return nil, errors.New("limit must be between 1 and 500")
}
apps := normalizeAppCodes(appCodes)
if len(apps) == 0 {
return nil, errors.New("at least one app_code is required for indexed preview")
}
if len(apps) > 32 {
return nil, errors.New("app_code filter count exceeds 32")
}
ids := normalizeEventIDs(eventIDs)
if len(ids) > 500 {
return nil, errors.New("event_id filter count exceeds 500")
}
candidates := make([]deliveryCandidate, 0, limit)
for _, sourceName := range []string{walletSource, roomSource} {
for _, appCodeValue := range apps {
branches := []struct {
indexName string
status string
dueColumn string
}{
{indexName: "idx_notice_delivery_retry", status: "retryable", dueColumn: "next_retry_at_ms"},
{indexName: "idx_notice_delivery_lock", status: "delivering", dueColumn: "lock_until_ms"},
}
for _, branch := range branches {
query, args := dueDeliveryBranch(sourceName, appCodeValue, branch.indexName, branch.status, branch.dueColumn, ids, limit, nowMS)
branchCandidates, err := queryDueDeliveryCandidates(ctx, db, query, args...)
if err != nil {
return nil, err
}
candidates = append(candidates, branchCandidates...)
}
}
}
sort.Slice(candidates, func(i, j int) bool {
if candidates[i].UpdatedAtMS != candidates[j].UpdatedAtMS {
return candidates[i].UpdatedAtMS < candidates[j].UpdatedAtMS
}
return candidates[i].Selector < candidates[j].Selector
})
if len(candidates) > limit {
candidates = candidates[:limit]
}
return candidates, nil
}
func loadExactDueDeliveries(ctx context.Context, db *sql.DB, selectors []deliverySelector, nowMS int64) ([]deliveryCandidate, error) {
if db == nil {
return nil, errors.New("notice database is not configured")
}
normalized, err := normalizeDeliverySelectors(selectors)
if err != nil {
return nil, err
}
candidates := make([]deliveryCandidate, 0, len(selectors))
for _, selector := range normalized {
key := selector.String()
var candidate deliveryCandidate
err = db.QueryRowContext(ctx, `
SELECT source_name, app_code, source_event_id, status, retry_count, updated_at_ms
FROM notice_delivery_events FORCE INDEX (PRIMARY)
WHERE source_name = ? AND app_code = ? AND source_event_id = ? AND channel = ?
AND ((status = 'retryable' AND next_retry_at_ms <= ?)
OR (status = 'delivering' AND lock_until_ms <= ?))`,
selector.SourceName,
selector.AppCode,
selector.EventID,
c2cChannel,
nowMS,
nowMS,
).Scan(&candidate.SourceName, &candidate.AppCode, &candidate.EventID, &candidate.Status, &candidate.RetryCount, &candidate.UpdatedAtMS)
if errors.Is(err, sql.ErrNoRows) {
return nil, fmt.Errorf("delivery %q does not exist or is no longer due", key)
}
if err != nil {
return nil, err
}
if candidate.SourceName != selector.SourceName || candidate.AppCode != selector.AppCode || candidate.EventID != selector.EventID {
return nil, fmt.Errorf("delivery %q returned a mismatched primary key", key)
}
candidate.Selector = key
candidates = append(candidates, candidate)
}
return candidates, nil
}
func normalizeDeliverySelectors(values []deliverySelector) ([]deliverySelector, error) {
normalized := make([]deliverySelector, 0, len(values))
seen := make(map[string]struct{}, len(values))
for _, value := range values {
selector := deliverySelector{
SourceName: strings.TrimSpace(value.SourceName),
AppCode: strings.TrimSpace(value.AppCode),
EventID: strings.TrimSpace(value.EventID),
}
if selector.SourceName != walletSource && selector.SourceName != roomSource {
return nil, fmt.Errorf("unsupported delivery source %q", selector.SourceName)
}
if selector.AppCode == "" || selector.EventID == "" {
return nil, errors.New("delivery app_code and event_id are required")
}
selector.AppCode = appcode.Normalize(selector.AppCode)
key := selector.String()
if _, exists := seen[key]; exists {
return nil, fmt.Errorf("duplicate delivery selector %q", key)
}
seen[key] = struct{}{}
normalized = append(normalized, selector)
}
return normalized, nil
}
func dueDeliveryBranch(sourceName string, appCodeValue string, indexName string, status string, dueColumn string, eventIDs []string, limit int, nowMS int64) (string, []any) {
// Callers pass only compile-time index/status/column tuples. Splitting status
// branches gives MySQL the complete (source,app,channel,status,due_time) prefix.
query := `
SELECT source_name, app_code, source_event_id, status, retry_count, updated_at_ms
FROM notice_delivery_events FORCE INDEX (` + indexName + `)
WHERE source_name = ? AND app_code = ? AND channel = ? AND status = ?
AND ` + dueColumn + ` <= ?`
args := []any{sourceName, appCodeValue, c2cChannel, status, nowMS}
if len(eventIDs) > 0 {
query += " AND source_event_id IN (" + placeholders(len(eventIDs)) + ")"
for _, eventID := range eventIDs {
args = append(args, eventID)
}
}
query += " ORDER BY updated_at_ms ASC, source_event_id ASC LIMIT ?"
args = append(args, limit)
return query, args
}
func queryDueDeliveryCandidates(ctx context.Context, db *sql.DB, query string, args ...any) ([]deliveryCandidate, error) {
rows, err := db.QueryContext(ctx, query, args...)
if err != nil {
return nil, err
}
defer rows.Close()
candidates := make([]deliveryCandidate, 0)
for rows.Next() {
var candidate deliveryCandidate
if err := rows.Scan(&candidate.SourceName, &candidate.AppCode, &candidate.EventID, &candidate.Status, &candidate.RetryCount, &candidate.UpdatedAtMS); err != nil {
return nil, err
}
candidate.Selector = deliverySelector{SourceName: candidate.SourceName, AppCode: candidate.AppCode, EventID: candidate.EventID}.String()
candidates = append(candidates, candidate)
}
return candidates, rows.Err()
}
func normalizeAppCodes(values []string) []string {
normalized := make([]string, 0, len(values))
seen := make(map[string]struct{}, len(values))
for _, value := range values {
if strings.TrimSpace(value) == "" {
continue
}
value = appcode.Normalize(value)
if _, exists := seen[value]; exists {
continue
}
seen[value] = struct{}{}
normalized = append(normalized, value)
}
return normalized
}
func normalizeEventIDs(values []string) []string {
normalized := make([]string, 0, len(values))
seen := make(map[string]struct{}, len(values))
for _, value := range values {
value = strings.TrimSpace(value)
if value == "" {
continue
}
if _, exists := seen[value]; exists {
continue
}
seen[value] = struct{}{}
normalized = append(normalized, value)
}
return normalized
}
func requireDelivered(ctx context.Context, db *sql.DB, candidate deliveryCandidate) error {
var status string
err := db.QueryRowContext(ctx, `
SELECT status
FROM notice_delivery_events
WHERE source_name = ? AND app_code = ? AND source_event_id = ? AND channel = ?`,
candidate.SourceName,
candidate.AppCode,
candidate.EventID,
c2cChannel,
).Scan(&status)
if err != nil {
return err
}
if status != "delivered" {
return fmt.Errorf("delivery status is %q after replay", status)
}
return nil
}
func walletWorkerOptions(cfg config.Config) walletnotice.WalletNoticeWorkerOptions {
worker := cfg.WalletNoticeWorker
return walletnotice.WalletNoticeWorkerOptions{
WorkerID: "wallet-notice-replay-" + cfg.NodeID,
BatchSize: worker.BatchSize,
LockTTL: worker.LockTTL,
PublishTimeout: worker.PublishTimeout,
MaxRetryCount: worker.MaxRetryCount,
InitialBackoff: worker.InitialBackoff,
MaxBackoff: worker.MaxBackoff,
}
}
func roomWorkerOptions(cfg config.Config) roomnotice.RoomNoticeWorkerOptions {
worker := cfg.RoomNoticeWorker
return roomnotice.RoomNoticeWorkerOptions{
WorkerID: "room-notice-replay-" + cfg.NodeID,
BatchSize: worker.BatchSize,
LockTTL: worker.LockTTL,
PublishTimeout: worker.PublishTimeout,
MaxRetryCount: worker.MaxRetryCount,
InitialBackoff: worker.InitialBackoff,
MaxBackoff: worker.MaxBackoff,
}
}
func placeholders(count int) string {
values := make([]string, count)
for index := range values {
values[index] = "?"
}
return strings.Join(values, ",")
}

View File

@ -0,0 +1,112 @@
package main
import (
"context"
"database/sql"
"regexp"
"strings"
"testing"
"github.com/DATA-DOG/go-sqlmock"
)
func TestDeliverySelectorFlagKeepsColonInsideEventID(t *testing.T) {
var values deliverySelectorFlag
if err := values.Set("wallet_outbox: FAMI :wallet:balance:evt-1"); err != nil {
t.Fatalf("parse exact delivery selector: %v", err)
}
if len(values) != 1 || values[0].SourceName != walletSource || values[0].AppCode != "fami" || values[0].EventID != "wallet:balance:evt-1" {
t.Fatalf("parsed selector mismatch: %+v", values)
}
if err := values.Set("unknown:lalu:event-2"); err == nil {
t.Fatal("unsupported source must be rejected before any database access")
}
if err := values.Set("room_outbox::event-3"); err == nil {
t.Fatal("empty app_code must not default to lalu")
}
}
func TestLoadExactDueDeliveriesUsesFullPrimaryKey(t *testing.T) {
db, mock, err := sqlmock.New()
if err != nil {
t.Fatalf("create sqlmock: %v", err)
}
defer db.Close()
nowMS := int64(1_700_000_000_000)
query := regexp.QuoteMeta(`
SELECT source_name, app_code, source_event_id, status, retry_count, updated_at_ms
FROM notice_delivery_events FORCE INDEX (PRIMARY)
WHERE source_name = ? AND app_code = ? AND source_event_id = ? AND channel = ?
AND ((status = 'retryable' AND next_retry_at_ms <= ?)
OR (status = 'delivering' AND lock_until_ms <= ?))`)
mock.ExpectQuery(query).
WithArgs(walletSource, "fami", "wallet:evt-1", c2cChannel, nowMS, nowMS).
WillReturnRows(sqlmock.NewRows([]string{"source_name", "app_code", "source_event_id", "status", "retry_count", "updated_at_ms"}).
AddRow(walletSource, "fami", "wallet:evt-1", "retryable", 2, nowMS-1000))
candidates, err := loadExactDueDeliveries(context.Background(), db, []deliverySelector{{SourceName: walletSource, AppCode: " FAMI ", EventID: "wallet:evt-1"}}, nowMS)
if err != nil {
t.Fatalf("load exact due delivery: %v", err)
}
if len(candidates) != 1 || candidates[0].Selector != "wallet_outbox:fami:wallet:evt-1" || candidates[0].RetryCount != 2 {
t.Fatalf("exact candidate mismatch: %+v", candidates)
}
if err := mock.ExpectationsWereMet(); err != nil {
t.Fatalf("exact query did not use the expected primary key: %v", err)
}
}
func TestLoadExactDueDeliveriesRejectsDuplicateBeforeQuery(t *testing.T) {
db, mock, err := sqlmock.New()
if err != nil {
t.Fatalf("create sqlmock: %v", err)
}
defer db.Close()
selectors := []deliverySelector{
{SourceName: roomSource, AppCode: "lalu", EventID: "evt-1"},
{SourceName: roomSource, AppCode: " LALU ", EventID: "evt-1"},
}
if _, err := loadExactDueDeliveries(context.Background(), db, selectors, 1000); err == nil || !strings.Contains(err.Error(), "duplicate") {
t.Fatalf("duplicate selector error = %v", err)
}
if err := mock.ExpectationsWereMet(); err != nil {
t.Fatalf("duplicate validation unexpectedly queried MySQL: %v", err)
}
}
func TestLoadExactDueDeliveriesRejectsNotDueRow(t *testing.T) {
db, mock, err := sqlmock.New()
if err != nil {
t.Fatalf("create sqlmock: %v", err)
}
defer db.Close()
mock.ExpectQuery("FROM notice_delivery_events FORCE INDEX \\(PRIMARY\\)").
WillReturnError(sql.ErrNoRows)
_, err = loadExactDueDeliveries(context.Background(), db, []deliverySelector{{SourceName: roomSource, AppCode: "lalu", EventID: "evt-1"}}, 1000)
if err == nil || !strings.Contains(err.Error(), "no longer due") {
t.Fatalf("not-due selector error = %v", err)
}
}
func TestDueDeliveryBranchUsesCompleteRetryIndexPrefix(t *testing.T) {
query, args := dueDeliveryBranch(walletSource, "huwaa", "idx_notice_delivery_retry", "retryable", "next_retry_at_ms", []string{"evt-1", "evt-2"}, 20, 1234)
for _, fragment := range []string{
"FORCE INDEX (idx_notice_delivery_retry)",
"source_name = ? AND app_code = ? AND channel = ? AND status = ?",
"next_retry_at_ms <= ?",
"source_event_id IN (?,?)",
} {
if !strings.Contains(query, fragment) {
t.Fatalf("indexed preview query is missing %q:\n%s", fragment, query)
}
}
wantArgs := []any{walletSource, "huwaa", c2cChannel, "retryable", int64(1234), "evt-1", "evt-2", 20}
if len(args) != len(wantArgs) {
t.Fatalf("preview args = %v, want %v", args, wantArgs)
}
for index := range args {
if args[index] != wantArgs[index] {
t.Fatalf("preview arg[%d] = %#v, want %#v", index, args[index], wantArgs[index])
}
}
}

View File

@ -251,6 +251,47 @@ func (r *MySQLRepository) ClaimRoomKickEnvelope(ctx context.Context, workerID st
return event, true, nil return event, true, nil
} }
// LoadRoomOutboxEnvelope loads one immutable owner event for an exact offline
// replay. Requiring the full primary key prevents repair tooling from scanning or
// claiming room-service state that was not selected by an operator.
func (r *MySQLRepository) LoadRoomOutboxEnvelope(ctx context.Context, appCode string, eventID string) (*roomeventsv1.EventEnvelope, error) {
if r == nil || r.db == nil {
return nil, xerr.New(xerr.Unavailable, "notice repository is not configured")
}
appCode = strings.TrimSpace(appCode)
eventID = strings.TrimSpace(eventID)
if appCode == "" || eventID == "" {
return nil, xerr.New(xerr.InvalidArgument, "app_code and event_id are required")
}
appCode = appcode.Normalize(appCode)
var (
ownerRoomID string
envelopeBytes []byte
)
query := fmt.Sprintf(`
SELECT room_id, envelope
FROM %s
WHERE app_code = ? AND event_id = ? AND event_type = ?`, r.roomOutboxTable)
if err := r.db.QueryRowContext(ctx, query, appCode, eventID, eventRoomUserKicked).Scan(&ownerRoomID, &envelopeBytes); err != nil {
return nil, err
}
var envelope roomeventsv1.EventEnvelope
if err := proto.Unmarshal(envelopeBytes, &envelope); err != nil {
return nil, err
}
// Validate both the requested identity and the current consumer contract. A
// mismatched historical blob must never be delivered under another event's
// notice_delivery_events key.
if envelope.GetAppCode() != appCode || envelope.GetEventId() != eventID || envelope.GetRoomId() != ownerRoomID {
return nil, fmt.Errorf("room outbox envelope identity does not match requested key")
}
if _, err := roomKickEventFromEnvelope(&envelope, envelope.GetOccurredAtMs()); err != nil {
return nil, err
}
return &envelope, nil
}
func (r *MySQLRepository) lockRoomKickEvent(ctx context.Context, tx *sql.Tx, candidate roomKickCandidate) (RoomKickEvent, error) { func (r *MySQLRepository) lockRoomKickEvent(ctx context.Context, tx *sql.Tx, candidate roomKickCandidate) (RoomKickEvent, error) {
var ( var (
event RoomKickEvent event RoomKickEvent

View File

@ -0,0 +1,68 @@
package roomnotice
import (
"context"
"regexp"
"testing"
"github.com/DATA-DOG/go-sqlmock"
"google.golang.org/protobuf/proto"
roomeventsv1 "hyapp.local/api/proto/events/room/v1"
)
func TestLoadRoomOutboxEnvelopeRejectsEmptyAppInsteadOfDefaultingToLalu(t *testing.T) {
db, mock, err := sqlmock.New()
if err != nil {
t.Fatalf("create sqlmock: %v", err)
}
defer db.Close()
repository := &MySQLRepository{db: db, roomOutboxTable: "`hyapp_room`.`room_outbox`"}
if _, err := repository.LoadRoomOutboxEnvelope(context.Background(), "", "evt-1"); err == nil {
t.Fatal("empty app_code must be rejected before owner table access")
}
if err := mock.ExpectationsWereMet(); err != nil {
t.Fatalf("empty key unexpectedly queried room owner table: %v", err)
}
}
func TestLoadRoomOutboxEnvelopeUsesExactImmutableOwnerEnvelope(t *testing.T) {
db, mock, err := sqlmock.New()
if err != nil {
t.Fatalf("create sqlmock: %v", err)
}
defer db.Close()
body, err := proto.Marshal(&roomeventsv1.RoomUserKicked{ActorUserId: 7, TargetUserId: 42})
if err != nil {
t.Fatalf("marshal room kick body: %v", err)
}
envelopeBytes, err := proto.Marshal(&roomeventsv1.EventEnvelope{
AppCode: "huwaa",
EventId: "room:kick:evt-1",
EventType: eventRoomUserKicked,
RoomId: "room-1",
RoomVersion: 9,
OccurredAtMs: 1700000000000,
Body: body,
})
if err != nil {
t.Fatalf("marshal room envelope: %v", err)
}
repository := &MySQLRepository{db: db, roomOutboxTable: "`hyapp_room`.`room_outbox`"}
mock.ExpectQuery(regexp.QuoteMeta(`
SELECT room_id, envelope
FROM `+"`hyapp_room`.`room_outbox`"+`
WHERE app_code = ? AND event_id = ? AND event_type = ?`)).
WithArgs("huwaa", "room:kick:evt-1", eventRoomUserKicked).
WillReturnRows(sqlmock.NewRows([]string{"room_id", "envelope"}).AddRow("room-1", envelopeBytes))
envelope, err := repository.LoadRoomOutboxEnvelope(context.Background(), " HUWAA ", "room:kick:evt-1")
if err != nil {
t.Fatalf("load exact room owner envelope: %v", err)
}
if envelope.GetAppCode() != "huwaa" || envelope.GetEventId() != "room:kick:evt-1" || envelope.GetRoomId() != "room-1" {
t.Fatalf("room owner snapshot mismatch: %+v", envelope)
}
if err := mock.ExpectationsWereMet(); err != nil {
t.Fatalf("room replay query mismatch: %v", err)
}
}

View File

@ -256,6 +256,61 @@ func (r *MySQLRepository) ClaimWalletBalanceMessage(ctx context.Context, workerI
return event, true, nil return event, true, nil
} }
// LoadWalletOutboxMessage loads one immutable owner event for an operator-selected
// delivery replay. The exact (app_code, event_id) key is mandatory so an offline
// repair cannot accidentally turn into an unbounded wallet_outbox scan.
func (r *MySQLRepository) LoadWalletOutboxMessage(ctx context.Context, appCode string, eventID string) (walletmq.WalletOutboxMessage, error) {
if r == nil || r.db == nil {
return walletmq.WalletOutboxMessage{}, xerr.New(xerr.Unavailable, "notice repository is not configured")
}
appCode = strings.TrimSpace(appCode)
eventID = strings.TrimSpace(eventID)
if appCode == "" || eventID == "" {
return walletmq.WalletOutboxMessage{}, xerr.New(xerr.InvalidArgument, "app_code and event_id are required")
}
appCode = appcode.Normalize(appCode)
message := walletmq.WalletOutboxMessage{MessageType: walletmq.MessageTypeWalletOutboxEvent}
query := fmt.Sprintf(`
SELECT app_code, event_id, event_type, transaction_id, command_id, user_id,
asset_type, available_delta, frozen_delta,
COALESCE(CAST(payload AS CHAR), '{}'), created_at_ms
FROM %s
WHERE app_code = ? AND event_id = ?
AND event_type IN (?, ?) AND asset_type <> ?`, r.walletOutboxTable)
err := r.db.QueryRowContext(ctx, query, appCode, eventID, eventWalletBalance, eventVIPActivated, robotCoinAssetType).Scan(
&message.AppCode,
&message.EventID,
&message.EventType,
&message.TransactionID,
&message.CommandID,
&message.UserID,
&message.AssetType,
&message.AvailableDelta,
&message.FrozenDelta,
&message.PayloadJSON,
&message.OccurredAtMS,
)
if err != nil {
return walletmq.WalletOutboxMessage{}, err
}
// Reuse the live MQ decoder boundary before any external side effect. A
// malformed historical row must stay retryable instead of being sent with a
// subtly different repair-only payload.
encoded, err := walletmq.EncodeWalletOutboxMessage(message)
if err != nil {
return walletmq.WalletOutboxMessage{}, err
}
decoded, err := walletmq.DecodeWalletOutboxMessage(encoded)
if err != nil {
return walletmq.WalletOutboxMessage{}, err
}
if decoded.AppCode != appCode || decoded.EventID != eventID {
return walletmq.WalletOutboxMessage{}, fmt.Errorf("wallet outbox message identity does not match requested key")
}
return decoded, nil
}
func walletBalanceEventFromMessage(message walletmq.WalletOutboxMessage) (WalletBalanceEvent, error) { func walletBalanceEventFromMessage(message walletmq.WalletOutboxMessage) (WalletBalanceEvent, error) {
event := WalletBalanceEvent{ event := WalletBalanceEvent{
AppCode: appcode.Normalize(message.AppCode), AppCode: appcode.Normalize(message.AppCode),

View File

@ -0,0 +1,56 @@
package walletnotice
import (
"context"
"regexp"
"testing"
"github.com/DATA-DOG/go-sqlmock"
)
func TestLoadWalletOutboxMessageRejectsEmptyAppInsteadOfDefaultingToLalu(t *testing.T) {
db, mock, err := sqlmock.New()
if err != nil {
t.Fatalf("create sqlmock: %v", err)
}
defer db.Close()
repository := &MySQLRepository{db: db, walletOutboxTable: "`hyapp_wallet`.`wallet_outbox`"}
if _, err := repository.LoadWalletOutboxMessage(context.Background(), " ", "evt-1"); err == nil {
t.Fatal("empty app_code must be rejected before owner table access")
}
if err := mock.ExpectationsWereMet(); err != nil {
t.Fatalf("empty key unexpectedly queried wallet owner table: %v", err)
}
}
func TestLoadWalletOutboxMessageUsesExactImmutablePrivateFact(t *testing.T) {
db, mock, err := sqlmock.New()
if err != nil {
t.Fatalf("create sqlmock: %v", err)
}
defer db.Close()
repository := &MySQLRepository{db: db, walletOutboxTable: "`hyapp_wallet`.`wallet_outbox`"}
mock.ExpectQuery(regexp.QuoteMeta(`
SELECT app_code, event_id, event_type, transaction_id, command_id, user_id,
asset_type, available_delta, frozen_delta,
COALESCE(CAST(payload AS CHAR), '{}'), created_at_ms
FROM `+"`hyapp_wallet`.`wallet_outbox`"+`
WHERE app_code = ? AND event_id = ?
AND event_type IN (?, ?) AND asset_type <> ?`)).
WithArgs("fami", "wallet:evt-1", eventWalletBalance, eventVIPActivated, robotCoinAssetType).
WillReturnRows(sqlmock.NewRows([]string{
"app_code", "event_id", "event_type", "transaction_id", "command_id", "user_id",
"asset_type", "available_delta", "frozen_delta", "payload", "created_at_ms",
}).AddRow("fami", "wallet:evt-1", eventWalletBalance, "tx-1", "cmd-1", int64(42), "COIN", int64(100), int64(0), `{"balance_after":100}`, int64(1700000000000)))
message, err := repository.LoadWalletOutboxMessage(context.Background(), " FAMI ", "wallet:evt-1")
if err != nil {
t.Fatalf("load exact wallet owner fact: %v", err)
}
if message.AppCode != "fami" || message.EventID != "wallet:evt-1" || message.UserID != 42 || message.PayloadJSON != `{"balance_after":100}` {
t.Fatalf("wallet owner snapshot mismatch: %+v", message)
}
if err := mock.ExpectationsWereMet(); err != nil {
t.Fatalf("wallet replay query mismatch: %v", err)
}
}

View File

@ -516,12 +516,26 @@ func (s *Service) ensureRobotRoomParticipants(ctx context.Context, config RobotR
activeRobots := onlineRobotIDs(config.RobotUserIDs, online) activeRobots := onlineRobotIDs(config.RobotUserIDs, online)
missing := activeTarget - len(activeRobots) missing := activeTarget - len(activeRobots)
if missing > 0 { if missing > 0 {
// 首次补回缺席机器人也必须从当前 Room Cell 快照选择真实麦位。候选池下标可能
// 大于 seat_count同一轮成功补位后立刻占记避免后续候选复用尚未写回旧快照的麦位。
occupiedSeats := make(map[int32]bool, len(snapshot.GetMicSeats()))
for _, seat := range snapshot.GetMicSeats() {
if seat.GetUserId() > 0 {
occupiedSeats[seat.GetSeatNo()] = true
}
}
for _, userID := range robotRoomFillCandidates(config, online, missing) { for _, userID := range robotRoomFillCandidates(config, online, missing) {
// active 机器人房从持久化配置恢复时,旧进程可能已经把机器人 stale 成 left // active 机器人房从持久化配置恢复时,旧进程可能已经把机器人 stale 成 left
// 这里只补齐后台配置的活跃人数,不再把整个候选池一次性塞回房间。 // 这里只补齐后台配置的活跃人数,不再把整个候选池一次性塞回房间。
if err := s.joinRobotRoomUser(roomCtx, config, userID, 0, fmt.Sprintf("runtime-join:%d:%d", userID, time.Now().UTC().UnixNano())); err != nil { seatNo := robotRestoreSeatNo(config, userID, snapshot.GetMicSeats(), occupiedSeats)
if seatNo <= 0 {
logx.Warn(roomCtx, "robot_room_runtime_join_restore_skipped", slog.String("room_id", config.RoomID), slog.Int64("robot_user_id", userID), slog.String("reason", "no_free_seat"))
continue
}
if err := s.joinRobotRoomUser(roomCtx, config, userID, seatNo, fmt.Sprintf("runtime-join:%d:%d", userID, time.Now().UTC().UnixNano())); err != nil {
return err return err
} }
occupiedSeats[seatNo] = true
online[userID] = true online[userID] = true
activeRobots = append(activeRobots, userID) activeRobots = append(activeRobots, userID)
} }
@ -544,14 +558,11 @@ func (s *Service) ensureRobotRoomParticipants(ctx context.Context, config RobotR
if userID <= 0 || seated[userID] { if userID <= 0 || seated[userID] {
continue continue
} }
seatNo := preferredRobotSeatNo(config, userID) seatNo := robotRestoreSeatNo(config, userID, snapshot.GetMicSeats(), occupiedSeats)
if occupiedSeats[seatNo] {
seatNo = firstFreeRobotSeatNo(snapshot.GetMicSeats(), activeTarget)
if seatNo <= 0 { if seatNo <= 0 {
logx.Warn(roomCtx, "robot_room_runtime_mic_restore_skipped", slog.String("room_id", config.RoomID), slog.Int64("robot_user_id", userID), slog.String("reason", "no_free_seat")) logx.Warn(roomCtx, "robot_room_runtime_mic_restore_skipped", slog.String("room_id", config.RoomID), slog.Int64("robot_user_id", userID), slog.String("reason", "no_free_seat"))
continue continue
} }
}
// 虚拟麦位只服务机器人房展示若并发恢复导致机器人已上麦Conflict 可视为达成目标。 // 虚拟麦位只服务机器人房展示若并发恢复导致机器人已上麦Conflict 可视为达成目标。
if _, err := s.RobotVirtualMicUp(roomCtx, RobotMicUpInput{ if _, err := s.RobotVirtualMicUp(roomCtx, RobotMicUpInput{
Meta: robotRoomCommandMeta(config, config.OwnerRobotUserID, fmt.Sprintf("runtime-mic:%d:%d", userID, time.Now().UTC().UnixNano())), Meta: robotRoomCommandMeta(config, config.OwnerRobotUserID, fmt.Sprintf("runtime-mic:%d:%d", userID, time.Now().UTC().UnixNano())),
@ -568,6 +579,39 @@ func (s *Service) ensureRobotRoomParticipants(ctx context.Context, config RobotR
return nil return nil
} }
// robotRestoreSeatNo 只接受当前 Room Cell 快照里真实存在且空闲的麦位。
// robot_user_ids 是候选池而不是麦位表:历史配置可包含数十个轮换机器人,因此其下标
// 可能远大于当前 10/15/20 个麦位;这种 preferred 必须回退到快照中的首个空麦,不能
// 把不存在的 seat_no 发给 RobotVirtualMicUp 后让每 10 秒恢复扫描永久失败。
func robotRestoreSeatNo(config RobotRoomConfig, userID int64, seats []*roomv1.SeatState, occupied map[int32]bool) int32 {
preferred := preferredRobotSeatNo(config, userID)
if preferred > 0 && robotSnapshotSeatFree(seats, preferred) && !occupied[preferred] {
return preferred
}
// snapshot 是本轮恢复开始时的只读副本occupied 还包含本轮刚成功分配、尚未反映到
// 该副本的麦位。回退时同时检查二者,避免多个待恢复机器人重复选择同一个空麦。
var firstFree int32
for _, seat := range seats {
seatNo := seat.GetSeatNo()
if seatNo <= 0 || seat.GetUserId() > 0 || seat.GetLocked() || occupied[seatNo] {
continue
}
if firstFree == 0 || seatNo < firstFree {
firstFree = seatNo
}
}
return firstFree
}
func robotSnapshotSeatFree(seats []*roomv1.SeatState, seatNo int32) bool {
for _, seat := range seats {
if seat.GetSeatNo() == seatNo {
return seat.GetUserId() == 0 && !seat.GetLocked()
}
}
return false
}
func (s *Service) runRobotRoomRuntime(ctx context.Context, config RobotRoomConfig) { func (s *Service) runRobotRoomRuntime(ctx context.Context, config RobotRoomConfig) {
activity := newRobotRoomActivity(config.RobotUserIDs) activity := newRobotRoomActivity(config.RobotUserIDs)
if snapshot, err := s.currentSnapshot(appcode.WithContext(ctx, config.AppCode), config.RoomID); err == nil { if snapshot, err := s.currentSnapshot(appcode.WithContext(ctx, config.AppCode), config.RoomID); err == nil {

View File

@ -89,6 +89,101 @@ func TestRobotRoomRuntimeStartRestoresLeftRobots(t *testing.T) {
} }
} }
func TestRobotRoomRuntimeStartRestoresHighIndexOwnerToRealSeat(t *testing.T) {
ctx := appcode.WithContext(context.Background(), appcode.Default)
repository := mysqltest.NewRepository(t)
svc := roomservice.New(roomservice.Config{
NodeID: "node-robot-runtime-high-index-restore-test",
LeaseTTL: 10 * time.Second,
RankLimit: 20,
SnapshotEveryN: 1,
}, router.NewMemoryDirectory(), repository, followTestWallet{}, integration.NewNoopRoomEventPublisher(), integration.NewNoopOutboxPublisher())
// 候选池有 12 个机器人而房间只有 10 个麦位owner 排序后位于第 12 位。
// bootstrap 会把 active owner 放到真实 seat 1运行时恢复不能再按候选池下标猜 seat 12。
candidates := make([]int64, 12)
for index := range candidates {
candidates[index] = int64(2001 + index)
}
ownerID := candidates[len(candidates)-1]
created, err := svc.AdminCreateRobotRoom(ctx, &roomv1.AdminCreateRobotRoomRequest{
Meta: roomservice.NewRequestMeta("", 9001),
OwnerRobotUserId: ownerID,
CandidateRobotUserIds: candidates,
MinRobotCount: 2,
MaxRobotCount: 2,
SeatCount: 10,
RoomName: "Robot Runtime High Index Restore",
RoomAvatar: testRoomCoverURL,
VisibleRegionId: 686,
GiftRule: &roomv1.AdminRobotRoomGiftRule{
GiftIds: []string{"normal_gift"},
LuckyGiftIds: []string{"lucky_gift"},
NormalGiftIntervalMs: 10000,
LuckyComboMin: 1,
LuckyComboMax: 1,
LuckyPauseMinMs: 5000,
LuckyPauseMaxMs: 5000,
},
AdminId: 9001,
})
if err != nil {
t.Fatalf("create high-index-owner robot room failed: %v", err)
}
roomID := created.GetRoom().GetRoomId()
robotIDs := created.GetRoom().GetRobotUserIds()
if len(robotIDs) != len(candidates) || robotIDs[len(robotIDs)-1] != ownerID {
t.Fatalf("test requires owner at preferred seat 12, got owner=%d robots=%v", ownerID, robotIDs)
}
initial, err := svc.GetRoomSnapshot(ctx, &roomv1.GetRoomSnapshotRequest{
Meta: &roomv1.RequestMeta{AppCode: appcode.Default, RoomId: roomID},
RoomId: roomID,
ViewerUserId: ownerID,
})
if err != nil {
t.Fatalf("initial high-index-owner snapshot failed: %v", err)
}
if _, err := svc.AdminSetRobotRoomStatus(ctx, &roomv1.AdminSetRobotRoomStatusRequest{
Meta: roomservice.NewRequestMeta(roomID, 9001),
Status: "stopped",
AdminId: 9001,
}); err != nil {
t.Fatalf("stop high-index-owner robot room failed: %v", err)
}
for _, user := range initial.GetRoom().GetOnlineUsers() {
if _, err := svc.LeaveRoom(ctx, &roomv1.LeaveRoomRequest{Meta: roomservice.NewRequestMeta(roomID, user.GetUserId())}); err != nil {
t.Fatalf("leave initial robot %d failed: %v", user.GetUserId(), err)
}
}
if _, err := svc.AdminSetRobotRoomStatus(ctx, &roomv1.AdminSetRobotRoomStatusRequest{
Meta: roomservice.NewRequestMeta(roomID, 9001),
Status: "active",
AdminId: 9001,
}); err != nil {
t.Fatalf("restart high-index-owner robot room failed: %v", err)
}
restored, err := svc.GetRoomSnapshot(ctx, &roomv1.GetRoomSnapshotRequest{
Meta: &roomv1.RequestMeta{AppCode: appcode.Default, RoomId: roomID},
RoomId: roomID,
ViewerUserId: ownerID,
})
if err != nil {
t.Fatalf("snapshot after high-index-owner restore failed: %v", err)
}
ownerSeat := int32(0)
for _, seat := range restored.GetRoom().GetMicSeats() {
if seat.GetUserId() == ownerID {
ownerSeat = seat.GetSeatNo()
break
}
}
if ownerSeat <= 0 || ownerSeat > int32(len(restored.GetRoom().GetMicSeats())) {
t.Fatalf("high-index owner must be restored to a real snapshot seat, got seat=%d", ownerSeat)
}
}
func snapshotContainsUser(snapshot *roomv1.RoomSnapshot, userID int64) bool { func snapshotContainsUser(snapshot *roomv1.RoomSnapshot, userID int64) bool {
for _, user := range snapshot.GetOnlineUsers() { for _, user := range snapshot.GetOnlineUsers() {
if user.GetUserId() == userID { if user.GetUserId() == userID {

View File

@ -123,6 +123,51 @@ func TestNormalizeRobotRoomGiftRuleAppliesBehaviorDefaults(t *testing.T) {
} }
} }
func TestRobotRestoreSeatNoFallsBackWhenPreferredExceedsSnapshot(t *testing.T) {
robots := make([]int64, 12)
for index := range robots {
robots[index] = int64(1001 + index)
}
seats := make([]*roomv1.SeatState, 10)
for index := range seats {
seats[index] = &roomv1.SeatState{SeatNo: int32(index + 1)}
}
seats[0].UserId = robots[0]
// 候选池第 12 个机器人历史 preferred=12但当前房间只有 10 个麦位;
// 恢复必须选择真实存在的首个空麦 2而不是把 seat=12 交给 Room Cell。
got := robotRestoreSeatNo(
RobotRoomConfig{RobotUserIDs: robots, SeatCount: 10},
robots[11],
seats,
map[int32]bool{1: true},
)
if got != 2 {
t.Fatalf("out-of-range preferred seat must fall back to first free snapshot seat: got %d want 2", got)
}
got = robotRestoreSeatNo(
RobotRoomConfig{RobotUserIDs: robots, SeatCount: 10},
robots[11],
seats,
map[int32]bool{1: true, 2: true},
)
if got != 3 {
t.Fatalf("same-pass occupied seat must not be selected twice: got %d want 3", got)
}
seats[2].Locked = true
got = robotRestoreSeatNo(
RobotRoomConfig{RobotUserIDs: robots, SeatCount: 10},
robots[11],
seats,
map[int32]bool{1: true, 2: true},
)
if got != 4 {
t.Fatalf("locked snapshot seat is not available for restore: got %d want 4", got)
}
}
func TestRobotRoomActivityPrefersOtherReplacement(t *testing.T) { func TestRobotRoomActivityPrefersOtherReplacement(t *testing.T) {
activity := newRobotRoomActivity([]int64{1001, 1002, 1003}) activity := newRobotRoomActivity([]int64{1001, 1002, 1003})
activity.markActive(1001) activity.markActive(1001)

View File

@ -11,6 +11,7 @@ import (
"time" "time"
"hyapp/pkg/appcode" "hyapp/pkg/appcode"
"hyapp/pkg/apptracking"
mysqlstorage "hyapp/services/statistics-service/internal/storage/mysql" mysqlstorage "hyapp/services/statistics-service/internal/storage/mysql"
) )
@ -326,7 +327,7 @@ func (s *queryHTTPServer) appEvents(w http.ResponseWriter, r *http.Request) {
writeJSON(w, http.StatusBadRequest, map[string]any{"error": "invalid json"}) writeJSON(w, http.StatusBadRequest, map[string]any{"error": "invalid json"})
return return
} }
if len(body.Events) == 0 || len(body.Events) > 50 { if !apptracking.ValidBatchSize(len(body.Events)) {
writeJSON(w, http.StatusBadRequest, map[string]any{"error": "events batch is invalid"}) writeJSON(w, http.StatusBadRequest, map[string]any{"error": "events batch is invalid"})
return return
} }

View File

@ -13,6 +13,7 @@ import (
"github.com/DATA-DOG/go-sqlmock" "github.com/DATA-DOG/go-sqlmock"
"hyapp/internal/testutil/mysqlschema" "hyapp/internal/testutil/mysqlschema"
"hyapp/pkg/appcode" "hyapp/pkg/appcode"
"hyapp/pkg/apptracking"
) )
func TestGiftDebitConsumptionOwnersDoNotOverlap(t *testing.T) { func TestGiftDebitConsumptionOwnersDoNotOverlap(t *testing.T) {
@ -627,7 +628,7 @@ func TestConsumeAppTrackingEventsRejectsInvalidProperties(t *testing.T) {
properties json.RawMessage properties json.RawMessage
}{ }{
{name: "invalid json", properties: json.RawMessage(`{"bad"`)}, {name: "invalid json", properties: json.RawMessage(`{"bad"`)},
{name: "too large json", properties: json.RawMessage(`"` + strings.Repeat("a", appTrackingMaxPropertiesBytes+1) + `"`)}, {name: "too large json", properties: json.RawMessage(`"` + strings.Repeat("a", apptracking.MaxPropertiesBytes+1) + `"`)},
} { } {
t.Run(test.name, func(t *testing.T) { t.Run(test.name, func(t *testing.T) {
_, err := repository.ConsumeAppTrackingEvents(appcode.WithContext(ctx, "lalu"), []AppTrackingEvent{ _, err := repository.ConsumeAppTrackingEvents(appcode.WithContext(ctx, "lalu"), []AppTrackingEvent{
@ -640,6 +641,29 @@ func TestConsumeAppTrackingEventsRejectsInvalidProperties(t *testing.T) {
} }
} }
func TestNormalizeAppTrackingEventRejects185ByteErrorCodeWithoutTruncation(t *testing.T) {
base := AppTrackingEvent{
AppCode: "lalu",
EventID: "evt-error-code-limit",
EventName: "pay_failed",
DeviceID: "dev-1",
OccurredAtMS: time.Date(2026, 7, 13, 0, 0, 0, 0, time.UTC).UnixMilli(),
}
base.ErrorCode = strings.Repeat("e", apptracking.MaxErrorCodeBytes)
normalized, _, err := normalizeAppTrackingEvent(context.Background(), base, base.OccurredAtMS)
if err != nil {
t.Fatalf("error_code at shared byte limit must remain valid: %v", err)
}
if normalized.ErrorCode != base.ErrorCode {
t.Fatalf("valid error_code must be stored intact: got=%q want=%q", normalized.ErrorCode, base.ErrorCode)
}
base.ErrorCode = strings.Repeat("e", 185)
if _, _, err := normalizeAppTrackingEvent(context.Background(), base, base.OccurredAtMS); err == nil {
t.Fatal("185-byte error_code must be rejected instead of truncated or deferred to MySQL")
}
}
func TestConsumeUserRegisteredCountsOnlyNewRegistration(t *testing.T) { func TestConsumeUserRegisteredCountsOnlyNewRegistration(t *testing.T) {
ctx := context.Background() ctx := context.Background()
_, file, _, _ := runtime.Caller(0) _, file, _, _ := runtime.Caller(0)

View File

@ -1,7 +1,6 @@
package mysql package mysql
import ( import (
"bytes"
"context" "context"
"database/sql" "database/sql"
"encoding/json" "encoding/json"
@ -11,6 +10,7 @@ import (
mysqlerr "github.com/go-sql-driver/mysql" mysqlerr "github.com/go-sql-driver/mysql"
"hyapp/pkg/appcode" "hyapp/pkg/appcode"
"hyapp/pkg/apptracking"
"hyapp/pkg/xerr" "hyapp/pkg/xerr"
) )
@ -23,11 +23,6 @@ const (
SourceFinance = "finance" SourceFinance = "finance"
) )
const (
appTrackingMaxBatchSize = 50
appTrackingMaxPropertiesBytes = 4096
)
// Repository owns the statistics read model. Online queries must use aggregate // Repository owns the statistics read model. Online queries must use aggregate
// tables; the optional activity connection only reads lucky-gift day snapshots, // tables; the optional activity connection only reads lucky-gift day snapshots,
// never the high-cardinality draw fact table. // never the high-cardinality draw fact table.
@ -1670,7 +1665,7 @@ func (r *Repository) ConsumeAppTrackingEvents(ctx context.Context, events []AppT
} }
nowMS := time.Now().UTC().UnixMilli() nowMS := time.Now().UTC().UnixMilli()
result := AppTrackingReportResult{Accepted: true, Received: len(events), ServerTimeMS: nowMS} result := AppTrackingReportResult{Accepted: true, Received: len(events), ServerTimeMS: nowMS}
if len(events) == 0 || len(events) > appTrackingMaxBatchSize { if !apptracking.ValidBatchSize(len(events)) {
return AppTrackingReportResult{}, xerr.New(xerr.InvalidArgument, "app tracking events batch is invalid") return AppTrackingReportResult{}, xerr.New(xerr.InvalidArgument, "app tracking events batch is invalid")
} }
tx, err := r.db.BeginTx(ctx, nil) tx, err := r.db.BeginTx(ctx, nil)
@ -2329,62 +2324,65 @@ func normalizeAppTrackingEvent(ctx context.Context, event AppTrackingEvent, rece
if strings.TrimSpace(event.AppCode) == "" { if strings.TrimSpace(event.AppCode) == "" {
event.AppCode = appcode.FromContext(ctx) event.AppCode = appcode.FromContext(ctx)
} }
event.AppCode = appcode.Normalize(event.AppCode) event.AppCode, err = normalizeAppTrackingText(appcode.Normalize(event.AppCode), apptracking.MaxAppCodeBytes)
event.EventID, err = normalizeAppTrackingText(event.EventID, 160)
if err != nil { if err != nil {
return AppTrackingEvent{}, "", err return AppTrackingEvent{}, "", err
} }
event.EventName, err = normalizeAppTrackingText(event.EventName, 96) event.EventID, err = normalizeAppTrackingText(event.EventID, apptracking.MaxEventIDBytes)
if err != nil {
return AppTrackingEvent{}, "", err
}
event.EventName, err = normalizeAppTrackingText(event.EventName, apptracking.MaxEventNameBytes)
if err != nil { if err != nil {
return AppTrackingEvent{}, "", err return AppTrackingEvent{}, "", err
} }
if event.EventID == "" || event.EventName == "" { if event.EventID == "" || event.EventName == "" {
return AppTrackingEvent{}, "", xerr.New(xerr.InvalidArgument, "app tracking event_id and event_name are required") return AppTrackingEvent{}, "", xerr.New(xerr.InvalidArgument, "app tracking event_id and event_name are required")
} }
event.EventType, err = normalizeAppTrackingText(event.EventType, 64) event.EventType, err = normalizeAppTrackingText(event.EventType, apptracking.MaxEventTypeBytes)
if err != nil { if err != nil {
return AppTrackingEvent{}, "", err return AppTrackingEvent{}, "", err
} }
event.Screen, err = normalizeAppTrackingText(event.Screen, 128) event.Screen, err = normalizeAppTrackingText(event.Screen, apptracking.MaxScreenBytes)
if err != nil { if err != nil {
return AppTrackingEvent{}, "", err return AppTrackingEvent{}, "", err
} }
event.TargetType, err = normalizeAppTrackingText(event.TargetType, 64) event.TargetType, err = normalizeAppTrackingText(event.TargetType, apptracking.MaxTargetTypeBytes)
if err != nil { if err != nil {
return AppTrackingEvent{}, "", err return AppTrackingEvent{}, "", err
} }
event.TargetID, err = normalizeAppTrackingText(event.TargetID, 128) event.TargetID, err = normalizeAppTrackingText(event.TargetID, apptracking.MaxTargetIDBytes)
if err != nil { if err != nil {
return AppTrackingEvent{}, "", err return AppTrackingEvent{}, "", err
} }
event.DeviceID, err = normalizeAppTrackingText(event.DeviceID, 128) event.DeviceID, err = normalizeAppTrackingText(event.DeviceID, apptracking.MaxDeviceIDBytes)
if err != nil { if err != nil {
return AppTrackingEvent{}, "", err return AppTrackingEvent{}, "", err
} }
if event.UserID <= 0 && event.DeviceID == "" { if event.UserID <= 0 && event.DeviceID == "" {
return AppTrackingEvent{}, "", xerr.New(xerr.InvalidArgument, "anonymous app tracking event requires device_id") return AppTrackingEvent{}, "", xerr.New(xerr.InvalidArgument, "anonymous app tracking event requires device_id")
} }
event.SessionID, err = normalizeAppTrackingText(event.SessionID, 160) event.SessionID, err = normalizeAppTrackingText(event.SessionID, apptracking.MaxSessionIDBytes)
if err != nil { if err != nil {
return AppTrackingEvent{}, "", err return AppTrackingEvent{}, "", err
} }
event.Platform, err = normalizeAppTrackingText(strings.ToLower(event.Platform), 32) event.Platform, err = normalizeAppTrackingText(strings.ToLower(event.Platform), apptracking.MaxPlatformBytes)
if err != nil { if err != nil {
return AppTrackingEvent{}, "", err return AppTrackingEvent{}, "", err
} }
event.AppVersion, err = normalizeAppTrackingText(event.AppVersion, 64) event.AppVersion, err = normalizeAppTrackingText(event.AppVersion, apptracking.MaxAppVersionBytes)
if err != nil { if err != nil {
return AppTrackingEvent{}, "", err return AppTrackingEvent{}, "", err
} }
event.Language, err = normalizeAppTrackingText(event.Language, 32) event.Language, err = normalizeAppTrackingText(event.Language, apptracking.MaxLanguageBytes)
if err != nil { if err != nil {
return AppTrackingEvent{}, "", err return AppTrackingEvent{}, "", err
} }
event.Timezone, err = normalizeAppTrackingText(event.Timezone, 64) event.Timezone, err = normalizeAppTrackingText(event.Timezone, apptracking.MaxTimezoneBytes)
if err != nil { if err != nil {
return AppTrackingEvent{}, "", err return AppTrackingEvent{}, "", err
} }
event.ErrorCode, err = normalizeAppTrackingText(event.ErrorCode, 64) event.ErrorCode, err = normalizeAppTrackingText(event.ErrorCode, apptracking.MaxErrorCodeBytes)
if err != nil { if err != nil {
return AppTrackingEvent{}, "", err return AppTrackingEvent{}, "", err
} }
@ -2402,21 +2400,21 @@ func normalizeAppTrackingEvent(ctx context.Context, event AppTrackingEvent, rece
} }
func normalizeAppTrackingText(value string, maxBytes int) (string, error) { func normalizeAppTrackingText(value string, maxBytes int) (string, error) {
value = strings.TrimSpace(value) value, ok := apptracking.NormalizeText(value, maxBytes)
if maxBytes > 0 && len(value) > maxBytes { if !ok {
return "", xerr.New(xerr.InvalidArgument, "app tracking text field is too long") return "", xerr.New(xerr.InvalidArgument, "app tracking text field is too long")
} }
return value, nil return value, nil
} }
func normalizeAppTrackingProperties(raw json.RawMessage) (string, error) { func normalizeAppTrackingProperties(raw json.RawMessage) (string, error) {
trimmed := bytes.TrimSpace(raw) trimmed, ok := apptracking.NormalizeProperties(raw)
if len(trimmed) == 0 || bytes.Equal(trimmed, []byte("null")) { if !ok {
return "{}", nil
}
if len(trimmed) > appTrackingMaxPropertiesBytes || !json.Valid(trimmed) {
return "", xerr.New(xerr.InvalidArgument, "app tracking properties_json is invalid") return "", xerr.New(xerr.InvalidArgument, "app tracking properties_json is invalid")
} }
if len(trimmed) == 0 {
return "{}", nil
}
return string(trimmed), nil return string(trimmed), nil
} }

View File

@ -50,6 +50,7 @@ cp_intimacy_leaderboard:
key_prefix: "user:cp_intimacy_leaderboard" key_prefix: "user:cp_intimacy_leaderboard"
outbox_worker: outbox_worker:
enabled: true enabled: true
app_codes: []
poll_interval: "1s" poll_interval: "1s"
batch_size: 100 batch_size: 100
publish_timeout: "3s" publish_timeout: "3s"

View File

@ -50,6 +50,8 @@ cp_intimacy_leaderboard:
key_prefix: "user:cp_intimacy_leaderboard" key_prefix: "user:cp_intimacy_leaderboard"
outbox_worker: outbox_worker:
enabled: true enabled: true
# 空值自动发现全部 App历史积压回放时先设 ["lalu", "fami"],核对后再加入 "huwaa"。
app_codes: []
poll_interval: "1s" poll_interval: "1s"
batch_size: 100 batch_size: 100
publish_timeout: "3s" publish_timeout: "3s"

View File

@ -51,6 +51,7 @@ cp_intimacy_leaderboard:
key_prefix: "user:cp_intimacy_leaderboard" key_prefix: "user:cp_intimacy_leaderboard"
outbox_worker: outbox_worker:
enabled: true enabled: true
app_codes: []
poll_interval: "1s" poll_interval: "1s"
batch_size: 100 batch_size: 100
publish_timeout: "3s" publish_timeout: "3s"

View File

@ -20,6 +20,7 @@ import (
"hyapp/pkg/healthhttp" "hyapp/pkg/healthhttp"
"hyapp/pkg/idgen" "hyapp/pkg/idgen"
"hyapp/pkg/logx" "hyapp/pkg/logx"
"hyapp/pkg/outboxpartition"
"hyapp/pkg/rocketmqx" "hyapp/pkg/rocketmqx"
"hyapp/pkg/roommq" "hyapp/pkg/roommq"
serviceapp "hyapp/pkg/servicekit/app" serviceapp "hyapp/pkg/servicekit/app"
@ -66,6 +67,8 @@ type App struct {
mqConsumers []*rocketmqx.Consumer mqConsumers []*rocketmqx.Consumer
// userOutboxProducer 把 user_outbox 事实发布到 RocketMQ供统计等读模型消费。 // userOutboxProducer 把 user_outbox 事实发布到 RocketMQ供统计等读模型消费。
userOutboxProducer *rocketmqx.Producer userOutboxProducer *rocketmqx.Producer
// userOutboxPartitions 缓存真实 App 列表并提供跨 worker 的公平轮转游标。
userOutboxPartitions *outboxpartition.Partitions
// roomConn 是低频平台治理调用 room-service 的 gRPC 连接。 // roomConn 是低频平台治理调用 room-service 的 gRPC 连接。
roomConn *grpc.ClientConn roomConn *grpc.ClientConn
// activityConn 是注册完成后触发注册奖励的 gRPC 连接。 // activityConn 是注册完成后触发注册奖励的 gRPC 连接。
@ -435,6 +438,7 @@ func New(cfg config.Config) (*App, error) {
mysqlRepo: mysqlRepo, mysqlRepo: mysqlRepo,
mqConsumers: mqConsumers, mqConsumers: mqConsumers,
userOutboxProducer: userOutboxProducer, userOutboxProducer: userOutboxProducer,
userOutboxPartitions: outboxpartition.New(cfg.OutboxWorker.AppCodes, outboxpartition.DefaultDiscoveryRefreshInterval),
roomConn: roomConn, roomConn: roomConn,
activityConn: activityConn, activityConn: activityConn,
walletConn: walletConn, walletConn: walletConn,
@ -539,7 +543,18 @@ func (a *App) runUserOutboxWorker() {
} }
func (a *App) processUserOutboxBatch(workerID string) (int, error) { func (a *App) processUserOutboxBatch(workerID string) (int, error) {
records, err := a.mysqlRepo.ClaimPendingUserOutbox(a.workers.Context(), workerID, a.cfg.OutboxWorker.BatchSize) ctx := a.workers.Context()
appCodes, discoveryErr := a.userOutboxPartitions.Resolve(ctx, a.mysqlRepo.ListUserOutboxAppCodes)
if discoveryErr != nil {
if len(appCodes) == 0 {
return 0, discoveryErr
}
// 发现刷新失败时沿用旧列表;缓存会在五分钟后再试,不阻断既有 App 的事实发布。
logx.Warn(ctx, "user_outbox_app_discovery_stale", slog.String("worker_id", workerID), slog.String("error", discoveryErr.Error()))
}
records, err := outboxpartition.ClaimFair(ctx, a.userOutboxPartitions.Order(appCodes), a.cfg.OutboxWorker.BatchSize, func(claimCtx context.Context, appCode string, limit int) ([]mysqlstorage.UserOutboxRecord, error) {
return a.mysqlRepo.ClaimPendingUserOutbox(appcode.WithContext(claimCtx, appCode), workerID, limit)
})
if err != nil { if err != nil {
return 0, err return 0, err
} }

View File

@ -10,6 +10,7 @@ import (
"hyapp/pkg/appcode" "hyapp/pkg/appcode"
"hyapp/pkg/configx" "hyapp/pkg/configx"
"hyapp/pkg/logx" "hyapp/pkg/logx"
"hyapp/pkg/outboxpartition"
"hyapp/pkg/tencentim" "hyapp/pkg/tencentim"
) )
@ -216,6 +217,8 @@ type CPIntimacyLeaderboardConfig struct {
// OutboxWorkerConfig 保存 user_outbox MQ 发布策略。 // OutboxWorkerConfig 保存 user_outbox MQ 发布策略。
type OutboxWorkerConfig struct { type OutboxWorkerConfig struct {
Enabled bool `yaml:"enabled"` Enabled bool `yaml:"enabled"`
// AppCodes 是可投递租户 allowlist空值使用低频数据库发现非空支持按 App 灰度回放。
AppCodes []string `yaml:"app_codes"`
PollInterval time.Duration `yaml:"poll_interval"` PollInterval time.Duration `yaml:"poll_interval"`
BatchSize int `yaml:"batch_size"` BatchSize int `yaml:"batch_size"`
PublishTimeout time.Duration `yaml:"publish_timeout"` PublishTimeout time.Duration `yaml:"publish_timeout"`
@ -492,6 +495,7 @@ func Load(path string) (Config, error) {
if cfg.OutboxWorker.PublishTimeout <= 0 { if cfg.OutboxWorker.PublishTimeout <= 0 {
cfg.OutboxWorker.PublishTimeout = 3 * time.Second cfg.OutboxWorker.PublishTimeout = 3 * time.Second
} }
cfg.OutboxWorker.AppCodes = outboxpartition.Normalize(cfg.OutboxWorker.AppCodes, false)
if cfg.OutboxWorker.Enabled && !cfg.RocketMQ.UserOutbox.Enabled { if cfg.OutboxWorker.Enabled && !cfg.RocketMQ.UserOutbox.Enabled {
return Config{}, errors.New("outbox_worker requires rocketmq.user_outbox.enabled") return Config{}, errors.New("outbox_worker requires rocketmq.user_outbox.enabled")
} }

View File

@ -0,0 +1,32 @@
package config
import (
"os"
"path/filepath"
"strings"
"testing"
)
func TestOutboxWorkerAppCodesNormalizeForStagedReplay(t *testing.T) {
path := filepath.Join(t.TempDir(), "config.yaml")
if err := os.WriteFile(path, []byte("outbox_worker:\n app_codes: [' LALU ', fami, HUWAA, fami]\n"), 0o600); err != nil {
t.Fatalf("write config: %v", err)
}
cfg, err := Load(path)
if err != nil {
t.Fatalf("load staged outbox config: %v", err)
}
if got := strings.Join(cfg.OutboxWorker.AppCodes, ","); got != "lalu,fami,huwaa" {
t.Fatalf("normalized outbox app_codes = %q", got)
}
}
func TestLocalOutboxWorkerEmptyAppCodesUsesDiscovery(t *testing.T) {
cfg, err := Load("../../configs/config.yaml")
if err != nil {
t.Fatalf("load local config: %v", err)
}
if len(cfg.OutboxWorker.AppCodes) != 0 {
t.Fatalf("local empty app_codes must retain automatic discovery: %v", cfg.OutboxWorker.AppCodes)
}
}

View File

@ -25,6 +25,29 @@ type UserOutboxRecord struct {
LockUntilMS int64 LockUntilMS int64
} }
// ListUserOutboxAppCodes enumerates real tenant keys from the owner table.
// PRIMARY begins with app_code, and the worker caches this discovery for five
// minutes; normal claims continue to use status/lock indexes within one App.
func (r *Repository) ListUserOutboxAppCodes(ctx context.Context) ([]string, error) {
if r == nil || r.db == nil {
return nil, xerr.New(xerr.Unavailable, "mysql repository is not configured")
}
rows, err := r.db.QueryContext(ctx, `SELECT DISTINCT app_code FROM user_outbox FORCE INDEX (PRIMARY) ORDER BY app_code`)
if err != nil {
return nil, err
}
defer rows.Close()
appCodes := make([]string, 0, 4)
for rows.Next() {
var appCode string
if err := rows.Scan(&appCode); err != nil {
return nil, err
}
appCodes = append(appCodes, appCode)
}
return appCodes, rows.Err()
}
// ClaimPendingUserOutbox locks pending user facts so one worker can publish them. // ClaimPendingUserOutbox locks pending user facts so one worker can publish them.
func (r *Repository) ClaimPendingUserOutbox(ctx context.Context, workerID string, batchSize int) ([]UserOutboxRecord, error) { func (r *Repository) ClaimPendingUserOutbox(ctx context.Context, workerID string, batchSize int) ([]UserOutboxRecord, error) {
if r == nil || r.db == nil { if r == nil || r.db == nil {

View File

@ -0,0 +1,30 @@
package mysql
import (
"context"
"regexp"
"testing"
"github.com/DATA-DOG/go-sqlmock"
)
func TestListUserOutboxAppCodesForcesPrimaryPrefixIndex(t *testing.T) {
db, mock, err := sqlmock.New()
if err != nil {
t.Fatalf("create sqlmock: %v", err)
}
defer db.Close()
mock.ExpectQuery(regexp.QuoteMeta(`SELECT DISTINCT app_code FROM user_outbox FORCE INDEX (PRIMARY) ORDER BY app_code`)).
WillReturnRows(sqlmock.NewRows([]string{"app_code"}).AddRow("fami").AddRow("huwaa").AddRow("lalu"))
apps, err := (&Repository{db: db}).ListUserOutboxAppCodes(context.Background())
if err != nil {
t.Fatalf("list user outbox apps: %v", err)
}
if len(apps) != 3 || apps[0] != "fami" || apps[1] != "huwaa" || apps[2] != "lalu" {
t.Fatalf("user outbox apps mismatch: %v", apps)
}
if err := mock.ExpectationsWereMet(); err != nil {
t.Fatalf("user app discovery query mismatch: %v", err)
}
}

View File

@ -150,6 +150,7 @@ external_recharge:
v5pay_return_url: "https://h5.global-interaction.com/recharge/index.html?env=test" v5pay_return_url: "https://h5.global-interaction.com/recharge/index.html?env=test"
outbox_worker: outbox_worker:
enabled: true enabled: true
app_codes: []
poll_interval: "1s" poll_interval: "1s"
batch_size: 200 batch_size: 200
concurrency: 4 concurrency: 4
@ -159,6 +160,7 @@ outbox_worker:
max_backoff: "5m" max_backoff: "5m"
realtime_outbox_worker: realtime_outbox_worker:
enabled: true enabled: true
app_codes: []
poll_interval: "500ms" poll_interval: "500ms"
batch_size: 50 batch_size: 50
concurrency: 2 concurrency: 2

View File

@ -150,6 +150,8 @@ external_recharge:
v5pay_return_url: "https://h5.global-interaction.com/recharge/index.html" v5pay_return_url: "https://h5.global-interaction.com/recharge/index.html"
outbox_worker: outbox_worker:
enabled: true enabled: true
# 空值自动发现全部 App历史积压回放时先设 ["lalu", "fami"],核对后再加入 "huwaa"。
app_codes: []
poll_interval: "1s" poll_interval: "1s"
batch_size: 200 batch_size: 200
concurrency: 8 concurrency: 8
@ -159,6 +161,8 @@ outbox_worker:
max_backoff: "5m" max_backoff: "5m"
realtime_outbox_worker: realtime_outbox_worker:
enabled: true enabled: true
# 实时 lane 必须与普通 lane 使用同一阶段 allowlist避免同表出现跨 App 抢占差异。
app_codes: []
poll_interval: "500ms" poll_interval: "500ms"
batch_size: 50 batch_size: 50
concurrency: 4 concurrency: 4

View File

@ -153,6 +153,7 @@ external_recharge_reconcile_worker:
batch_size: 50 batch_size: 50
outbox_worker: outbox_worker:
enabled: true enabled: true
app_codes: []
poll_interval: "1s" poll_interval: "1s"
batch_size: 100 batch_size: 100
concurrency: 1 concurrency: 1
@ -162,6 +163,7 @@ outbox_worker:
max_backoff: "5m" max_backoff: "5m"
realtime_outbox_worker: realtime_outbox_worker:
enabled: true enabled: true
app_codes: []
poll_interval: "500ms" poll_interval: "500ms"
batch_size: 50 batch_size: 50
concurrency: 2 concurrency: 2

View File

@ -13,6 +13,7 @@ import (
"hyapp/pkg/grpchealth" "hyapp/pkg/grpchealth"
"hyapp/pkg/healthhttp" "hyapp/pkg/healthhttp"
"hyapp/pkg/logx" "hyapp/pkg/logx"
"hyapp/pkg/outboxpartition"
"hyapp/pkg/rocketmqx" "hyapp/pkg/rocketmqx"
servicegrpc "hyapp/pkg/servicekit/grpcserver" servicegrpc "hyapp/pkg/servicekit/grpcserver"
servicehealth "hyapp/pkg/servicekit/health" servicehealth "hyapp/pkg/servicekit/health"
@ -43,6 +44,9 @@ type App struct {
projectionConsumer *rocketmqx.Consumer projectionConsumer *rocketmqx.Consumer
outboxWorkerCfg config.OutboxWorkerConfig outboxWorkerCfg config.OutboxWorkerConfig
realtimeOutboxWorkerCfg config.OutboxWorkerConfig realtimeOutboxWorkerCfg config.OutboxWorkerConfig
// 普通与实时 lane 共享同一张表但可独立配置事件类型;各自维护 App 轮转游标,避免一个 lane 的吞吐改变另一个 lane 的公平性。
outboxPartitions *outboxpartition.Partitions
realtimeOutboxPartitions *outboxpartition.Partitions
projectionWorkerCfg config.ProjectionWorkerConfig projectionWorkerCfg config.ProjectionWorkerConfig
googlePaidSyncWorkerCfg config.GooglePaidSyncWorkerConfig googlePaidSyncWorkerCfg config.GooglePaidSyncWorkerConfig
externalRechargeReconcileWorkerCfg config.ExternalRechargeReconcileWorkerConfig externalRechargeReconcileWorkerCfg config.ExternalRechargeReconcileWorkerConfig
@ -204,6 +208,8 @@ func New(cfg config.Config) (*App, error) {
projectionConsumer: projectionConsumer, projectionConsumer: projectionConsumer,
outboxWorkerCfg: cfg.OutboxWorker, outboxWorkerCfg: cfg.OutboxWorker,
realtimeOutboxWorkerCfg: cfg.RealtimeOutboxWorker, realtimeOutboxWorkerCfg: cfg.RealtimeOutboxWorker,
outboxPartitions: outboxpartition.New(cfg.OutboxWorker.AppCodes, outboxpartition.DefaultDiscoveryRefreshInterval),
realtimeOutboxPartitions: outboxpartition.New(cfg.RealtimeOutboxWorker.AppCodes, outboxpartition.DefaultDiscoveryRefreshInterval),
projectionWorkerCfg: cfg.ProjectionWorker, projectionWorkerCfg: cfg.ProjectionWorker,
googlePaidSyncWorkerCfg: googlePaidSyncWorkerCfg, googlePaidSyncWorkerCfg: googlePaidSyncWorkerCfg,
externalRechargeReconcileWorkerCfg: cfg.ExternalRechargeReconcileWorker, externalRechargeReconcileWorkerCfg: cfg.ExternalRechargeReconcileWorker,

View File

@ -14,10 +14,10 @@ func (a *App) runBackgroundWorkers() {
// 普通账务 worker 在实时通道可用时跳过红包 UI 事件,避免两个 worker 抢同一行。 // 普通账务 worker 在实时通道可用时跳过红包 UI 事件,避免两个 worker 抢同一行。
excludedRealtimeTypes = a.realtimeOutboxWorkerCfg.EventTypes excludedRealtimeTypes = a.realtimeOutboxWorkerCfg.EventTypes
} }
a.startWalletOutboxWorkers(ctx, "wallet-outbox", a.outboxWorkerCfg, a.outboxProducer, a.walletOutboxTopic, nil, excludedRealtimeTypes) a.startWalletOutboxWorkers(ctx, "wallet-outbox", a.outboxWorkerCfg, a.outboxPartitions, a.outboxProducer, a.walletOutboxTopic, nil, excludedRealtimeTypes)
} }
if a.realtimeOutboxWorkerCfg.Enabled && a.realtimeOutboxProducer != nil { if a.realtimeOutboxWorkerCfg.Enabled && a.realtimeOutboxProducer != nil {
a.startWalletOutboxWorkers(ctx, "wallet-realtime-outbox", a.realtimeOutboxWorkerCfg, a.realtimeOutboxProducer, a.realtimeWalletOutboxTopic, a.realtimeOutboxWorkerCfg.EventTypes, nil) a.startWalletOutboxWorkers(ctx, "wallet-realtime-outbox", a.realtimeOutboxWorkerCfg, a.realtimeOutboxPartitions, a.realtimeOutboxProducer, a.realtimeWalletOutboxTopic, a.realtimeOutboxWorkerCfg.EventTypes, nil)
} }
if a.externalRechargeReconcileWorkerCfg.Enabled { if a.externalRechargeReconcileWorkerCfg.Enabled {
a.startExternalRechargeReconcileWorker(ctx) a.startExternalRechargeReconcileWorker(ctx)

View File

@ -10,13 +10,14 @@ import (
"hyapp/pkg/appcode" "hyapp/pkg/appcode"
"hyapp/pkg/logx" "hyapp/pkg/logx"
"hyapp/pkg/outboxpartition"
"hyapp/pkg/rocketmqx" "hyapp/pkg/rocketmqx"
"hyapp/pkg/walletmq" "hyapp/pkg/walletmq"
"hyapp/services/wallet-service/internal/config" "hyapp/services/wallet-service/internal/config"
mysqlstorage "hyapp/services/wallet-service/internal/storage/mysql" mysqlstorage "hyapp/services/wallet-service/internal/storage/mysql"
) )
func (a *App) startWalletOutboxWorkers(ctx context.Context, workerName string, options config.OutboxWorkerConfig, producer *rocketmqx.Producer, topic string, includeEventTypes []string, excludeEventTypes []string) { func (a *App) startWalletOutboxWorkers(ctx context.Context, workerName string, options config.OutboxWorkerConfig, partitions *outboxpartition.Partitions, producer *rocketmqx.Producer, topic string, includeEventTypes []string, excludeEventTypes []string) {
if options.Concurrency <= 0 { if options.Concurrency <= 0 {
options.Concurrency = 1 options.Concurrency = 1
} }
@ -25,16 +26,16 @@ func (a *App) startWalletOutboxWorkers(ctx context.Context, workerName string, o
a.workers.Add(1) a.workers.Add(1)
go func() { go func() {
defer a.workers.Done() defer a.workers.Done()
a.runWalletOutboxWorker(ctx, workerID, options, producer, topic, includeEventTypes, excludeEventTypes) a.runWalletOutboxWorker(ctx, workerID, options, partitions, producer, topic, includeEventTypes, excludeEventTypes)
}() }()
} }
} }
func (a *App) runWalletOutboxWorker(ctx context.Context, workerID string, options config.OutboxWorkerConfig, producer *rocketmqx.Producer, topic string, includeEventTypes []string, excludeEventTypes []string) { func (a *App) runWalletOutboxWorker(ctx context.Context, workerID string, options config.OutboxWorkerConfig, partitions *outboxpartition.Partitions, producer *rocketmqx.Producer, topic string, includeEventTypes []string, excludeEventTypes []string) {
ticker := time.NewTicker(options.PollInterval) ticker := time.NewTicker(options.PollInterval)
defer ticker.Stop() defer ticker.Stop()
for { for {
processed, err := a.processWalletOutboxBatch(ctx, workerID, options, producer, topic, includeEventTypes, excludeEventTypes) processed, err := a.processWalletOutboxBatch(ctx, workerID, options, partitions, producer, topic, includeEventTypes, excludeEventTypes)
if err != nil && !errors.Is(err, context.Canceled) { if err != nil && !errors.Is(err, context.Canceled) {
logx.Error(ctx, "wallet_outbox_publish_failed", err, slog.String("worker_id", workerID)) logx.Error(ctx, "wallet_outbox_publish_failed", err, slog.String("worker_id", workerID))
} }
@ -49,19 +50,32 @@ func (a *App) runWalletOutboxWorker(ctx context.Context, workerID string, option
} }
} }
func (a *App) processWalletOutboxBatch(ctx context.Context, workerID string, options config.OutboxWorkerConfig, producer *rocketmqx.Producer, topic string, includeEventTypes []string, excludeEventTypes []string) (int, error) { func (a *App) processWalletOutboxBatch(ctx context.Context, workerID string, options config.OutboxWorkerConfig, partitions *outboxpartition.Partitions, producer *rocketmqx.Producer, topic string, includeEventTypes []string, excludeEventTypes []string) (int, error) {
if a.mysqlRepo == nil || producer == nil { if a.mysqlRepo == nil || producer == nil {
return 0, nil return 0, nil
} }
appCodes, discoveryErr := partitions.Resolve(ctx, a.mysqlRepo.ListWalletOutboxAppCodes)
if discoveryErr != nil {
if len(appCodes) == 0 {
return 0, discoveryErr
}
// 已知 App 继续投递,刷新失败只影响新租户发现;缓存层会延后下一次查询,避免热循环打 MySQL。
logx.Warn(ctx, "wallet_outbox_app_discovery_stale", slog.String("worker_id", workerID), slog.String("error", discoveryErr.Error()))
}
if len(appCodes) == 0 {
return 0, nil
}
processed := 0 processed := 0
for processed < options.BatchSize { for processed < options.BatchSize {
// Sync publish is intentionally claimed one record at a time. Claiming an entire // Sync publish is intentionally claimed one record at a time. Claiming an entire
// batch with one short lock lets later rows expire while earlier SendSync calls // batch with one short lock lets later rows expire while earlier SendSync calls
// are still running, so another replica can publish the same tail concurrently. // are still running, so another replica can publish the same tail concurrently.
records, err := a.mysqlRepo.ClaimPendingWalletOutboxFiltered(ctx, workerID, 1, time.Now().UTC().Add(walletOutboxSingleRecordLease(options.PublishTimeout)).UnixMilli(), mysqlstorage.WalletOutboxClaimFilter{ records, err := outboxpartition.ClaimFair(ctx, partitions.Order(appCodes), 1, func(claimCtx context.Context, appCode string, limit int) ([]mysqlstorage.WalletOutboxRecord, error) {
return a.mysqlRepo.ClaimPendingWalletOutboxFiltered(appcode.WithContext(claimCtx, appCode), workerID, limit, time.Now().UTC().Add(walletOutboxSingleRecordLease(options.PublishTimeout)).UnixMilli(), mysqlstorage.WalletOutboxClaimFilter{
IncludeEventTypes: includeEventTypes, IncludeEventTypes: includeEventTypes,
ExcludeEventTypes: excludeEventTypes, ExcludeEventTypes: excludeEventTypes,
}) })
})
if err != nil { if err != nil {
return processed, err return processed, err
} }

View File

@ -9,6 +9,7 @@ import (
"hyapp/pkg/configx" "hyapp/pkg/configx"
"hyapp/pkg/logx" "hyapp/pkg/logx"
"hyapp/pkg/outboxpartition"
) )
// Config 描述 wallet-service 启动配置。 // Config 描述 wallet-service 启动配置。
@ -103,6 +104,8 @@ type WalletOutboxMQConfig struct {
// OutboxWorkerConfig 控制 wallet_outbox pending 事件的 MQ fanout。 // OutboxWorkerConfig 控制 wallet_outbox pending 事件的 MQ fanout。
type OutboxWorkerConfig struct { type OutboxWorkerConfig struct {
Enabled bool `yaml:"enabled"` Enabled bool `yaml:"enabled"`
// AppCodes 是可投递租户 allowlist空值才自动发现便于积压回放按 App 分阶段放量。
AppCodes []string `yaml:"app_codes"`
PollInterval time.Duration `yaml:"poll_interval"` PollInterval time.Duration `yaml:"poll_interval"`
BatchSize int `yaml:"batch_size"` BatchSize int `yaml:"batch_size"`
Concurrency int `yaml:"concurrency"` Concurrency int `yaml:"concurrency"`
@ -820,6 +823,7 @@ func normalizeRocketMQConfig(cfg RocketMQConfig) (RocketMQConfig, error) {
} }
func normalizeOutboxWorkerConfig(cfg OutboxWorkerConfig, defaults OutboxWorkerConfig) OutboxWorkerConfig { func normalizeOutboxWorkerConfig(cfg OutboxWorkerConfig, defaults OutboxWorkerConfig) OutboxWorkerConfig {
cfg.AppCodes = outboxpartition.Normalize(cfg.AppCodes, false)
if cfg.PollInterval <= 0 { if cfg.PollInterval <= 0 {
cfg.PollInterval = defaults.PollInterval cfg.PollInterval = defaults.PollInterval
} }

View File

@ -31,11 +31,21 @@ func TestLoadLocalEnablesOutboxMQ(t *testing.T) {
if cfg.OutboxWorker.Concurrency != 1 || cfg.RealtimeOutboxWorker.Concurrency != 2 { if cfg.OutboxWorker.Concurrency != 1 || cfg.RealtimeOutboxWorker.Concurrency != 2 {
t.Fatalf("local outbox worker defaults mismatch: outbox=%+v realtime=%+v", cfg.OutboxWorker, cfg.RealtimeOutboxWorker) t.Fatalf("local outbox worker defaults mismatch: outbox=%+v realtime=%+v", cfg.OutboxWorker, cfg.RealtimeOutboxWorker)
} }
if len(cfg.OutboxWorker.AppCodes) != 0 || len(cfg.RealtimeOutboxWorker.AppCodes) != 0 {
t.Fatalf("empty app_codes must retain automatic discovery: outbox=%v realtime=%v", cfg.OutboxWorker.AppCodes, cfg.RealtimeOutboxWorker.AppCodes)
}
if cfg.BSCUSDT.Enabled || cfg.ExternalRecharge.USDTBEP20Enabled { if cfg.BSCUSDT.Enabled || cfg.ExternalRecharge.USDTBEP20Enabled {
t.Fatalf("bsc usdt receipt verification must stay disabled by default: bsc=%+v external=%+v", cfg.BSCUSDT, cfg.ExternalRecharge) t.Fatalf("bsc usdt receipt verification must stay disabled by default: bsc=%+v external=%+v", cfg.BSCUSDT, cfg.ExternalRecharge)
} }
} }
func TestOutboxWorkerAppCodesNormalizeForStagedReplay(t *testing.T) {
cfg := normalizeOutboxWorkerConfig(OutboxWorkerConfig{AppCodes: []string{" LALU ", "fami", "HUWAA", "fami"}}, Default().OutboxWorker)
if got := strings.Join(cfg.AppCodes, ","); got != "lalu,fami,huwaa" {
t.Fatalf("normalized outbox app_codes = %q", got)
}
}
func TestDefaultWalletEventTagModeIsLegacy(t *testing.T) { func TestDefaultWalletEventTagModeIsLegacy(t *testing.T) {
if got := Default().RocketMQ.WalletEventTagMode; got != WalletEventTagModeLegacy { if got := Default().RocketMQ.WalletEventTagMode; got != WalletEventTagModeLegacy {
t.Fatalf("default wallet event tag mode = %q, want %q", got, WalletEventTagModeLegacy) t.Fatalf("default wallet event tag mode = %q, want %q", got, WalletEventTagModeLegacy)

View File

@ -2385,6 +2385,45 @@ func TestCreditLuckyGiftRewardIsIdempotent(t *testing.T) {
} }
} }
// TestCreditWheelRewardReconciliationReturnsExistingTransaction 验证 Activity 补偿重复提交稳定 draw command 时,
// 钱包先按 command_id + request hash 找回既有交易和原回执,不会再次增加余额或重复生成 outbox。
func TestCreditWheelRewardReconciliationReturnsExistingTransaction(t *testing.T) {
repository := mysqltest.NewRepository(t)
svc := walletservice.New(repository)
command := ledger.WheelRewardCommand{
AppCode: "fami",
CommandID: "wheel_reward:wheel_draw_reconcile_1",
TargetUserID: 21102,
Amount: 288,
DrawID: "wheel_draw_reconcile_1",
WheelID: "classic",
SelectedTierID: "classic-coin-288",
VisibleRegionID: 10,
Reason: "wheel_reward",
}
first, err := svc.CreditWheelReward(context.Background(), command)
if err != nil {
t.Fatalf("CreditWheelReward failed: %v", err)
}
second, err := svc.CreditWheelReward(context.Background(), command)
if err != nil {
t.Fatalf("CreditWheelReward reconciliation retry failed: %v", err)
}
if first.TransactionID == "" || first.TransactionID != second.TransactionID || second.Balance.AvailableAmount != 288 {
t.Fatalf("wheel reward reconciliation receipt mismatch: first=%+v second=%+v", first, second)
}
if got := repository.CountRows("wallet_transactions", "app_code = ? AND command_id = ? AND biz_type = ?", "fami", command.CommandID, "wheel_reward"); got != 1 {
t.Fatalf("wheel reward reconciliation must keep one transaction, got %d", got)
}
if got := repository.CountRows("wallet_outbox", "app_code = ? AND transaction_id = ? AND event_type = ?", "fami", first.TransactionID, "WalletBalanceChanged"); got != 1 {
t.Fatalf("wheel reward reconciliation must keep one balance event, got %d", got)
}
if got := repository.CountRows("wallet_outbox", "app_code = ? AND transaction_id = ? AND event_type = ?", "fami", first.TransactionID, "WalletWheelRewardCredited"); got != 1 {
t.Fatalf("wheel reward reconciliation must keep one reward fact, got %d", got)
}
}
// TestTransferCoinFromSellerMovesDedicatedAssetToPlayerCoin 验证币商专用金币和玩家普通金币不会混账。 // TestTransferCoinFromSellerMovesDedicatedAssetToPlayerCoin 验证币商专用金币和玩家普通金币不会混账。
func TestTransferCoinFromSellerMovesDedicatedAssetToPlayerCoin(t *testing.T) { func TestTransferCoinFromSellerMovesDedicatedAssetToPlayerCoin(t *testing.T) {
repository := mysqltest.NewRepository(t) repository := mysqltest.NewRepository(t)

View File

@ -84,6 +84,29 @@ func (r *Repository) insertWalletOutbox(ctx context.Context, tx *sql.Tx, events
return nil return nil
} }
// ListWalletOutboxAppCodes enumerates the tenant key stored in this owner table.
// PRIMARY starts with app_code, so MySQL can use a loose ordered index scan;
// callers cache the result and never run this discovery query on every poll.
func (r *Repository) ListWalletOutboxAppCodes(ctx context.Context) ([]string, error) {
if r == nil || r.db == nil {
return nil, xerr.New(xerr.Unavailable, "mysql repository is not configured")
}
rows, err := r.db.QueryContext(ctx, `SELECT DISTINCT app_code FROM wallet_outbox FORCE INDEX (PRIMARY) ORDER BY app_code`)
if err != nil {
return nil, err
}
defer rows.Close()
appCodes := make([]string, 0, 4)
for rows.Next() {
var appCode string
if err := rows.Scan(&appCode); err != nil {
return nil, err
}
appCodes = append(appCodes, appCode)
}
return appCodes, rows.Err()
}
// ClaimPendingWalletOutbox atomically claims committed wallet facts for MQ fanout. // ClaimPendingWalletOutbox atomically claims committed wallet facts for MQ fanout.
func (r *Repository) ClaimPendingWalletOutbox(ctx context.Context, workerID string, limit int, lockUntilMS int64) ([]WalletOutboxRecord, error) { func (r *Repository) ClaimPendingWalletOutbox(ctx context.Context, workerID string, limit int, lockUntilMS int64) ([]WalletOutboxRecord, error) {
return r.ClaimPendingWalletOutboxFiltered(ctx, workerID, limit, lockUntilMS, WalletOutboxClaimFilter{}) return r.ClaimPendingWalletOutboxFiltered(ctx, workerID, limit, lockUntilMS, WalletOutboxClaimFilter{})

View File

@ -0,0 +1,30 @@
package mysql
import (
"context"
"regexp"
"testing"
"github.com/DATA-DOG/go-sqlmock"
)
func TestListWalletOutboxAppCodesForcesPrimaryPrefixIndex(t *testing.T) {
db, mock, err := sqlmock.New()
if err != nil {
t.Fatalf("create sqlmock: %v", err)
}
defer db.Close()
mock.ExpectQuery(regexp.QuoteMeta(`SELECT DISTINCT app_code FROM wallet_outbox FORCE INDEX (PRIMARY) ORDER BY app_code`)).
WillReturnRows(sqlmock.NewRows([]string{"app_code"}).AddRow("fami").AddRow("huwaa").AddRow("lalu"))
apps, err := (&Repository{db: db}).ListWalletOutboxAppCodes(context.Background())
if err != nil {
t.Fatalf("list wallet outbox apps: %v", err)
}
if len(apps) != 3 || apps[0] != "fami" || apps[1] != "huwaa" || apps[2] != "lalu" {
t.Fatalf("wallet outbox apps mismatch: %v", apps)
}
if err := mock.ExpectationsWereMet(); err != nil {
t.Fatalf("wallet app discovery query mismatch: %v", err)
}
}