修复多应用异步回放和生产异常
This commit is contained in:
parent
1d581ff493
commit
f0c4df7aa3
68
pkg/apptracking/limits.go
Normal file
68
pkg/apptracking/limits.go
Normal file
@ -0,0 +1,68 @@
|
|||||||
|
// Package apptracking defines the transport and persistence limits shared by the
|
||||||
|
// public App tracking endpoint and statistics-service.
|
||||||
|
package apptracking
|
||||||
|
|
||||||
|
import (
|
||||||
|
"bytes"
|
||||||
|
"encoding/json"
|
||||||
|
"strings"
|
||||||
|
)
|
||||||
|
|
||||||
|
const (
|
||||||
|
// MaxBatchSize keeps one request bounded before gateway fans it into the
|
||||||
|
// statistics write transaction.
|
||||||
|
MaxBatchSize = 50
|
||||||
|
// MaxPropertiesBytes bounds the raw JSON value after surrounding whitespace
|
||||||
|
// is removed. The value is rejected instead of truncated because truncation
|
||||||
|
// can change JSON meaning or produce invalid JSON.
|
||||||
|
MaxPropertiesBytes = 4096
|
||||||
|
|
||||||
|
// Text limits are byte limits, not rune limits. They mirror the durable
|
||||||
|
// app_tracking_events columns and therefore apply identically at both service
|
||||||
|
// boundaries; an oversized client value must never be deferred to MySQL.
|
||||||
|
MaxAppCodeBytes = 32
|
||||||
|
MaxEventIDBytes = 160
|
||||||
|
MaxEventNameBytes = 96
|
||||||
|
MaxEventTypeBytes = 64
|
||||||
|
MaxScreenBytes = 128
|
||||||
|
MaxTargetTypeBytes = 64
|
||||||
|
MaxTargetIDBytes = 128
|
||||||
|
MaxDeviceIDBytes = 128
|
||||||
|
MaxSessionIDBytes = 160
|
||||||
|
MaxPlatformBytes = 32
|
||||||
|
MaxAppVersionBytes = 64
|
||||||
|
MaxLanguageBytes = 32
|
||||||
|
MaxTimezoneBytes = 64
|
||||||
|
MaxErrorCodeBytes = 64
|
||||||
|
)
|
||||||
|
|
||||||
|
// ValidBatchSize rejects both empty reports and reports large enough to hold a
|
||||||
|
// statistics transaction open for an unbounded period.
|
||||||
|
func ValidBatchSize(size int) bool {
|
||||||
|
return size > 0 && size <= MaxBatchSize
|
||||||
|
}
|
||||||
|
|
||||||
|
// NormalizeText removes transport-only surrounding whitespace and checks the
|
||||||
|
// encoded byte length. Oversized values return false and are never shortened:
|
||||||
|
// identifiers and error codes are facts whose suffix may be significant.
|
||||||
|
func NormalizeText(value string, maxBytes int) (string, bool) {
|
||||||
|
value = strings.TrimSpace(value)
|
||||||
|
if maxBytes > 0 && len(value) > maxBytes {
|
||||||
|
return "", false
|
||||||
|
}
|
||||||
|
return value, true
|
||||||
|
}
|
||||||
|
|
||||||
|
// NormalizeProperties returns a detached, trimmed JSON value. Empty and JSON
|
||||||
|
// null both mean no properties; non-empty values must satisfy the same byte and
|
||||||
|
// syntax contract in gateway and statistics-service.
|
||||||
|
func NormalizeProperties(raw json.RawMessage) (json.RawMessage, bool) {
|
||||||
|
trimmed := bytes.TrimSpace(raw)
|
||||||
|
if len(trimmed) == 0 || bytes.Equal(trimmed, []byte("null")) {
|
||||||
|
return nil, true
|
||||||
|
}
|
||||||
|
if len(trimmed) > MaxPropertiesBytes || !json.Valid(trimmed) {
|
||||||
|
return nil, false
|
||||||
|
}
|
||||||
|
return json.RawMessage(bytes.Clone(trimmed)), true
|
||||||
|
}
|
||||||
36
pkg/apptracking/limits_test.go
Normal file
36
pkg/apptracking/limits_test.go
Normal file
@ -0,0 +1,36 @@
|
|||||||
|
package apptracking
|
||||||
|
|
||||||
|
import (
|
||||||
|
"encoding/json"
|
||||||
|
"strings"
|
||||||
|
"testing"
|
||||||
|
)
|
||||||
|
|
||||||
|
func TestNormalizeTextUsesBytesAndNeverTruncates(t *testing.T) {
|
||||||
|
if got, ok := NormalizeText(" "+strings.Repeat("e", MaxErrorCodeBytes)+" ", MaxErrorCodeBytes); !ok || got != strings.Repeat("e", MaxErrorCodeBytes) {
|
||||||
|
t.Fatalf("text at byte limit must be trimmed but otherwise preserved: ok=%v got=%q", ok, got)
|
||||||
|
}
|
||||||
|
if got, ok := NormalizeText(strings.Repeat("e", 185), MaxErrorCodeBytes); ok || got != "" {
|
||||||
|
t.Fatalf("oversized error_code must be rejected, never truncated: ok=%v got=%q", ok, got)
|
||||||
|
}
|
||||||
|
// 一个汉字编码为三个 UTF-8 bytes;这里防止调用方误把字段限制实现成 rune 数。
|
||||||
|
if got, ok := NormalizeText(strings.Repeat("界", 22), MaxErrorCodeBytes); ok || got != "" {
|
||||||
|
t.Fatalf("text limits must count UTF-8 bytes: ok=%v got=%q", ok, got)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestNormalizePropertiesAndBatchContract(t *testing.T) {
|
||||||
|
if !ValidBatchSize(1) || !ValidBatchSize(MaxBatchSize) || ValidBatchSize(0) || ValidBatchSize(MaxBatchSize+1) {
|
||||||
|
t.Fatalf("batch bounds do not match the shared contract")
|
||||||
|
}
|
||||||
|
got, ok := NormalizeProperties(json.RawMessage(` {"screen":"home"} `))
|
||||||
|
if !ok || string(got) != `{"screen":"home"}` {
|
||||||
|
t.Fatalf("valid properties mismatch: ok=%v got=%s", ok, got)
|
||||||
|
}
|
||||||
|
if got, ok := NormalizeProperties(json.RawMessage(`"` + strings.Repeat("x", MaxPropertiesBytes) + `"`)); ok || got != nil {
|
||||||
|
t.Fatalf("properties over the encoded byte limit must be rejected: ok=%v bytes=%d", ok, len(got))
|
||||||
|
}
|
||||||
|
if got, ok := NormalizeProperties(json.RawMessage(`{"broken"`)); ok || got != nil {
|
||||||
|
t.Fatalf("invalid properties JSON must be rejected: ok=%v got=%s", ok, got)
|
||||||
|
}
|
||||||
|
}
|
||||||
163
pkg/outboxpartition/partitions.go
Normal file
163
pkg/outboxpartition/partitions.go
Normal file
@ -0,0 +1,163 @@
|
|||||||
|
// Package outboxpartition provides the shared tenant scheduling rules used by
|
||||||
|
// owner-service outbox workers. It does not read or publish business facts;
|
||||||
|
// repositories retain ownership of indexed claims and delivery state.
|
||||||
|
package outboxpartition
|
||||||
|
|
||||||
|
import (
|
||||||
|
"context"
|
||||||
|
"fmt"
|
||||||
|
"sort"
|
||||||
|
"strings"
|
||||||
|
"sync"
|
||||||
|
"sync/atomic"
|
||||||
|
"time"
|
||||||
|
|
||||||
|
"hyapp/pkg/appcode"
|
||||||
|
)
|
||||||
|
|
||||||
|
const DefaultDiscoveryRefreshInterval = 5 * time.Minute
|
||||||
|
|
||||||
|
// Partitions resolves the App partitions visible to one outbox lane and rotates
|
||||||
|
// the first partition across claims. A configured allowlist is immutable for the
|
||||||
|
// process lifetime; an empty allowlist falls back to low-frequency DB discovery.
|
||||||
|
type Partitions struct {
|
||||||
|
configured []string
|
||||||
|
refreshInterval time.Duration
|
||||||
|
|
||||||
|
mu sync.Mutex
|
||||||
|
cached []string
|
||||||
|
hasCache bool
|
||||||
|
nextRefresh time.Time
|
||||||
|
cursor atomic.Uint64
|
||||||
|
}
|
||||||
|
|
||||||
|
// New creates a partition scheduler. Operators can stage a backlog safely by
|
||||||
|
// configuring only the Apps currently allowed to publish; an empty list keeps
|
||||||
|
// automatic discovery as the compatibility default for existing deployments.
|
||||||
|
func New(configured []string, refreshInterval time.Duration) *Partitions {
|
||||||
|
if refreshInterval <= 0 {
|
||||||
|
refreshInterval = DefaultDiscoveryRefreshInterval
|
||||||
|
}
|
||||||
|
return &Partitions{
|
||||||
|
configured: Normalize(configured, false),
|
||||||
|
refreshInterval: refreshInterval,
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// Resolve returns configured Apps without touching MySQL. In discovery mode it
|
||||||
|
// refreshes at most once per interval across all local worker goroutines. A
|
||||||
|
// failed refresh returns the last successful list alongside the error so the
|
||||||
|
// caller can log the stale condition without stopping already-known partitions.
|
||||||
|
func (p *Partitions) Resolve(ctx context.Context, discover func(context.Context) ([]string, error)) ([]string, error) {
|
||||||
|
if p == nil {
|
||||||
|
return nil, nil
|
||||||
|
}
|
||||||
|
if len(p.configured) > 0 {
|
||||||
|
return append([]string(nil), p.configured...), nil
|
||||||
|
}
|
||||||
|
if discover == nil {
|
||||||
|
return nil, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
p.mu.Lock()
|
||||||
|
defer p.mu.Unlock()
|
||||||
|
now := time.Now().UTC()
|
||||||
|
if p.hasCache && now.Before(p.nextRefresh) {
|
||||||
|
return append([]string(nil), p.cached...), nil
|
||||||
|
}
|
||||||
|
|
||||||
|
discovered, err := discover(ctx)
|
||||||
|
if err != nil {
|
||||||
|
// Move the next attempt out by the normal interval: a transient database
|
||||||
|
// error must not turn every hot poll into a DISTINCT query stampede.
|
||||||
|
p.nextRefresh = now.Add(p.refreshInterval)
|
||||||
|
if p.hasCache {
|
||||||
|
return append([]string(nil), p.cached...), err
|
||||||
|
}
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
p.cached = Normalize(discovered, true)
|
||||||
|
p.hasCache = true
|
||||||
|
p.nextRefresh = now.Add(p.refreshInterval)
|
||||||
|
return append([]string(nil), p.cached...), nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// Order rotates the first App atomically while preserving the remaining cyclic
|
||||||
|
// order. Multiple worker goroutines therefore share one fair cursor instead of
|
||||||
|
// independently preferring the first configured tenant.
|
||||||
|
func (p *Partitions) Order(appCodes []string) []string {
|
||||||
|
if len(appCodes) == 0 {
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
if p == nil {
|
||||||
|
return append([]string(nil), appCodes...)
|
||||||
|
}
|
||||||
|
start := int((p.cursor.Add(1) - 1) % uint64(len(appCodes)))
|
||||||
|
ordered := make([]string, 0, len(appCodes))
|
||||||
|
for offset := 0; offset < len(appCodes); offset++ {
|
||||||
|
ordered = append(ordered, appCodes[(start+offset)%len(appCodes)])
|
||||||
|
}
|
||||||
|
return ordered
|
||||||
|
}
|
||||||
|
|
||||||
|
// Normalize canonicalizes tenant keys, removes blanks and duplicates, and can
|
||||||
|
// sort DB-discovered keys for deterministic rotation. Explicit configuration
|
||||||
|
// preserves operator order so staged rollouts remain readable and predictable.
|
||||||
|
func Normalize(values []string, sorted bool) []string {
|
||||||
|
normalized := make([]string, 0, len(values))
|
||||||
|
seen := make(map[string]struct{}, len(values))
|
||||||
|
for _, value := range values {
|
||||||
|
if strings.TrimSpace(value) == "" {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
value = appcode.Normalize(value)
|
||||||
|
if _, exists := seen[value]; exists {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
seen[value] = struct{}{}
|
||||||
|
normalized = append(normalized, value)
|
||||||
|
}
|
||||||
|
if sorted {
|
||||||
|
sort.Strings(normalized)
|
||||||
|
}
|
||||||
|
return normalized
|
||||||
|
}
|
||||||
|
|
||||||
|
// ClaimFair gives every App one bounded first-pass quota, then lets non-empty
|
||||||
|
// partitions use capacity left by empty queues. Each callback remains App-scoped
|
||||||
|
// and is expected to perform its own indexed SKIP LOCKED transaction.
|
||||||
|
func ClaimFair[T any](ctx context.Context, orderedApps []string, limit int, claim func(context.Context, string, int) ([]T, error)) ([]T, error) {
|
||||||
|
if limit <= 0 || len(orderedApps) == 0 || claim == nil {
|
||||||
|
return nil, nil
|
||||||
|
}
|
||||||
|
quota := (limit + len(orderedApps) - 1) / len(orderedApps)
|
||||||
|
claimed := make([]T, 0, limit)
|
||||||
|
for pass := 0; pass < 2 && len(claimed) < limit; pass++ {
|
||||||
|
for _, app := range orderedApps {
|
||||||
|
remaining := limit - len(claimed)
|
||||||
|
if remaining <= 0 {
|
||||||
|
break
|
||||||
|
}
|
||||||
|
appLimit := remaining
|
||||||
|
if pass == 0 && appLimit > quota {
|
||||||
|
appLimit = quota
|
||||||
|
}
|
||||||
|
records, err := claim(ctx, app, appLimit)
|
||||||
|
if err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
if len(records) > appLimit {
|
||||||
|
// Silently truncating would strand already-claimed rows until lease
|
||||||
|
// expiry. Treat repository limit violations as explicit worker errors.
|
||||||
|
return nil, fmt.Errorf("outbox claim for app %q returned %d records above limit %d", app, len(records), appLimit)
|
||||||
|
}
|
||||||
|
claimed = append(claimed, records...)
|
||||||
|
}
|
||||||
|
if pass == 0 && len(claimed) == 0 {
|
||||||
|
// A globally empty first pass must stop here; a second identical pass
|
||||||
|
// only doubles indexed polling queries and cannot discover new work.
|
||||||
|
break
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return claimed, nil
|
||||||
|
}
|
||||||
96
pkg/outboxpartition/partitions_test.go
Normal file
96
pkg/outboxpartition/partitions_test.go
Normal file
@ -0,0 +1,96 @@
|
|||||||
|
package outboxpartition
|
||||||
|
|
||||||
|
import (
|
||||||
|
"context"
|
||||||
|
"errors"
|
||||||
|
"reflect"
|
||||||
|
"testing"
|
||||||
|
"time"
|
||||||
|
)
|
||||||
|
|
||||||
|
func TestConfiguredPartitionsNormalizeAndSkipDiscovery(t *testing.T) {
|
||||||
|
partitions := New([]string{" LALU ", "fami", "HUWAA", "fami", ""}, time.Hour)
|
||||||
|
discoverCalls := 0
|
||||||
|
apps, err := partitions.Resolve(context.Background(), func(context.Context) ([]string, error) {
|
||||||
|
discoverCalls++
|
||||||
|
return nil, errors.New("must not run")
|
||||||
|
})
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("resolve configured partitions: %v", err)
|
||||||
|
}
|
||||||
|
if want := []string{"lalu", "fami", "huwaa"}; !reflect.DeepEqual(apps, want) {
|
||||||
|
t.Fatalf("configured apps = %v, want %v", apps, want)
|
||||||
|
}
|
||||||
|
if discoverCalls != 0 {
|
||||||
|
t.Fatalf("configured allowlist unexpectedly queried discovery %d times", discoverCalls)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestDiscoveredPartitionsAreCachedAndStaleListSurvivesRefreshFailure(t *testing.T) {
|
||||||
|
partitions := New(nil, time.Hour)
|
||||||
|
discoverCalls := 0
|
||||||
|
discover := func(context.Context) ([]string, error) {
|
||||||
|
discoverCalls++
|
||||||
|
if discoverCalls == 1 {
|
||||||
|
return []string{"huwaa", " LALU ", "fami"}, nil
|
||||||
|
}
|
||||||
|
return nil, errors.New("mysql unavailable")
|
||||||
|
}
|
||||||
|
first, err := partitions.Resolve(context.Background(), discover)
|
||||||
|
if err != nil || !reflect.DeepEqual(first, []string{"fami", "huwaa", "lalu"}) {
|
||||||
|
t.Fatalf("initial discovery apps=%v err=%v", first, err)
|
||||||
|
}
|
||||||
|
second, err := partitions.Resolve(context.Background(), discover)
|
||||||
|
if err != nil || !reflect.DeepEqual(second, first) || discoverCalls != 1 {
|
||||||
|
t.Fatalf("cached discovery apps=%v err=%v calls=%d", second, err, discoverCalls)
|
||||||
|
}
|
||||||
|
|
||||||
|
// Force only the refresh boundary; production reaches this state after the
|
||||||
|
// five-minute interval without exposing a public cache mutation API.
|
||||||
|
partitions.mu.Lock()
|
||||||
|
partitions.nextRefresh = time.Time{}
|
||||||
|
partitions.mu.Unlock()
|
||||||
|
stale, err := partitions.Resolve(context.Background(), discover)
|
||||||
|
if err == nil || !reflect.DeepEqual(stale, first) {
|
||||||
|
t.Fatalf("failed refresh must retain stale apps: apps=%v err=%v", stale, err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestClaimFairRotatesLaluFamiHuwaaAndSkipsEmptyQueue(t *testing.T) {
|
||||||
|
partitions := New([]string{"lalu", "fami", "huwaa"}, time.Hour)
|
||||||
|
queues := map[string][]string{
|
||||||
|
"lalu": {"l1", "l2", "l3"},
|
||||||
|
"fami": nil,
|
||||||
|
"huwaa": {"h1", "h2", "h3"},
|
||||||
|
}
|
||||||
|
claim := func(_ context.Context, app string, limit int) ([]string, error) {
|
||||||
|
queue := queues[app]
|
||||||
|
if len(queue) < limit {
|
||||||
|
limit = len(queue)
|
||||||
|
}
|
||||||
|
result := append([]string(nil), queue[:limit]...)
|
||||||
|
queues[app] = queue[limit:]
|
||||||
|
return result, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
apps, err := partitions.Resolve(context.Background(), nil)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("resolve configured apps: %v", err)
|
||||||
|
}
|
||||||
|
first, err := ClaimFair(context.Background(), partitions.Order(apps), 3, claim)
|
||||||
|
if err != nil || !reflect.DeepEqual(first, []string{"l1", "h1", "l2"}) {
|
||||||
|
t.Fatalf("first fair claim records=%v err=%v", first, err)
|
||||||
|
}
|
||||||
|
second, err := ClaimFair(context.Background(), partitions.Order(apps), 3, claim)
|
||||||
|
if err != nil || !reflect.DeepEqual(second, []string{"h2", "l3", "h3"}) {
|
||||||
|
t.Fatalf("rotated fair claim records=%v err=%v", second, err)
|
||||||
|
}
|
||||||
|
emptyCalls := 0
|
||||||
|
empty, err := ClaimFair[string](context.Background(), partitions.Order(apps), 3, func(context.Context, string, int) ([]string, error) {
|
||||||
|
emptyCalls++
|
||||||
|
return nil, nil
|
||||||
|
})
|
||||||
|
if err != nil || len(empty) != 0 || emptyCalls != 3 {
|
||||||
|
t.Fatalf("empty queues records=%v err=%v calls=%d", empty, err, emptyCalls)
|
||||||
|
}
|
||||||
|
}
|
||||||
@ -1,6 +1,7 @@
|
|||||||
package xerr
|
package xerr
|
||||||
|
|
||||||
import (
|
import (
|
||||||
|
"context"
|
||||||
"errors"
|
"errors"
|
||||||
|
|
||||||
"google.golang.org/genproto/googleapis/rpc/errdetails"
|
"google.golang.org/genproto/googleapis/rpc/errdetails"
|
||||||
@ -27,6 +28,16 @@ func ToGRPCError(err error) error {
|
|||||||
return err
|
return err
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// context 终止描述的是调用链生命周期,而不是业务内部故障。仓储层会直接返回
|
||||||
|
// context.Canceled/DeadlineExceeded;若先走兜底 Internal,主动取消的并发 sibling
|
||||||
|
// 会在服务端被误报成 INTERNAL_ERROR,并把一次正常的快速失败伪装成数据库事故。
|
||||||
|
if errors.Is(err, context.Canceled) {
|
||||||
|
return status.Error(codes.Canceled, context.Canceled.Error())
|
||||||
|
}
|
||||||
|
if errors.Is(err, context.DeadlineExceeded) {
|
||||||
|
return status.Error(codes.DeadlineExceeded, context.DeadlineExceeded.Error())
|
||||||
|
}
|
||||||
|
|
||||||
domainErr, ok := As(err)
|
domainErr, ok := As(err)
|
||||||
if !ok {
|
if !ok {
|
||||||
domainErr = &Error{Code: Internal, Message: "internal error"}
|
domainErr = &Error{Code: Internal, Message: "internal error"}
|
||||||
|
|||||||
@ -1,6 +1,8 @@
|
|||||||
package xerr
|
package xerr
|
||||||
|
|
||||||
import (
|
import (
|
||||||
|
"context"
|
||||||
|
"fmt"
|
||||||
"go/ast"
|
"go/ast"
|
||||||
"go/parser"
|
"go/parser"
|
||||||
"go/token"
|
"go/token"
|
||||||
@ -29,6 +31,29 @@ func TestToGRPCErrorCarriesReason(t *testing.T) {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// TestToGRPCErrorPreservesContextTermination 防止并发聚合器主动取消 sibling RPC 时,
|
||||||
|
// database/sql 返回的 context 错误被兜底成 Internal,污染告警并掩盖真正的首个失败。
|
||||||
|
func TestToGRPCErrorPreservesContextTermination(t *testing.T) {
|
||||||
|
tests := []struct {
|
||||||
|
name string
|
||||||
|
err error
|
||||||
|
code codes.Code
|
||||||
|
}{
|
||||||
|
{name: "canceled", err: context.Canceled, code: codes.Canceled},
|
||||||
|
{name: "wrapped canceled", err: fmt.Errorf("query interrupted: %w", context.Canceled), code: codes.Canceled},
|
||||||
|
{name: "deadline exceeded", err: context.DeadlineExceeded, code: codes.DeadlineExceeded},
|
||||||
|
{name: "wrapped deadline exceeded", err: fmt.Errorf("query interrupted: %w", context.DeadlineExceeded), code: codes.DeadlineExceeded},
|
||||||
|
}
|
||||||
|
|
||||||
|
for _, test := range tests {
|
||||||
|
t.Run(test.name, func(t *testing.T) {
|
||||||
|
if got := status.Code(ToGRPCError(test.err)); got != test.code {
|
||||||
|
t.Fatalf("grpc code mismatch: got %s want %s", got, test.code)
|
||||||
|
}
|
||||||
|
})
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
func TestCatalogCoversDeclaredCodes(t *testing.T) {
|
func TestCatalogCoversDeclaredCodes(t *testing.T) {
|
||||||
declared := declaredCodesFromSource(t)
|
declared := declaredCodesFromSource(t)
|
||||||
|
|
||||||
|
|||||||
552
services/activity-service/cmd/audit-wheel-rewards/main.go
Normal file
552
services/activity-service/cmd/audit-wheel-rewards/main.go
Normal file
@ -0,0 +1,552 @@
|
|||||||
|
// Command audit-wheel-rewards performs a read-only, item-by-item reconciliation
|
||||||
|
// between unsettled WheelRewardSettlement facts and wallet owner receipts.
|
||||||
|
package main
|
||||||
|
|
||||||
|
import (
|
||||||
|
"context"
|
||||||
|
"crypto/sha256"
|
||||||
|
"database/sql"
|
||||||
|
"encoding/hex"
|
||||||
|
"encoding/json"
|
||||||
|
"errors"
|
||||||
|
"flag"
|
||||||
|
"fmt"
|
||||||
|
"log"
|
||||||
|
"os"
|
||||||
|
"sort"
|
||||||
|
"strconv"
|
||||||
|
"strings"
|
||||||
|
"time"
|
||||||
|
"unicode"
|
||||||
|
|
||||||
|
_ "github.com/go-sql-driver/mysql"
|
||||||
|
"hyapp/pkg/appcode"
|
||||||
|
"hyapp/services/activity-service/internal/config"
|
||||||
|
domain "hyapp/services/activity-service/internal/domain/wheel"
|
||||||
|
)
|
||||||
|
|
||||||
|
type outboxRow struct {
|
||||||
|
AppCode string
|
||||||
|
OutboxID string
|
||||||
|
Payload []byte
|
||||||
|
Status string
|
||||||
|
RetryCount int
|
||||||
|
CreatedMS int64
|
||||||
|
}
|
||||||
|
|
||||||
|
type rewardPayload struct {
|
||||||
|
AppCode string `json:"app_code"`
|
||||||
|
DrawID string `json:"draw_id"`
|
||||||
|
DrawIDs []string `json:"draw_ids"`
|
||||||
|
CommandID string `json:"command_id"`
|
||||||
|
UserID int64 `json:"user_id"`
|
||||||
|
WheelID string `json:"wheel_id"`
|
||||||
|
SelectedTierID string `json:"selected_tier_id"`
|
||||||
|
RewardType string `json:"reward_type"`
|
||||||
|
RewardID string `json:"reward_id"`
|
||||||
|
RewardCount int64 `json:"reward_count"`
|
||||||
|
RewardCoins int64 `json:"reward_coins"`
|
||||||
|
MetadataJSON string `json:"metadata_json"`
|
||||||
|
Rewards []domain.DrawReward `json:"rewards"`
|
||||||
|
VisibleRegionID int64 `json:"visible_region_id"`
|
||||||
|
}
|
||||||
|
|
||||||
|
type itemAudit struct {
|
||||||
|
CommandID string `json:"command_id"`
|
||||||
|
Type string `json:"type"`
|
||||||
|
State string `json:"state"`
|
||||||
|
ReceiptID string `json:"receipt_id,omitempty"`
|
||||||
|
Reason string `json:"reason,omitempty"`
|
||||||
|
}
|
||||||
|
|
||||||
|
type rowAudit struct {
|
||||||
|
AppCode string `json:"app_code"`
|
||||||
|
OutboxID string `json:"outbox_id"`
|
||||||
|
Status string `json:"outbox_status"`
|
||||||
|
State string `json:"state"`
|
||||||
|
Items []itemAudit `json:"items"`
|
||||||
|
}
|
||||||
|
|
||||||
|
type auditSummary struct {
|
||||||
|
Rows int `json:"rows"`
|
||||||
|
FullyPaid int `json:"fully_paid"`
|
||||||
|
FullyUnpaid int `json:"fully_unpaid"`
|
||||||
|
PartiallyPaid int `json:"partially_paid"`
|
||||||
|
Conflict int `json:"conflict"`
|
||||||
|
PaidItems int `json:"paid_items"`
|
||||||
|
UnpaidItems int `json:"unpaid_items"`
|
||||||
|
ConflictItems int `json:"conflict_items"`
|
||||||
|
AuditSHA256 string `json:"audit_sha256"`
|
||||||
|
}
|
||||||
|
|
||||||
|
type appCodeFlags []string
|
||||||
|
|
||||||
|
func (values *appCodeFlags) String() string {
|
||||||
|
return strings.Join(*values, ",")
|
||||||
|
}
|
||||||
|
|
||||||
|
func (values *appCodeFlags) Set(value string) error {
|
||||||
|
value = strings.TrimSpace(value)
|
||||||
|
if value == "" {
|
||||||
|
return fmt.Errorf("app-code must not be empty")
|
||||||
|
}
|
||||||
|
*values = append(*values, value)
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func main() {
|
||||||
|
configPath := flag.String("config", "services/activity-service/configs/config.yaml", "path to activity-service config")
|
||||||
|
walletDatabase := flag.String("wallet-database", "hyapp_wallet", "wallet owner database on the same MySQL cluster")
|
||||||
|
limit := flag.Int("limit", 500, "maximum unsettled rows, 1..500")
|
||||||
|
details := flag.Bool("details", false, "emit one JSON reconciliation record per outbox row")
|
||||||
|
var requestedAppCodes appCodeFlags
|
||||||
|
flag.Var(&requestedAppCodes, "app-code", "required App partition to audit; repeat for multiple Apps, for example --app-code lalu --app-code fami")
|
||||||
|
flag.Parse()
|
||||||
|
|
||||||
|
if *limit <= 0 || *limit > 500 {
|
||||||
|
log.Fatal("limit must be between 1 and 500")
|
||||||
|
}
|
||||||
|
walletDB := normalizedDatabaseName(*walletDatabase)
|
||||||
|
if walletDB == "" {
|
||||||
|
log.Fatal("wallet-database contains invalid characters")
|
||||||
|
}
|
||||||
|
appCodes, err := normalizeAuditAppCodes(requestedAppCodes)
|
||||||
|
if err != nil {
|
||||||
|
log.Fatal(err)
|
||||||
|
}
|
||||||
|
cfg, err := config.Load(*configPath)
|
||||||
|
if err != nil {
|
||||||
|
log.Fatal(err)
|
||||||
|
}
|
||||||
|
db, err := sql.Open("mysql", cfg.MySQLDSN)
|
||||||
|
if err != nil {
|
||||||
|
log.Fatal(err)
|
||||||
|
}
|
||||||
|
defer func() { _ = db.Close() }()
|
||||||
|
db.SetMaxOpenConns(2)
|
||||||
|
db.SetMaxIdleConns(1)
|
||||||
|
|
||||||
|
ctx, cancel := context.WithTimeout(context.Background(), 2*time.Minute)
|
||||||
|
defer cancel()
|
||||||
|
if err := db.PingContext(ctx); err != nil {
|
||||||
|
log.Fatal(err)
|
||||||
|
}
|
||||||
|
rows, err := loadUnsettledRows(ctx, db, appCodes, *limit)
|
||||||
|
if err != nil {
|
||||||
|
log.Fatal(err)
|
||||||
|
}
|
||||||
|
audits := make([]rowAudit, 0, len(rows))
|
||||||
|
for _, row := range rows {
|
||||||
|
audit, err := auditRow(ctx, db, walletDB, row)
|
||||||
|
if err != nil {
|
||||||
|
log.Fatalf("audit app_code=%s outbox_id=%s: %v", row.AppCode, row.OutboxID, err)
|
||||||
|
}
|
||||||
|
audits = append(audits, audit)
|
||||||
|
}
|
||||||
|
summary := summarize(audits)
|
||||||
|
if *details {
|
||||||
|
encoder := json.NewEncoder(os.Stdout)
|
||||||
|
for _, audit := range audits {
|
||||||
|
if err := encoder.Encode(audit); err != nil {
|
||||||
|
log.Fatal(err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
encoded, err := json.Marshal(summary)
|
||||||
|
if err != nil {
|
||||||
|
log.Fatal(err)
|
||||||
|
}
|
||||||
|
fmt.Println(string(encoded))
|
||||||
|
}
|
||||||
|
|
||||||
|
func loadUnsettledRows(ctx context.Context, db *sql.DB, requestedAppCodes []string, limit int) ([]outboxRow, error) {
|
||||||
|
if limit <= 0 || limit > 500 {
|
||||||
|
return nil, fmt.Errorf("limit must be between 1 and 500")
|
||||||
|
}
|
||||||
|
appCodes, err := normalizeAuditAppCodes(requestedAppCodes)
|
||||||
|
if err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
// activity_outbox 的线上索引都以 app_code 开头。每个 App 拆成 retry/lock 两个等值分区查询,
|
||||||
|
// 既能强制使用对应索引,也避免 status OR 让优化器退化为全表扫描。每个分支只取 global remaining,
|
||||||
|
// 合并出该 App 最早的一页后再进入下一 App,因此返回量严格不超过 limit,数据库返回行数也被约束在约 2*limit 内。
|
||||||
|
result := make([]outboxRow, 0, limit)
|
||||||
|
for _, appCodeValue := range appCodes {
|
||||||
|
remaining := limit - len(result)
|
||||||
|
if remaining <= 0 {
|
||||||
|
break
|
||||||
|
}
|
||||||
|
branches := []struct {
|
||||||
|
query string
|
||||||
|
args []any
|
||||||
|
}{
|
||||||
|
{
|
||||||
|
query: unsettledRetryQuery(),
|
||||||
|
args: []any{
|
||||||
|
appCodeValue,
|
||||||
|
domain.EventTypeWheelRewardSettlement,
|
||||||
|
domain.OutboxStatusPending,
|
||||||
|
domain.OutboxStatusRetryable,
|
||||||
|
remaining,
|
||||||
|
},
|
||||||
|
},
|
||||||
|
{
|
||||||
|
query: unsettledDeliveringQuery(),
|
||||||
|
args: []any{
|
||||||
|
appCodeValue,
|
||||||
|
domain.EventTypeWheelRewardSettlement,
|
||||||
|
domain.OutboxStatusDelivering,
|
||||||
|
remaining,
|
||||||
|
},
|
||||||
|
},
|
||||||
|
}
|
||||||
|
appCandidates := make([]outboxRow, 0, remaining)
|
||||||
|
seen := make(map[string]struct{}, remaining)
|
||||||
|
for _, branch := range branches {
|
||||||
|
rows, err := loadUnsettledBranch(ctx, db, branch.query, branch.args...)
|
||||||
|
if err != nil {
|
||||||
|
return nil, fmt.Errorf("load app_code=%s: %w", appCodeValue, err)
|
||||||
|
}
|
||||||
|
for _, row := range rows {
|
||||||
|
key := row.AppCode + "\x00" + row.OutboxID
|
||||||
|
if _, exists := seen[key]; exists {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
seen[key] = struct{}{}
|
||||||
|
appCandidates = append(appCandidates, row)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
sort.Slice(appCandidates, func(i, j int) bool {
|
||||||
|
if appCandidates[i].CreatedMS != appCandidates[j].CreatedMS {
|
||||||
|
return appCandidates[i].CreatedMS < appCandidates[j].CreatedMS
|
||||||
|
}
|
||||||
|
if appCandidates[i].OutboxID != appCandidates[j].OutboxID {
|
||||||
|
return appCandidates[i].OutboxID < appCandidates[j].OutboxID
|
||||||
|
}
|
||||||
|
return appCandidates[i].Status < appCandidates[j].Status
|
||||||
|
})
|
||||||
|
if len(appCandidates) > remaining {
|
||||||
|
appCandidates = appCandidates[:remaining]
|
||||||
|
}
|
||||||
|
result = append(result, appCandidates...)
|
||||||
|
}
|
||||||
|
return result, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func unsettledRetryQuery() string {
|
||||||
|
return `
|
||||||
|
SELECT app_code, outbox_id, payload, status, retry_count, created_at_ms
|
||||||
|
FROM activity_outbox FORCE INDEX (idx_activity_outbox_event_retry)
|
||||||
|
WHERE app_code = ? AND event_type = ? AND status IN (?, ?)
|
||||||
|
ORDER BY created_at_ms, outbox_id
|
||||||
|
LIMIT ?`
|
||||||
|
}
|
||||||
|
|
||||||
|
func unsettledDeliveringQuery() string {
|
||||||
|
return `
|
||||||
|
SELECT app_code, outbox_id, payload, status, retry_count, created_at_ms
|
||||||
|
FROM activity_outbox FORCE INDEX (idx_activity_outbox_event_lock)
|
||||||
|
WHERE app_code = ? AND event_type = ? AND status = ?
|
||||||
|
ORDER BY created_at_ms, outbox_id
|
||||||
|
LIMIT ?`
|
||||||
|
}
|
||||||
|
|
||||||
|
func loadUnsettledBranch(ctx context.Context, db *sql.DB, query string, args ...any) ([]outboxRow, error) {
|
||||||
|
rows, err := db.QueryContext(ctx, query, args...)
|
||||||
|
if err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
defer rows.Close()
|
||||||
|
result := make([]outboxRow, 0)
|
||||||
|
for rows.Next() {
|
||||||
|
var row outboxRow
|
||||||
|
if err := rows.Scan(&row.AppCode, &row.OutboxID, &row.Payload, &row.Status, &row.RetryCount, &row.CreatedMS); err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
result = append(result, row)
|
||||||
|
}
|
||||||
|
return result, rows.Err()
|
||||||
|
}
|
||||||
|
|
||||||
|
func normalizeAuditAppCodes(values []string) ([]string, error) {
|
||||||
|
if len(values) == 0 {
|
||||||
|
return nil, fmt.Errorf("at least one --app-code is required")
|
||||||
|
}
|
||||||
|
const maxAuditAppCodes = 16
|
||||||
|
seen := make(map[string]struct{}, len(values))
|
||||||
|
result := make([]string, 0, len(values))
|
||||||
|
for _, value := range values {
|
||||||
|
if strings.TrimSpace(value) == "" {
|
||||||
|
return nil, fmt.Errorf("app-code must not be empty")
|
||||||
|
}
|
||||||
|
value = appcode.Normalize(value)
|
||||||
|
if _, exists := seen[value]; exists {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
seen[value] = struct{}{}
|
||||||
|
result = append(result, value)
|
||||||
|
}
|
||||||
|
if len(result) == 0 {
|
||||||
|
return nil, fmt.Errorf("at least one --app-code is required")
|
||||||
|
}
|
||||||
|
if len(result) > maxAuditAppCodes {
|
||||||
|
return nil, fmt.Errorf("at most %d app-code values are allowed", maxAuditAppCodes)
|
||||||
|
}
|
||||||
|
sort.Strings(result)
|
||||||
|
return result, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func auditRow(ctx context.Context, db *sql.DB, walletDB string, row outboxRow) (rowAudit, error) {
|
||||||
|
var payload rewardPayload
|
||||||
|
if err := json.Unmarshal(row.Payload, &payload); err != nil {
|
||||||
|
return rowAudit{}, fmt.Errorf("decode immutable payload: %w", err)
|
||||||
|
}
|
||||||
|
row.AppCode = appcode.Normalize(row.AppCode)
|
||||||
|
if payload.AppCode != "" && appcode.Normalize(payload.AppCode) != row.AppCode {
|
||||||
|
return rowAudit{}, fmt.Errorf("payload app_code does not match outbox partition")
|
||||||
|
}
|
||||||
|
payload.DrawID = strings.TrimSpace(payload.DrawID)
|
||||||
|
payload.WheelID = strings.TrimSpace(payload.WheelID)
|
||||||
|
if payload.DrawID == "" || payload.WheelID == "" || payload.UserID <= 0 {
|
||||||
|
return rowAudit{}, fmt.Errorf("payload identity is incomplete")
|
||||||
|
}
|
||||||
|
rewards := payload.Rewards
|
||||||
|
if len(rewards) == 0 {
|
||||||
|
rewards = []domain.DrawReward{{
|
||||||
|
SelectedTierID: payload.SelectedTierID,
|
||||||
|
RewardType: payload.RewardType,
|
||||||
|
RewardID: payload.RewardID,
|
||||||
|
RewardCount: payload.RewardCount,
|
||||||
|
RewardCoins: payload.RewardCoins,
|
||||||
|
MetadataJSON: payload.MetadataJSON,
|
||||||
|
}}
|
||||||
|
}
|
||||||
|
if len(rewards) == 0 {
|
||||||
|
return rowAudit{}, fmt.Errorf("payload reward snapshot is empty")
|
||||||
|
}
|
||||||
|
|
||||||
|
audit := rowAudit{AppCode: row.AppCode, OutboxID: row.OutboxID, Status: row.Status, Items: make([]itemAudit, 0, len(rewards))}
|
||||||
|
for index, reward := range rewards {
|
||||||
|
reward.RewardType = strings.ToLower(strings.TrimSpace(reward.RewardType))
|
||||||
|
var item itemAudit
|
||||||
|
var err error
|
||||||
|
switch reward.RewardType {
|
||||||
|
case domain.RewardTypeCoin:
|
||||||
|
item, err = auditCoin(ctx, db, walletDB, row.AppCode, payload, reward, index, len(rewards))
|
||||||
|
case domain.RewardTypeGift, domain.RewardTypeProp, domain.RewardTypeDress:
|
||||||
|
item, err = auditResource(ctx, db, walletDB, row.AppCode, payload, reward, index, len(rewards))
|
||||||
|
default:
|
||||||
|
item = itemAudit{Type: reward.RewardType, State: "conflict", Reason: "unsupported reward type"}
|
||||||
|
}
|
||||||
|
if err != nil {
|
||||||
|
return rowAudit{}, err
|
||||||
|
}
|
||||||
|
audit.Items = append(audit.Items, item)
|
||||||
|
}
|
||||||
|
audit.State = aggregateRowState(audit.Items)
|
||||||
|
return audit, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func auditCoin(ctx context.Context, db *sql.DB, walletDB string, app string, payload rewardPayload, reward domain.DrawReward, index int, total int) (itemAudit, error) {
|
||||||
|
commandID := rewardCommandID("wheel_reward", payload.DrawID, index, total)
|
||||||
|
item := itemAudit{CommandID: commandID, Type: domain.RewardTypeCoin, State: "unpaid"}
|
||||||
|
query := coinAuditQuery(walletDB)
|
||||||
|
var transactionID, bizType, statusValue, metadataJSON string
|
||||||
|
err := db.QueryRowContext(ctx, query, app, commandID).Scan(&transactionID, &bizType, &statusValue, &metadataJSON)
|
||||||
|
if err != nil {
|
||||||
|
// database/sql 和 driver/observer 包装都可能给 ErrNoRows 增加上下文;审计必须把这种稳定语义归为 unpaid,而不是中止整批核账。
|
||||||
|
if errors.Is(err, sql.ErrNoRows) {
|
||||||
|
return item, nil
|
||||||
|
}
|
||||||
|
return itemAudit{}, err
|
||||||
|
}
|
||||||
|
metadata := map[string]any{}
|
||||||
|
if err := json.Unmarshal([]byte(metadataJSON), &metadata); err != nil {
|
||||||
|
item.State, item.Reason = "conflict", "wallet metadata is invalid"
|
||||||
|
return item, nil
|
||||||
|
}
|
||||||
|
// wallet_transactions 和 resource_grants 共用 succeeded 终态;completed 从未是当前账本状态,不能把真实已发金币误判为冲突。
|
||||||
|
if bizType != "wheel_reward" || statusValue != "succeeded" || jsonInt64(metadata, "target_user_id") != payload.UserID || jsonInt64(metadata, "amount") != reward.RewardCoins || jsonString(metadata, "draw_id") != payload.DrawID || jsonString(metadata, "wheel_id") != payload.WheelID || jsonString(metadata, "selected_tier_id") != strings.TrimSpace(reward.SelectedTierID) || jsonString(metadata, "reason") != "wheel_reward" || jsonInt64(metadata, "visible_region_id") != payload.VisibleRegionID {
|
||||||
|
item.State, item.Reason = "conflict", "existing wallet transaction does not match immutable reward snapshot"
|
||||||
|
return item, nil
|
||||||
|
}
|
||||||
|
item.State = "paid"
|
||||||
|
item.ReceiptID = transactionID
|
||||||
|
return item, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func coinAuditQuery(walletDB string) string {
|
||||||
|
return fmt.Sprintf(`
|
||||||
|
SELECT transaction_id, biz_type, status, COALESCE(CAST(metadata_json AS CHAR), '{}')
|
||||||
|
FROM %s.wallet_transactions
|
||||||
|
WHERE app_code = ? AND command_id = ?`, walletDB)
|
||||||
|
}
|
||||||
|
|
||||||
|
func auditResource(ctx context.Context, db *sql.DB, walletDB string, app string, payload rewardPayload, reward domain.DrawReward, index int, total int) (itemAudit, error) {
|
||||||
|
commandID := rewardCommandID("wheel_resource", payload.DrawID, index, total)
|
||||||
|
item := itemAudit{CommandID: commandID, Type: reward.RewardType, State: "unpaid"}
|
||||||
|
resourceID := rewardResourceID(reward)
|
||||||
|
quantity := reward.RewardCount
|
||||||
|
if quantity <= 0 {
|
||||||
|
quantity = 1
|
||||||
|
}
|
||||||
|
durationMS := metadataInt64(reward.MetadataJSON, "duration_ms", "durationMs", "validity_ms", "validityMs")
|
||||||
|
query := resourceAuditQuery(walletDB)
|
||||||
|
var grantID, statusValue, grantSource, subjectType, subjectID, reason string
|
||||||
|
var targetUserID, operatorUserID, actualQuantity, actualDurationMS, actualResourceID int64
|
||||||
|
err := db.QueryRowContext(ctx, query, app, commandID).Scan(&grantID, &statusValue, &targetUserID, &grantSource, &subjectType, &subjectID, &reason, &operatorUserID, &actualQuantity, &actualDurationMS, &actualResourceID)
|
||||||
|
if err != nil {
|
||||||
|
if errors.Is(err, sql.ErrNoRows) {
|
||||||
|
return item, nil
|
||||||
|
}
|
||||||
|
return itemAudit{}, err
|
||||||
|
}
|
||||||
|
if resourceID <= 0 || statusValue != "succeeded" || targetUserID != payload.UserID || grantSource != "wheel_reward" || subjectType != "resource" || subjectID != strconv.FormatInt(resourceID, 10) || reason != "wheel_reward" || operatorUserID != payload.UserID || actualQuantity != quantity || actualDurationMS != durationMS || actualResourceID != resourceID {
|
||||||
|
item.State, item.Reason = "conflict", "existing resource grant does not match immutable reward snapshot"
|
||||||
|
return item, nil
|
||||||
|
}
|
||||||
|
item.State = "paid"
|
||||||
|
item.ReceiptID = grantID
|
||||||
|
return item, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func resourceAuditQuery(walletDB string) string {
|
||||||
|
// 每个 SELECT 列只出现一次且与 Scan 顺序一一对应;显式逐列排版避免核账命令在生产才暴露漏逗号或重复列。
|
||||||
|
return fmt.Sprintf(`
|
||||||
|
SELECT rg.grant_id,
|
||||||
|
rg.status,
|
||||||
|
rg.target_user_id,
|
||||||
|
rg.grant_source,
|
||||||
|
rg.grant_subject_type,
|
||||||
|
rg.grant_subject_id,
|
||||||
|
rg.reason,
|
||||||
|
rg.operator_user_id,
|
||||||
|
COALESCE(rgi.quantity, 0),
|
||||||
|
COALESCE(rgi.duration_ms, 0),
|
||||||
|
COALESCE(rgi.resource_id, 0)
|
||||||
|
FROM %s.resource_grants rg
|
||||||
|
LEFT JOIN %s.resource_grant_items rgi
|
||||||
|
ON rgi.app_code = rg.app_code AND rgi.grant_id = rg.grant_id
|
||||||
|
WHERE rg.app_code = ? AND rg.command_id = ?
|
||||||
|
ORDER BY rgi.grant_item_id
|
||||||
|
LIMIT 1`, walletDB, walletDB)
|
||||||
|
}
|
||||||
|
|
||||||
|
func aggregateRowState(items []itemAudit) string {
|
||||||
|
paid, unpaid, conflict := 0, 0, 0
|
||||||
|
for _, item := range items {
|
||||||
|
switch item.State {
|
||||||
|
case "paid":
|
||||||
|
paid++
|
||||||
|
case "unpaid":
|
||||||
|
unpaid++
|
||||||
|
default:
|
||||||
|
conflict++
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if conflict > 0 {
|
||||||
|
return "conflict"
|
||||||
|
}
|
||||||
|
if paid == len(items) {
|
||||||
|
return "fully_paid"
|
||||||
|
}
|
||||||
|
if unpaid == len(items) {
|
||||||
|
return "fully_unpaid"
|
||||||
|
}
|
||||||
|
return "partially_paid"
|
||||||
|
}
|
||||||
|
|
||||||
|
func summarize(audits []rowAudit) auditSummary {
|
||||||
|
summary := auditSummary{Rows: len(audits)}
|
||||||
|
canonical := make([]string, 0, len(audits))
|
||||||
|
for _, audit := range audits {
|
||||||
|
switch audit.State {
|
||||||
|
case "fully_paid":
|
||||||
|
summary.FullyPaid++
|
||||||
|
case "fully_unpaid":
|
||||||
|
summary.FullyUnpaid++
|
||||||
|
case "partially_paid":
|
||||||
|
summary.PartiallyPaid++
|
||||||
|
default:
|
||||||
|
summary.Conflict++
|
||||||
|
}
|
||||||
|
for _, item := range audit.Items {
|
||||||
|
switch item.State {
|
||||||
|
case "paid":
|
||||||
|
summary.PaidItems++
|
||||||
|
case "unpaid":
|
||||||
|
summary.UnpaidItems++
|
||||||
|
default:
|
||||||
|
summary.ConflictItems++
|
||||||
|
}
|
||||||
|
}
|
||||||
|
encoded, _ := json.Marshal(audit)
|
||||||
|
canonical = append(canonical, string(encoded))
|
||||||
|
}
|
||||||
|
sort.Strings(canonical)
|
||||||
|
sum := sha256.Sum256([]byte(strings.Join(canonical, "\n")))
|
||||||
|
summary.AuditSHA256 = hex.EncodeToString(sum[:])
|
||||||
|
return summary
|
||||||
|
}
|
||||||
|
|
||||||
|
func rewardCommandID(prefix string, drawID string, index int, total int) string {
|
||||||
|
if total <= 1 {
|
||||||
|
return prefix + ":" + drawID
|
||||||
|
}
|
||||||
|
return fmt.Sprintf("%s:%s:%d", prefix, drawID, index+1)
|
||||||
|
}
|
||||||
|
|
||||||
|
func rewardResourceID(reward domain.DrawReward) int64 {
|
||||||
|
if value := metadataInt64(reward.MetadataJSON, "resource_id", "resourceId"); value > 0 {
|
||||||
|
return value
|
||||||
|
}
|
||||||
|
value, _ := strconv.ParseInt(strings.TrimSpace(reward.RewardID), 10, 64)
|
||||||
|
return value
|
||||||
|
}
|
||||||
|
|
||||||
|
func metadataInt64(raw string, keys ...string) int64 {
|
||||||
|
metadata := map[string]any{}
|
||||||
|
if err := json.Unmarshal([]byte(strings.TrimSpace(raw)), &metadata); err != nil {
|
||||||
|
return 0
|
||||||
|
}
|
||||||
|
for _, key := range keys {
|
||||||
|
if value := jsonInt64(metadata, key); value > 0 {
|
||||||
|
return value
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return 0
|
||||||
|
}
|
||||||
|
|
||||||
|
func jsonInt64(values map[string]any, key string) int64 {
|
||||||
|
switch value := values[key].(type) {
|
||||||
|
case float64:
|
||||||
|
return int64(value)
|
||||||
|
case json.Number:
|
||||||
|
parsed, _ := value.Int64()
|
||||||
|
return parsed
|
||||||
|
case string:
|
||||||
|
parsed, _ := strconv.ParseInt(strings.TrimSpace(value), 10, 64)
|
||||||
|
return parsed
|
||||||
|
default:
|
||||||
|
return 0
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func jsonString(values map[string]any, key string) string {
|
||||||
|
value, _ := values[key].(string)
|
||||||
|
return strings.TrimSpace(value)
|
||||||
|
}
|
||||||
|
|
||||||
|
func normalizedDatabaseName(value string) string {
|
||||||
|
value = strings.TrimSpace(value)
|
||||||
|
if value == "" {
|
||||||
|
return ""
|
||||||
|
}
|
||||||
|
for _, char := range value {
|
||||||
|
if char == '_' || unicode.IsDigit(char) || unicode.IsLetter(char) {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
return ""
|
||||||
|
}
|
||||||
|
return value
|
||||||
|
}
|
||||||
306
services/activity-service/cmd/audit-wheel-rewards/main_test.go
Normal file
306
services/activity-service/cmd/audit-wheel-rewards/main_test.go
Normal file
@ -0,0 +1,306 @@
|
|||||||
|
package main
|
||||||
|
|
||||||
|
import (
|
||||||
|
"context"
|
||||||
|
"database/sql"
|
||||||
|
"encoding/json"
|
||||||
|
"fmt"
|
||||||
|
"regexp"
|
||||||
|
"testing"
|
||||||
|
|
||||||
|
"github.com/DATA-DOG/go-sqlmock"
|
||||||
|
domain "hyapp/services/activity-service/internal/domain/wheel"
|
||||||
|
)
|
||||||
|
|
||||||
|
func TestAuditRowCoinStates(t *testing.T) {
|
||||||
|
tests := []struct {
|
||||||
|
name string
|
||||||
|
prepare func(sqlmock.Sqlmock, rewardPayload)
|
||||||
|
wantState string
|
||||||
|
wantItem string
|
||||||
|
}{
|
||||||
|
{
|
||||||
|
name: "fully paid",
|
||||||
|
prepare: func(mock sqlmock.Sqlmock, payload rewardPayload) {
|
||||||
|
metadata, _ := json.Marshal(map[string]any{
|
||||||
|
"target_user_id": payload.UserID,
|
||||||
|
"amount": int64(200),
|
||||||
|
"draw_id": payload.DrawID,
|
||||||
|
"wheel_id": payload.WheelID,
|
||||||
|
"selected_tier_id": "coin-200",
|
||||||
|
"reason": "wheel_reward",
|
||||||
|
"visible_region_id": payload.VisibleRegionID,
|
||||||
|
})
|
||||||
|
mock.ExpectQuery(regexp.QuoteMeta(coinAuditQuery("hyapp_wallet"))).
|
||||||
|
WithArgs("fami", "wheel_reward:draw-1").
|
||||||
|
WillReturnRows(coinAuditRows().AddRow("wtx-1", "wheel_reward", "succeeded", string(metadata)))
|
||||||
|
},
|
||||||
|
wantState: "fully_paid",
|
||||||
|
wantItem: "paid",
|
||||||
|
},
|
||||||
|
{
|
||||||
|
name: "fully unpaid uses errors Is",
|
||||||
|
prepare: func(mock sqlmock.Sqlmock, _ rewardPayload) {
|
||||||
|
mock.ExpectQuery(regexp.QuoteMeta(coinAuditQuery("hyapp_wallet"))).
|
||||||
|
WithArgs("fami", "wheel_reward:draw-1").
|
||||||
|
WillReturnError(fmt.Errorf("wrapped wallet miss: %w", sql.ErrNoRows))
|
||||||
|
},
|
||||||
|
wantState: "fully_unpaid",
|
||||||
|
wantItem: "unpaid",
|
||||||
|
},
|
||||||
|
{
|
||||||
|
name: "conflict on immutable amount",
|
||||||
|
prepare: func(mock sqlmock.Sqlmock, payload rewardPayload) {
|
||||||
|
metadata, _ := json.Marshal(map[string]any{
|
||||||
|
"target_user_id": payload.UserID,
|
||||||
|
"amount": int64(201),
|
||||||
|
"draw_id": payload.DrawID,
|
||||||
|
"wheel_id": payload.WheelID,
|
||||||
|
"selected_tier_id": "coin-200",
|
||||||
|
"reason": "wheel_reward",
|
||||||
|
"visible_region_id": payload.VisibleRegionID,
|
||||||
|
})
|
||||||
|
mock.ExpectQuery(regexp.QuoteMeta(coinAuditQuery("hyapp_wallet"))).
|
||||||
|
WithArgs("fami", "wheel_reward:draw-1").
|
||||||
|
WillReturnRows(coinAuditRows().AddRow("wtx-conflict", "wheel_reward", "succeeded", string(metadata)))
|
||||||
|
},
|
||||||
|
wantState: "conflict",
|
||||||
|
wantItem: "conflict",
|
||||||
|
},
|
||||||
|
{
|
||||||
|
name: "conflict on biz type",
|
||||||
|
prepare: func(mock sqlmock.Sqlmock, payload rewardPayload) {
|
||||||
|
metadata := matchingCoinMetadata(payload)
|
||||||
|
mock.ExpectQuery(regexp.QuoteMeta(coinAuditQuery("hyapp_wallet"))).
|
||||||
|
WithArgs("fami", "wheel_reward:draw-1").
|
||||||
|
WillReturnRows(coinAuditRows().AddRow("wtx-wrong-biz", "task_reward", "succeeded", metadata))
|
||||||
|
},
|
||||||
|
wantState: "conflict",
|
||||||
|
wantItem: "conflict",
|
||||||
|
},
|
||||||
|
{
|
||||||
|
name: "conflict on visible region",
|
||||||
|
prepare: func(mock sqlmock.Sqlmock, payload rewardPayload) {
|
||||||
|
metadata := matchingCoinMetadata(payload)
|
||||||
|
var values map[string]any
|
||||||
|
_ = json.Unmarshal([]byte(metadata), &values)
|
||||||
|
values["visible_region_id"] = int64(99)
|
||||||
|
mismatch, _ := json.Marshal(values)
|
||||||
|
mock.ExpectQuery(regexp.QuoteMeta(coinAuditQuery("hyapp_wallet"))).
|
||||||
|
WithArgs("fami", "wheel_reward:draw-1").
|
||||||
|
WillReturnRows(coinAuditRows().AddRow("wtx-wrong-region", "wheel_reward", "succeeded", string(mismatch)))
|
||||||
|
},
|
||||||
|
wantState: "conflict",
|
||||||
|
wantItem: "conflict",
|
||||||
|
},
|
||||||
|
}
|
||||||
|
|
||||||
|
for _, test := range tests {
|
||||||
|
t.Run(test.name, func(t *testing.T) {
|
||||||
|
db, mock, err := sqlmock.New()
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("create sqlmock: %v", err)
|
||||||
|
}
|
||||||
|
defer db.Close()
|
||||||
|
payload := rewardPayload{
|
||||||
|
AppCode: "fami",
|
||||||
|
DrawID: "draw-1",
|
||||||
|
UserID: 1001,
|
||||||
|
WheelID: "classic",
|
||||||
|
VisibleRegionID: 9,
|
||||||
|
Rewards: []domain.DrawReward{{SelectedTierID: "coin-200", RewardType: domain.RewardTypeCoin, RewardCoins: 200}},
|
||||||
|
}
|
||||||
|
test.prepare(mock, payload)
|
||||||
|
audit, err := auditRow(context.Background(), db, "hyapp_wallet", auditOutboxRow(t, payload))
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("auditRow returned error: %v", err)
|
||||||
|
}
|
||||||
|
if audit.State != test.wantState || len(audit.Items) != 1 || audit.Items[0].State != test.wantItem {
|
||||||
|
t.Fatalf("unexpected audit: %+v", audit)
|
||||||
|
}
|
||||||
|
if err := mock.ExpectationsWereMet(); err != nil {
|
||||||
|
t.Fatalf("unmet SQL expectations: %v", err)
|
||||||
|
}
|
||||||
|
})
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestAuditRowResourceStates(t *testing.T) {
|
||||||
|
tests := []struct {
|
||||||
|
name string
|
||||||
|
prepare func(sqlmock.Sqlmock)
|
||||||
|
wantState string
|
||||||
|
wantItem string
|
||||||
|
}{
|
||||||
|
{
|
||||||
|
name: "fully paid",
|
||||||
|
prepare: func(mock sqlmock.Sqlmock) {
|
||||||
|
mock.ExpectQuery(regexp.QuoteMeta(resourceAuditQuery("hyapp_wallet"))).
|
||||||
|
WithArgs("fami", "wheel_resource:draw-1").
|
||||||
|
WillReturnRows(resourceAuditRows().AddRow("rgr-1", "succeeded", int64(1001), "wheel_reward", "resource", "326", "wheel_reward", int64(1001), int64(2), int64(5000), int64(326)))
|
||||||
|
},
|
||||||
|
wantState: "fully_paid",
|
||||||
|
wantItem: "paid",
|
||||||
|
},
|
||||||
|
{
|
||||||
|
name: "fully unpaid uses errors Is",
|
||||||
|
prepare: func(mock sqlmock.Sqlmock) {
|
||||||
|
mock.ExpectQuery(regexp.QuoteMeta(resourceAuditQuery("hyapp_wallet"))).
|
||||||
|
WithArgs("fami", "wheel_resource:draw-1").
|
||||||
|
WillReturnError(fmt.Errorf("wrapped resource miss: %w", sql.ErrNoRows))
|
||||||
|
},
|
||||||
|
wantState: "fully_unpaid",
|
||||||
|
wantItem: "unpaid",
|
||||||
|
},
|
||||||
|
{
|
||||||
|
name: "conflict rejects legacy active status",
|
||||||
|
prepare: func(mock sqlmock.Sqlmock) {
|
||||||
|
mock.ExpectQuery(regexp.QuoteMeta(resourceAuditQuery("hyapp_wallet"))).
|
||||||
|
WithArgs("fami", "wheel_resource:draw-1").
|
||||||
|
WillReturnRows(resourceAuditRows().AddRow("rgr-conflict", "active", int64(1001), "wheel_reward", "resource", "326", "wheel_reward", int64(1001), int64(2), int64(5000), int64(326)))
|
||||||
|
},
|
||||||
|
wantState: "conflict",
|
||||||
|
wantItem: "conflict",
|
||||||
|
},
|
||||||
|
}
|
||||||
|
|
||||||
|
for _, test := range tests {
|
||||||
|
t.Run(test.name, func(t *testing.T) {
|
||||||
|
db, mock, err := sqlmock.New()
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("create sqlmock: %v", err)
|
||||||
|
}
|
||||||
|
defer db.Close()
|
||||||
|
payload := rewardPayload{
|
||||||
|
AppCode: "fami",
|
||||||
|
DrawID: "draw-1",
|
||||||
|
UserID: 1001,
|
||||||
|
WheelID: "classic",
|
||||||
|
Rewards: []domain.DrawReward{{
|
||||||
|
SelectedTierID: "gift-326",
|
||||||
|
RewardType: domain.RewardTypeGift,
|
||||||
|
RewardID: "326",
|
||||||
|
RewardCount: 2,
|
||||||
|
MetadataJSON: `{"resource_id":326,"duration_ms":5000}`,
|
||||||
|
}},
|
||||||
|
}
|
||||||
|
test.prepare(mock)
|
||||||
|
audit, err := auditRow(context.Background(), db, "hyapp_wallet", auditOutboxRow(t, payload))
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("auditRow returned error: %v", err)
|
||||||
|
}
|
||||||
|
if audit.State != test.wantState || len(audit.Items) != 1 || audit.Items[0].State != test.wantItem {
|
||||||
|
t.Fatalf("unexpected audit: %+v", audit)
|
||||||
|
}
|
||||||
|
if err := mock.ExpectationsWereMet(); err != nil {
|
||||||
|
t.Fatalf("unmet SQL expectations: %v", err)
|
||||||
|
}
|
||||||
|
})
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestLoadUnsettledRowsUsesAppScopedIndexesAndGlobalLimit(t *testing.T) {
|
||||||
|
db, mock, err := sqlmock.New()
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("create sqlmock: %v", err)
|
||||||
|
}
|
||||||
|
defer db.Close()
|
||||||
|
const limit = 3
|
||||||
|
columns := []string{"app_code", "outbox_id", "payload", "status", "retry_count", "created_at_ms"}
|
||||||
|
|
||||||
|
// normalizeAuditAppCodes sorts and deduplicates, so SQL order is deterministic regardless of flag order.
|
||||||
|
mock.ExpectQuery(regexp.QuoteMeta(unsettledRetryQuery())).
|
||||||
|
WithArgs("fami", domain.EventTypeWheelRewardSettlement, domain.OutboxStatusPending, domain.OutboxStatusRetryable, limit).
|
||||||
|
WillReturnRows(sqlmock.NewRows(columns).
|
||||||
|
AddRow("fami", "fami-pending", []byte(`{}`), domain.OutboxStatusPending, 0, int64(20)))
|
||||||
|
mock.ExpectQuery(regexp.QuoteMeta(unsettledDeliveringQuery())).
|
||||||
|
WithArgs("fami", domain.EventTypeWheelRewardSettlement, domain.OutboxStatusDelivering, limit).
|
||||||
|
WillReturnRows(sqlmock.NewRows(columns).
|
||||||
|
AddRow("fami", "fami-delivering", []byte(`{}`), domain.OutboxStatusDelivering, 1, int64(10)))
|
||||||
|
mock.ExpectQuery(regexp.QuoteMeta(unsettledRetryQuery())).
|
||||||
|
WithArgs("lalu", domain.EventTypeWheelRewardSettlement, domain.OutboxStatusPending, domain.OutboxStatusRetryable, 1).
|
||||||
|
WillReturnRows(sqlmock.NewRows(columns).
|
||||||
|
AddRow("lalu", "lalu-oldest", []byte(`{}`), domain.OutboxStatusRetryable, 2, int64(5)))
|
||||||
|
mock.ExpectQuery(regexp.QuoteMeta(unsettledDeliveringQuery())).
|
||||||
|
WithArgs("lalu", domain.EventTypeWheelRewardSettlement, domain.OutboxStatusDelivering, 1).
|
||||||
|
WillReturnRows(sqlmock.NewRows(columns).
|
||||||
|
AddRow("lalu", "lalu-delivering", []byte(`{}`), domain.OutboxStatusDelivering, 1, int64(15)))
|
||||||
|
|
||||||
|
rows, err := loadUnsettledRows(context.Background(), db, []string{"lalu", "fami", "fami"}, limit)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("loadUnsettledRows returned error: %v", err)
|
||||||
|
}
|
||||||
|
if len(rows) != limit {
|
||||||
|
t.Fatalf("global limit mismatch: got=%d want=%d rows=%+v", len(rows), limit, rows)
|
||||||
|
}
|
||||||
|
wantIDs := []string{"fami-delivering", "fami-pending", "lalu-oldest"}
|
||||||
|
for index, wantID := range wantIDs {
|
||||||
|
if rows[index].OutboxID != wantID {
|
||||||
|
t.Fatalf("deterministic order[%d]=%q want=%q rows=%+v", index, rows[index].OutboxID, wantID, rows)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if err := mock.ExpectationsWereMet(); err != nil {
|
||||||
|
t.Fatalf("unmet SQL expectations: %v", err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestNormalizeAuditAppCodesRequiresRepeatedExplicitScope(t *testing.T) {
|
||||||
|
if _, err := normalizeAuditAppCodes(nil); err == nil {
|
||||||
|
t.Fatal("missing --app-code must be rejected before opening or querying MySQL")
|
||||||
|
}
|
||||||
|
values, err := normalizeAuditAppCodes([]string{" LALU ", "fami", "lalu", "huwaa"})
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("normalizeAuditAppCodes returned error: %v", err)
|
||||||
|
}
|
||||||
|
want := []string{"fami", "huwaa", "lalu"}
|
||||||
|
if len(values) != len(want) {
|
||||||
|
t.Fatalf("normalized app count=%d want=%d values=%+v", len(values), len(want), values)
|
||||||
|
}
|
||||||
|
for index := range want {
|
||||||
|
if values[index] != want[index] {
|
||||||
|
t.Fatalf("normalized app[%d]=%q want=%q", index, values[index], want[index])
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func auditOutboxRow(t *testing.T, payload rewardPayload) outboxRow {
|
||||||
|
t.Helper()
|
||||||
|
body, err := json.Marshal(payload)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("marshal payload: %v", err)
|
||||||
|
}
|
||||||
|
return outboxRow{AppCode: payload.AppCode, OutboxID: "wheel_reward_" + payload.DrawID, Payload: body, Status: domain.OutboxStatusPending}
|
||||||
|
}
|
||||||
|
|
||||||
|
func resourceAuditRows() *sqlmock.Rows {
|
||||||
|
return sqlmock.NewRows([]string{
|
||||||
|
"grant_id",
|
||||||
|
"status",
|
||||||
|
"target_user_id",
|
||||||
|
"grant_source",
|
||||||
|
"grant_subject_type",
|
||||||
|
"grant_subject_id",
|
||||||
|
"reason",
|
||||||
|
"operator_user_id",
|
||||||
|
"quantity",
|
||||||
|
"duration_ms",
|
||||||
|
"resource_id",
|
||||||
|
})
|
||||||
|
}
|
||||||
|
|
||||||
|
func coinAuditRows() *sqlmock.Rows {
|
||||||
|
return sqlmock.NewRows([]string{"transaction_id", "biz_type", "status", "metadata_json"})
|
||||||
|
}
|
||||||
|
|
||||||
|
func matchingCoinMetadata(payload rewardPayload) string {
|
||||||
|
body, _ := json.Marshal(map[string]any{
|
||||||
|
"target_user_id": payload.UserID,
|
||||||
|
"amount": int64(200),
|
||||||
|
"draw_id": payload.DrawID,
|
||||||
|
"wheel_id": payload.WheelID,
|
||||||
|
"selected_tier_id": "coin-200",
|
||||||
|
"reason": "wheel_reward",
|
||||||
|
"visible_region_id": payload.VisibleRegionID,
|
||||||
|
})
|
||||||
|
return string(body)
|
||||||
|
}
|
||||||
@ -30,6 +30,13 @@ user_leaderboard_worker:
|
|||||||
redis_password: ""
|
redis_password: ""
|
||||||
redis_db: 0
|
redis_db: 0
|
||||||
key_prefix: "activity:user_leaderboard"
|
key_prefix: "activity:user_leaderboard"
|
||||||
|
wheel_reward_worker:
|
||||||
|
enabled: true
|
||||||
|
app_codes: ["lalu", "fami", "huwaa"]
|
||||||
|
poll_interval: "1s"
|
||||||
|
batch_size: 50
|
||||||
|
lock_ttl: "30s"
|
||||||
|
max_retry: 16
|
||||||
message_action_confirm_worker:
|
message_action_confirm_worker:
|
||||||
enabled: true
|
enabled: true
|
||||||
outbox_poll_interval: "1s"
|
outbox_poll_interval: "1s"
|
||||||
|
|||||||
@ -30,6 +30,13 @@ user_leaderboard_worker:
|
|||||||
redis_password: "TENCENT_REDIS_PASSWORD"
|
redis_password: "TENCENT_REDIS_PASSWORD"
|
||||||
redis_db: 0
|
redis_db: 0
|
||||||
key_prefix: "activity:user_leaderboard"
|
key_prefix: "activity:user_leaderboard"
|
||||||
|
wheel_reward_worker:
|
||||||
|
enabled: true
|
||||||
|
app_codes: ["lalu", "fami", "huwaa"]
|
||||||
|
poll_interval: "1s"
|
||||||
|
batch_size: 50
|
||||||
|
lock_ttl: "30s"
|
||||||
|
max_retry: 16
|
||||||
message_action_confirm_worker:
|
message_action_confirm_worker:
|
||||||
enabled: true
|
enabled: true
|
||||||
outbox_poll_interval: "1s"
|
outbox_poll_interval: "1s"
|
||||||
|
|||||||
@ -30,6 +30,13 @@ user_leaderboard_worker:
|
|||||||
redis_password: ""
|
redis_password: ""
|
||||||
redis_db: 0
|
redis_db: 0
|
||||||
key_prefix: "activity:user_leaderboard"
|
key_prefix: "activity:user_leaderboard"
|
||||||
|
wheel_reward_worker:
|
||||||
|
enabled: true
|
||||||
|
app_codes: ["lalu", "fami", "huwaa"]
|
||||||
|
poll_interval: "1s"
|
||||||
|
batch_size: 50
|
||||||
|
lock_ttl: "30s"
|
||||||
|
max_retry: 16
|
||||||
message_action_confirm_worker:
|
message_action_confirm_worker:
|
||||||
enabled: true
|
enabled: true
|
||||||
outbox_poll_interval: "1s"
|
outbox_poll_interval: "1s"
|
||||||
|
|||||||
@ -21,6 +21,7 @@ import (
|
|||||||
firstrechargeservice "hyapp/services/activity-service/internal/service/firstrecharge"
|
firstrechargeservice "hyapp/services/activity-service/internal/service/firstrecharge"
|
||||||
inviteactivityservice "hyapp/services/activity-service/internal/service/inviteactivity"
|
inviteactivityservice "hyapp/services/activity-service/internal/service/inviteactivity"
|
||||||
messageservice "hyapp/services/activity-service/internal/service/message"
|
messageservice "hyapp/services/activity-service/internal/service/message"
|
||||||
|
wheelservice "hyapp/services/activity-service/internal/service/wheel"
|
||||||
mysqlstorage "hyapp/services/activity-service/internal/storage/mysql"
|
mysqlstorage "hyapp/services/activity-service/internal/storage/mysql"
|
||||||
)
|
)
|
||||||
|
|
||||||
@ -37,10 +38,13 @@ type App struct {
|
|||||||
cumulativeRecharge *cumulativerechargeservice.Service
|
cumulativeRecharge *cumulativerechargeservice.Service
|
||||||
inviteActivityReward *inviteactivityservice.Service
|
inviteActivityReward *inviteactivityservice.Service
|
||||||
actionConfirm *messageservice.ActionConfirmService
|
actionConfirm *messageservice.ActionConfirmService
|
||||||
|
wheel *wheelservice.Service
|
||||||
broadcastWorkerEnabled bool
|
broadcastWorkerEnabled bool
|
||||||
messageActionWorkerEnabled bool
|
messageActionWorkerEnabled bool
|
||||||
|
wheelRewardWorkerEnabled bool
|
||||||
workerNodeID string
|
workerNodeID string
|
||||||
messageActionWorkerOptions config.MessageActionConfirmWorkerConfig
|
messageActionWorkerOptions config.MessageActionConfirmWorkerConfig
|
||||||
|
wheelRewardWorkerOptions config.WheelRewardWorkerConfig
|
||||||
messageActionOutboxTopic string
|
messageActionOutboxTopic string
|
||||||
userLeaderboardRedisClose func() error
|
userLeaderboardRedisClose func() error
|
||||||
mqConsumers []*rocketmqx.Consumer
|
mqConsumers []*rocketmqx.Consumer
|
||||||
@ -133,10 +137,13 @@ func New(cfg config.Config) (*App, error) {
|
|||||||
cumulativeRecharge: services.cumulativeRecharge,
|
cumulativeRecharge: services.cumulativeRecharge,
|
||||||
inviteActivityReward: services.inviteActivityReward,
|
inviteActivityReward: services.inviteActivityReward,
|
||||||
actionConfirm: services.actionConfirm,
|
actionConfirm: services.actionConfirm,
|
||||||
|
wheel: services.wheel,
|
||||||
broadcastWorkerEnabled: cfg.Broadcast.Enabled,
|
broadcastWorkerEnabled: cfg.Broadcast.Enabled,
|
||||||
messageActionWorkerEnabled: cfg.MessageActionConfirmWorker.Enabled,
|
messageActionWorkerEnabled: cfg.MessageActionConfirmWorker.Enabled,
|
||||||
|
wheelRewardWorkerEnabled: cfg.WheelRewardWorker.Enabled,
|
||||||
workerNodeID: cfg.NodeID,
|
workerNodeID: cfg.NodeID,
|
||||||
messageActionWorkerOptions: cfg.MessageActionConfirmWorker,
|
messageActionWorkerOptions: cfg.MessageActionConfirmWorker,
|
||||||
|
wheelRewardWorkerOptions: cfg.WheelRewardWorker,
|
||||||
messageActionOutboxTopic: cfg.RocketMQ.MessageActionOutbox.Topic,
|
messageActionOutboxTopic: cfg.RocketMQ.MessageActionOutbox.Topic,
|
||||||
userLeaderboardRedisClose: userLeaderboardRedisClose,
|
userLeaderboardRedisClose: userLeaderboardRedisClose,
|
||||||
mqConsumers: mqConsumers,
|
mqConsumers: mqConsumers,
|
||||||
@ -166,6 +173,18 @@ func (a *App) Run() error {
|
|||||||
a.runMessageActionOutboxWorker(ctx)
|
a.runMessageActionOutboxWorker(ctx)
|
||||||
})
|
})
|
||||||
}
|
}
|
||||||
|
if a.wheelRewardWorkerEnabled && a.wheel != nil && a.workers != nil {
|
||||||
|
a.workers.Go(func(ctx context.Context) {
|
||||||
|
a.wheel.RunRewardWorker(ctx, wheelservice.RewardWorkerOptions{
|
||||||
|
WorkerID: a.workerNodeID + ":wheel-reward",
|
||||||
|
AppCodes: a.wheelRewardWorkerOptions.AppCodes,
|
||||||
|
PollInterval: a.wheelRewardWorkerOptions.PollInterval,
|
||||||
|
BatchSize: a.wheelRewardWorkerOptions.BatchSize,
|
||||||
|
LockTTL: a.wheelRewardWorkerOptions.LockTTL,
|
||||||
|
MaxRetry: a.wheelRewardWorkerOptions.MaxRetry,
|
||||||
|
})
|
||||||
|
})
|
||||||
|
}
|
||||||
err := a.server.Serve(a.listener)
|
err := a.server.Serve(a.listener)
|
||||||
if a.workers != nil {
|
if a.workers != nil {
|
||||||
a.workers.StopAndWait()
|
a.workers.StopAndWait()
|
||||||
|
|||||||
@ -40,6 +40,8 @@ type Config struct {
|
|||||||
TaskEventWorker TaskEventWorkerConfig `yaml:"task_event_worker"`
|
TaskEventWorker TaskEventWorkerConfig `yaml:"task_event_worker"`
|
||||||
// UserLeaderboardWorker 控制用户送礼榜单对 wallet_outbox 送礼事实的 Redis 投影。
|
// UserLeaderboardWorker 控制用户送礼榜单对 wallet_outbox 送礼事实的 Redis 投影。
|
||||||
UserLeaderboardWorker UserLeaderboardWorkerConfig `yaml:"user_leaderboard_worker"`
|
UserLeaderboardWorker UserLeaderboardWorkerConfig `yaml:"user_leaderboard_worker"`
|
||||||
|
// WheelRewardWorker 补偿抽奖事务已经确定、但同步钱包发奖未完成的 WheelRewardSettlement outbox。
|
||||||
|
WheelRewardWorker WheelRewardWorkerConfig `yaml:"wheel_reward_worker"`
|
||||||
// TencentIM 是 activity-service 发送全局/区域播报群消息的 REST 配置。
|
// TencentIM 是 activity-service 发送全局/区域播报群消息的 REST 配置。
|
||||||
TencentIM TencentIMConfig `yaml:"tencent_im"`
|
TencentIM TencentIMConfig `yaml:"tencent_im"`
|
||||||
// Broadcast 控制 IM 播报群 reconciler、outbox worker 和贵重礼物阈值。
|
// Broadcast 控制 IM 播报群 reconciler、outbox worker 和贵重礼物阈值。
|
||||||
@ -120,6 +122,17 @@ type UserLeaderboardWorkerConfig struct {
|
|||||||
KeyPrefix string `yaml:"key_prefix"`
|
KeyPrefix string `yaml:"key_prefix"`
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// WheelRewardWorkerConfig 控制转盘发奖 outbox 的租户分区、锁和重试边界。
|
||||||
|
// AppCodes 必须显式覆盖线上业务 App;worker 每轮逐 App claim,禁止空 context 默认只处理 lalu。
|
||||||
|
type WheelRewardWorkerConfig struct {
|
||||||
|
Enabled bool `yaml:"enabled"`
|
||||||
|
AppCodes []string `yaml:"app_codes"`
|
||||||
|
PollInterval time.Duration `yaml:"poll_interval"`
|
||||||
|
BatchSize int `yaml:"batch_size"`
|
||||||
|
LockTTL time.Duration `yaml:"lock_ttl"`
|
||||||
|
MaxRetry int `yaml:"max_retry"`
|
||||||
|
}
|
||||||
|
|
||||||
// TencentIMConfig 描述 activity-service 调用腾讯云 IM REST API 的配置。
|
// TencentIMConfig 描述 activity-service 调用腾讯云 IM REST API 的配置。
|
||||||
type TencentIMConfig struct {
|
type TencentIMConfig struct {
|
||||||
Enabled bool `yaml:"enabled"`
|
Enabled bool `yaml:"enabled"`
|
||||||
@ -266,6 +279,14 @@ func Default() Config {
|
|||||||
UserLeaderboardWorker: UserLeaderboardWorkerConfig{
|
UserLeaderboardWorker: UserLeaderboardWorkerConfig{
|
||||||
Enabled: false,
|
Enabled: false,
|
||||||
},
|
},
|
||||||
|
WheelRewardWorker: WheelRewardWorkerConfig{
|
||||||
|
Enabled: true,
|
||||||
|
AppCodes: []string{"lalu", "fami", "huwaa"},
|
||||||
|
PollInterval: time.Second,
|
||||||
|
BatchSize: 50,
|
||||||
|
LockTTL: 30 * time.Second,
|
||||||
|
MaxRetry: 16,
|
||||||
|
},
|
||||||
MessageActionConfirmWorker: MessageActionConfirmWorkerConfig{
|
MessageActionConfirmWorker: MessageActionConfirmWorkerConfig{
|
||||||
Enabled: true,
|
Enabled: true,
|
||||||
OutboxPollInterval: time.Second,
|
OutboxPollInterval: time.Second,
|
||||||
@ -447,6 +468,22 @@ func Load(path string) (Config, error) {
|
|||||||
}
|
}
|
||||||
cfg.UserLeaderboardWorker.RedisAddr = strings.TrimSpace(cfg.UserLeaderboardWorker.RedisAddr)
|
cfg.UserLeaderboardWorker.RedisAddr = strings.TrimSpace(cfg.UserLeaderboardWorker.RedisAddr)
|
||||||
cfg.UserLeaderboardWorker.KeyPrefix = strings.TrimSpace(cfg.UserLeaderboardWorker.KeyPrefix)
|
cfg.UserLeaderboardWorker.KeyPrefix = strings.TrimSpace(cfg.UserLeaderboardWorker.KeyPrefix)
|
||||||
|
cfg.WheelRewardWorker.AppCodes = normalizeAppCodes(cfg.WheelRewardWorker.AppCodes)
|
||||||
|
if len(cfg.WheelRewardWorker.AppCodes) == 0 {
|
||||||
|
cfg.WheelRewardWorker.AppCodes = append([]string(nil), Default().WheelRewardWorker.AppCodes...)
|
||||||
|
}
|
||||||
|
if cfg.WheelRewardWorker.PollInterval <= 0 {
|
||||||
|
cfg.WheelRewardWorker.PollInterval = time.Second
|
||||||
|
}
|
||||||
|
if cfg.WheelRewardWorker.BatchSize <= 0 {
|
||||||
|
cfg.WheelRewardWorker.BatchSize = 50
|
||||||
|
}
|
||||||
|
if cfg.WheelRewardWorker.LockTTL <= 0 {
|
||||||
|
cfg.WheelRewardWorker.LockTTL = 30 * time.Second
|
||||||
|
}
|
||||||
|
if cfg.WheelRewardWorker.MaxRetry <= 0 {
|
||||||
|
cfg.WheelRewardWorker.MaxRetry = 16
|
||||||
|
}
|
||||||
rocketMQ, err := normalizeRocketMQConfig(cfg.RocketMQ)
|
rocketMQ, err := normalizeRocketMQConfig(cfg.RocketMQ)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return Config{}, err
|
return Config{}, err
|
||||||
|
|||||||
@ -27,6 +27,9 @@ func TestLoadLocalEnablesOutboxMQConsumers(t *testing.T) {
|
|||||||
if cfg.UserLeaderboardWorker.RedisAddr != "127.0.0.1:13379" || cfg.UserLeaderboardWorker.KeyPrefix != "activity:user_leaderboard" {
|
if cfg.UserLeaderboardWorker.RedisAddr != "127.0.0.1:13379" || cfg.UserLeaderboardWorker.KeyPrefix != "activity:user_leaderboard" {
|
||||||
t.Fatalf("local user leaderboard worker redis config mismatch: %+v", cfg.UserLeaderboardWorker)
|
t.Fatalf("local user leaderboard worker redis config mismatch: %+v", cfg.UserLeaderboardWorker)
|
||||||
}
|
}
|
||||||
|
if !cfg.WheelRewardWorker.Enabled || len(cfg.WheelRewardWorker.AppCodes) != 3 || cfg.WheelRewardWorker.AppCodes[1] != "fami" || cfg.WheelRewardWorker.BatchSize <= 0 || cfg.WheelRewardWorker.LockTTL <= 0 {
|
||||||
|
t.Fatalf("local wheel reward compensation must cover every app with bounded claim settings: %+v", cfg.WheelRewardWorker)
|
||||||
|
}
|
||||||
if cfg.RocketMQ.WalletOutbox.InviteActivityConsumerGroup == "" || cfg.RocketMQ.WalletOutbox.UserLeaderboardConsumerGroup == "" || cfg.RocketMQ.WalletOutbox.VIPRebateNoticeConsumerGroup == "" || cfg.RocketMQ.UserOutbox.InviteActivityConsumerGroup == "" {
|
if cfg.RocketMQ.WalletOutbox.InviteActivityConsumerGroup == "" || cfg.RocketMQ.WalletOutbox.UserLeaderboardConsumerGroup == "" || cfg.RocketMQ.WalletOutbox.VIPRebateNoticeConsumerGroup == "" || cfg.RocketMQ.UserOutbox.InviteActivityConsumerGroup == "" {
|
||||||
t.Fatalf("local config must configure invite activity consumer groups: wallet=%+v user=%+v", cfg.RocketMQ.WalletOutbox, cfg.RocketMQ.UserOutbox)
|
t.Fatalf("local config must configure invite activity consumer groups: wallet=%+v user=%+v", cfg.RocketMQ.WalletOutbox, cfg.RocketMQ.UserOutbox)
|
||||||
}
|
}
|
||||||
@ -58,6 +61,9 @@ func TestLoadTencentExampleEnablesRoomOutboxMQ(t *testing.T) {
|
|||||||
if !cfg.UserLeaderboardWorker.Enabled || cfg.UserLeaderboardWorker.RedisAddr == "" || cfg.UserLeaderboardWorker.RedisPassword == "" || cfg.RocketMQ.WalletOutbox.UserLeaderboardConsumerGroup == "" {
|
if !cfg.UserLeaderboardWorker.Enabled || cfg.UserLeaderboardWorker.RedisAddr == "" || cfg.UserLeaderboardWorker.RedisPassword == "" || cfg.RocketMQ.WalletOutbox.UserLeaderboardConsumerGroup == "" {
|
||||||
t.Fatalf("tencent example must configure user leaderboard worker: worker=%+v wallet=%+v", cfg.UserLeaderboardWorker, cfg.RocketMQ.WalletOutbox)
|
t.Fatalf("tencent example must configure user leaderboard worker: worker=%+v wallet=%+v", cfg.UserLeaderboardWorker, cfg.RocketMQ.WalletOutbox)
|
||||||
}
|
}
|
||||||
|
if !cfg.WheelRewardWorker.Enabled || len(cfg.WheelRewardWorker.AppCodes) != 3 || cfg.WheelRewardWorker.AppCodes[2] != "huwaa" || cfg.WheelRewardWorker.MaxRetry != 16 {
|
||||||
|
t.Fatalf("tencent example must configure multi-app wheel reward compensation: %+v", cfg.WheelRewardWorker)
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
func TestLoadRejectsLuckyGiftConsumerWithoutBroadcastDelivery(t *testing.T) {
|
func TestLoadRejectsLuckyGiftConsumerWithoutBroadcastDelivery(t *testing.T) {
|
||||||
|
|||||||
@ -13,6 +13,12 @@ const (
|
|||||||
RewardTypeDress = "dress"
|
RewardTypeDress = "dress"
|
||||||
|
|
||||||
EventTypeWheelRewardSettlement = "WheelRewardSettlement"
|
EventTypeWheelRewardSettlement = "WheelRewardSettlement"
|
||||||
|
|
||||||
|
OutboxStatusPending = "pending"
|
||||||
|
OutboxStatusDelivering = "delivering"
|
||||||
|
OutboxStatusRetryable = "retryable"
|
||||||
|
OutboxStatusDelivered = "delivered"
|
||||||
|
OutboxStatusFailed = "failed"
|
||||||
)
|
)
|
||||||
|
|
||||||
// RuleConfig 是转盘配置的不可变版本快照。
|
// RuleConfig 是转盘配置的不可变版本快照。
|
||||||
@ -113,3 +119,19 @@ type DrawQuery struct {
|
|||||||
Page int32
|
Page int32
|
||||||
PageSize int32
|
PageSize int32
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// RewardOutboxRecord 是转盘发奖补偿 worker 的持久化工作项。
|
||||||
|
// Payload 始终保留抽奖事务写入时的不可变奖品快照;补偿时不能回读当前奖档配置,否则运营改配置后会改变历史应发权益。
|
||||||
|
type RewardOutboxRecord struct {
|
||||||
|
AppCode string
|
||||||
|
OutboxID string
|
||||||
|
Payload []byte
|
||||||
|
Status string
|
||||||
|
RetryCount int
|
||||||
|
NextRetryAtMS int64
|
||||||
|
LockedBy string
|
||||||
|
LockUntilMS int64
|
||||||
|
LastError string
|
||||||
|
CreatedAtMS int64
|
||||||
|
UpdatedAtMS int64
|
||||||
|
}
|
||||||
|
|||||||
@ -221,6 +221,11 @@ func (s *Service) HandleRoomEvent(ctx context.Context, envelope *roomeventsv1.Ev
|
|||||||
}
|
}
|
||||||
eventCtx := appcode.WithContext(ctx, envelope.GetAppCode())
|
eventCtx := appcode.WithContext(ctx, envelope.GetAppCode())
|
||||||
ownerUserID, err := s.fetchRoomOwner(eventCtx, roomID)
|
ownerUserID, err := s.fetchRoomOwner(eventCtx, roomID)
|
||||||
|
if xerr.IsCode(err, xerr.NotFound) {
|
||||||
|
// RoomGiftSent 已经是 room-service 提交过的历史事实;若消费时房间已被删除,当前 owner 已不可恢复,
|
||||||
|
// 该事件对“开业房主流水”只能收敛为 skipped。这里只吞精确 NotFound,网络、超时和内部错误仍返回 MQ 重试,避免把临时依赖故障误判成业务终态。
|
||||||
|
return 0, nil
|
||||||
|
}
|
||||||
if err != nil || ownerUserID <= 0 {
|
if err != nil || ownerUserID <= 0 {
|
||||||
return 0, err
|
return 0, err
|
||||||
}
|
}
|
||||||
|
|||||||
@ -6,6 +6,8 @@ import (
|
|||||||
"time"
|
"time"
|
||||||
|
|
||||||
"google.golang.org/grpc"
|
"google.golang.org/grpc"
|
||||||
|
"google.golang.org/grpc/codes"
|
||||||
|
"google.golang.org/grpc/status"
|
||||||
"google.golang.org/protobuf/proto"
|
"google.golang.org/protobuf/proto"
|
||||||
roomeventsv1 "hyapp.local/api/proto/events/room/v1"
|
roomeventsv1 "hyapp.local/api/proto/events/room/v1"
|
||||||
roomv1 "hyapp.local/api/proto/room/v1"
|
roomv1 "hyapp.local/api/proto/room/v1"
|
||||||
@ -109,6 +111,46 @@ func TestHandleRoomEventScoresApplicantRoomOwner(t *testing.T) {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func TestHandleRoomEventTreatsDeletedRoomAsTerminalSkip(t *testing.T) {
|
||||||
|
repo := &agencyOpeningFakeRepo{consumeStatus: domain.EventStatusConsumed}
|
||||||
|
room := &agencyOpeningFakeRoomClient{err: status.Error(codes.NotFound, "room not found")}
|
||||||
|
svc := New(repo, nil, nil, room)
|
||||||
|
body, err := proto.Marshal(&roomeventsv1.RoomGiftSent{SenderUserId: 9, TargetUserId: 88, GiftValue: 900})
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("marshal gift: %v", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
count, err := svc.HandleRoomEvent(context.Background(), &roomeventsv1.EventEnvelope{
|
||||||
|
EventId: "deleted-room-gift", RoomId: "deleted-room", EventType: "RoomGiftSent", AppCode: "lalu", Body: body,
|
||||||
|
})
|
||||||
|
if err != nil || count != 0 {
|
||||||
|
t.Fatalf("deleted room must be terminal skipped, count=%d err=%v", count, err)
|
||||||
|
}
|
||||||
|
if repo.consumed.EventID != "" {
|
||||||
|
t.Fatalf("deleted room must not create agency turnover: %+v", repo.consumed)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestHandleRoomEventKeepsTransientRoomFailureRetryable(t *testing.T) {
|
||||||
|
repo := &agencyOpeningFakeRepo{consumeStatus: domain.EventStatusConsumed}
|
||||||
|
room := &agencyOpeningFakeRoomClient{err: status.Error(codes.Unavailable, "room service unavailable")}
|
||||||
|
svc := New(repo, nil, nil, room)
|
||||||
|
body, err := proto.Marshal(&roomeventsv1.RoomGiftSent{SenderUserId: 9, TargetUserId: 88, GiftValue: 900})
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("marshal gift: %v", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
count, err := svc.HandleRoomEvent(context.Background(), &roomeventsv1.EventEnvelope{
|
||||||
|
EventId: "transient-room-gift", RoomId: "room-1", EventType: "RoomGiftSent", AppCode: "lalu", Body: body,
|
||||||
|
})
|
||||||
|
if err == nil || status.Code(err) != codes.Unavailable || count != 0 {
|
||||||
|
t.Fatalf("transient room failure must remain retryable, count=%d err=%v", count, err)
|
||||||
|
}
|
||||||
|
if repo.consumed.EventID != "" {
|
||||||
|
t.Fatalf("transient room failure must not create partial projection: %+v", repo.consumed)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
type agencyOpeningFakeRepo struct {
|
type agencyOpeningFakeRepo struct {
|
||||||
currentCycle domain.Cycle
|
currentCycle domain.Cycle
|
||||||
hasCurrent bool
|
hasCurrent bool
|
||||||
@ -179,10 +221,14 @@ func (s agencyOpeningFakeAgencySource) ResolveOwnerAgency(context.Context, int64
|
|||||||
type agencyOpeningFakeRoomClient struct {
|
type agencyOpeningFakeRoomClient struct {
|
||||||
ownerUserID int64
|
ownerUserID int64
|
||||||
roomID string
|
roomID string
|
||||||
|
err error
|
||||||
}
|
}
|
||||||
|
|
||||||
func (c *agencyOpeningFakeRoomClient) AdminGetRoom(_ context.Context, req *roomv1.AdminGetRoomRequest, _ ...grpc.CallOption) (*roomv1.AdminGetRoomResponse, error) {
|
func (c *agencyOpeningFakeRoomClient) AdminGetRoom(_ context.Context, req *roomv1.AdminGetRoomRequest, _ ...grpc.CallOption) (*roomv1.AdminGetRoomResponse, error) {
|
||||||
c.roomID = req.GetRoomId()
|
c.roomID = req.GetRoomId()
|
||||||
|
if c.err != nil {
|
||||||
|
return nil, c.err
|
||||||
|
}
|
||||||
return &roomv1.AdminGetRoomResponse{
|
return &roomv1.AdminGetRoomResponse{
|
||||||
Room: &roomv1.AdminRoomListItem{OwnerUserId: c.ownerUserID},
|
Room: &roomv1.AdminRoomListItem{OwnerUserId: c.ownerUserID},
|
||||||
}, nil
|
}, nil
|
||||||
|
|||||||
@ -2,6 +2,7 @@ package wheel
|
|||||||
|
|
||||||
import (
|
import (
|
||||||
"context"
|
"context"
|
||||||
|
"crypto/sha256"
|
||||||
"encoding/json"
|
"encoding/json"
|
||||||
"fmt"
|
"fmt"
|
||||||
"log/slog"
|
"log/slog"
|
||||||
@ -24,6 +25,9 @@ type Repository interface {
|
|||||||
ListWheelDraws(ctx context.Context, query domain.DrawQuery) ([]domain.DrawResult, int64, error)
|
ListWheelDraws(ctx context.Context, query domain.DrawQuery) ([]domain.DrawResult, int64, error)
|
||||||
GetWheelDrawSummary(ctx context.Context, query domain.DrawQuery) (domain.DrawSummary, error)
|
GetWheelDrawSummary(ctx context.Context, query domain.DrawQuery) (domain.DrawSummary, error)
|
||||||
MarkWheelDrawsGranted(ctx context.Context, appCode string, drawIDs []string, transactionID string, nowMS int64) error
|
MarkWheelDrawsGranted(ctx context.Context, appCode string, drawIDs []string, transactionID string, nowMS int64) error
|
||||||
|
ClaimWheelRewardOutbox(ctx context.Context, workerID string, nowMS int64, lockUntilMS int64, limit int, maxRetry int) ([]domain.RewardOutboxRecord, error)
|
||||||
|
MarkWheelRewardOutboxDelivered(ctx context.Context, outboxID string, workerID string, drawIDs []string, transactionID string, nowMS int64) error
|
||||||
|
MarkWheelRewardOutboxFailed(ctx context.Context, outboxID string, workerID string, nowMS int64, nextRetryAtMS int64, maxRetry int, lastError string) error
|
||||||
}
|
}
|
||||||
|
|
||||||
type WalletClient interface {
|
type WalletClient interface {
|
||||||
@ -128,26 +132,41 @@ func (s *Service) fulfillWheelRewardFastPath(ctx context.Context, cmd domain.Dra
|
|||||||
if result.RewardStatus == domain.StatusGranted {
|
if result.RewardStatus == domain.StatusGranted {
|
||||||
return result
|
return result
|
||||||
}
|
}
|
||||||
if s.wallet == nil {
|
fulfilled, err := s.fulfillWheelReward(ctx, cmd, result)
|
||||||
logx.Warn(ctx, "wheel_reward_wallet_not_configured", slog.String("draw_id", result.DrawID))
|
if err != nil {
|
||||||
|
// 抽奖结果已经在事务内确认,发放失败时不能回滚抽奖事实;保留 pending/outbox,补偿 worker 会使用相同 command_id 继续核账和补发。
|
||||||
|
logx.Error(ctx, "wheel_reward_fast_path_failed", err,
|
||||||
|
slog.String("draw_id", result.DrawID),
|
||||||
|
slog.String("command_id", result.CommandID),
|
||||||
|
)
|
||||||
return result
|
return result
|
||||||
}
|
}
|
||||||
|
result = fulfilled
|
||||||
|
drawIDs := []string{result.DrawID}
|
||||||
|
if result.DrawID == "" {
|
||||||
|
drawIDs = append([]string(nil), result.DrawIDs...)
|
||||||
|
}
|
||||||
|
if err := s.repository.MarkWheelDrawsGranted(appcode.WithContext(context.WithoutCancel(ctx), appcode.FromContext(ctx)), appcode.FromContext(ctx), drawIDs, result.WalletTransactionID, s.now().UTC().UnixMilli()); err != nil {
|
||||||
|
logx.Error(ctx, "wheel_reward_fast_path_mark_granted_failed", err, slog.String("draw_id", result.DrawID), slog.String("wallet_transaction_id", result.WalletTransactionID))
|
||||||
|
}
|
||||||
|
return result
|
||||||
|
}
|
||||||
|
|
||||||
|
// fulfillWheelReward 调用 wallet owner 的幂等账务命令完成一组奖品发放。
|
||||||
|
// CreditWheelReward/GrantResource 在钱包事务内都会先按稳定 command_id 和 request hash 查询既有交易或权益;因此 worker 在超时、进程崩溃或标记 delivered 失败后重放时,先得到原收据而不是再次入账。
|
||||||
|
func (s *Service) fulfillWheelReward(ctx context.Context, cmd domain.DrawCommand, result domain.DrawResult) (domain.DrawResult, error) {
|
||||||
|
if s.wallet == nil {
|
||||||
|
return result, xerr.New(xerr.Unavailable, "wheel reward wallet client is not configured")
|
||||||
|
}
|
||||||
rewards := wheelResultRewards(result)
|
rewards := wheelResultRewards(result)
|
||||||
if len(rewards) == 0 {
|
if len(rewards) == 0 {
|
||||||
return result
|
return result, xerr.New(xerr.InvalidArgument, "wheel reward snapshot is empty")
|
||||||
}
|
}
|
||||||
receipts := make([]string, 0, len(rewards))
|
receipts := make([]string, 0, len(rewards))
|
||||||
for index, reward := range rewards {
|
for index, reward := range rewards {
|
||||||
receipt, coinBalance, err := s.fulfillWheelRewardItem(ctx, cmd, result, reward, index, len(rewards))
|
receipt, coinBalance, err := s.fulfillWheelRewardItem(ctx, cmd, result, reward, index, len(rewards))
|
||||||
if err != nil {
|
if err != nil {
|
||||||
// 抽奖结果已经在事务内确认,发放失败时不能回滚抽奖事实;保留 pending/outbox,补偿 worker 或重试可以用同一 command_id 幂等补发。
|
return result, fmt.Errorf("fulfill reward item %d type=%s tier=%s: %w", index+1, reward.RewardType, reward.SelectedTierID, err)
|
||||||
logx.Error(ctx, "wheel_reward_fast_path_failed", err,
|
|
||||||
slog.String("draw_id", result.DrawID),
|
|
||||||
slog.String("command_id", result.CommandID),
|
|
||||||
slog.String("reward_type", reward.RewardType),
|
|
||||||
slog.String("selected_tier_id", reward.SelectedTierID),
|
|
||||||
)
|
|
||||||
return result
|
|
||||||
}
|
}
|
||||||
if receipt != "" {
|
if receipt != "" {
|
||||||
receipts = append(receipts, receipt)
|
receipts = append(receipts, receipt)
|
||||||
@ -157,15 +176,20 @@ func (s *Service) fulfillWheelRewardFastPath(ctx context.Context, cmd domain.Dra
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
result.RewardStatus = domain.StatusGranted
|
result.RewardStatus = domain.StatusGranted
|
||||||
result.WalletTransactionID = strings.Join(receipts, ",")
|
result.WalletTransactionID = wheelRewardReceiptReference(receipts)
|
||||||
drawIDs := []string{result.DrawID}
|
return result, nil
|
||||||
if result.DrawID == "" {
|
}
|
||||||
drawIDs = append([]string(nil), result.DrawIDs...)
|
|
||||||
|
func wheelRewardReceiptReference(receipts []string) string {
|
||||||
|
joined := strings.Join(receipts, ",")
|
||||||
|
const rewardTransactionIDMaxLength = 128
|
||||||
|
if len(joined) <= rewardTransactionIDMaxLength {
|
||||||
|
return joined
|
||||||
}
|
}
|
||||||
if err := s.repository.MarkWheelDrawsGranted(appcode.WithContext(context.WithoutCancel(ctx), appcode.FromContext(ctx)), appcode.FromContext(ctx), drawIDs, result.WalletTransactionID, s.now().UTC().UnixMilli()); err != nil {
|
// wheel_draw_records 的历史字段是 VARCHAR(128),而钱包 transaction/grant ID 各自约 68 字符;
|
||||||
logx.Error(ctx, "wheel_reward_fast_path_mark_granted_failed", err, slog.String("draw_id", result.DrawID), slog.String("wallet_transaction_id", result.WalletTransactionID))
|
// 多奖品直接逗号拼接会让钱包已经全部发放后仍无法标记 granted。超长集合保存稳定摘要,逐项原收据仍由钱包 owner 按派生 command_id 可审计查询。
|
||||||
}
|
sum := sha256.Sum256([]byte(joined))
|
||||||
return result
|
return fmt.Sprintf("batch:%d:%x", len(receipts), sum)
|
||||||
}
|
}
|
||||||
|
|
||||||
func (s *Service) fulfillWheelRewardItem(ctx context.Context, cmd domain.DrawCommand, result domain.DrawResult, reward domain.DrawReward, index, total int) (string, int64, error) {
|
func (s *Service) fulfillWheelRewardItem(ctx context.Context, cmd domain.DrawCommand, result domain.DrawResult, reward domain.DrawReward, index, total int) (string, int64, error) {
|
||||||
|
|||||||
@ -2,6 +2,8 @@ package wheel
|
|||||||
|
|
||||||
import (
|
import (
|
||||||
"context"
|
"context"
|
||||||
|
"errors"
|
||||||
|
"strings"
|
||||||
"testing"
|
"testing"
|
||||||
"time"
|
"time"
|
||||||
|
|
||||||
@ -96,11 +98,35 @@ func TestDrawFulfillsBatchCoinAndResourceRewards(t *testing.T) {
|
|||||||
if result.CoinBalanceAfter != 9900 {
|
if result.CoinBalanceAfter != 9900 {
|
||||||
t.Fatalf("coin balance mismatch: %d", result.CoinBalanceAfter)
|
t.Fatalf("coin balance mismatch: %d", result.CoinBalanceAfter)
|
||||||
}
|
}
|
||||||
|
if len(result.WalletTransactionID) > 128 {
|
||||||
|
t.Fatalf("batch reward receipt reference exceeds wheel_draw_records boundary: %d", len(result.WalletTransactionID))
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestWheelRewardReceiptReferenceCompactsMultiRewardReceipts(t *testing.T) {
|
||||||
|
receipts := []string{
|
||||||
|
"wtx_aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa",
|
||||||
|
"rgr_bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb",
|
||||||
|
}
|
||||||
|
first := wheelRewardReceiptReference(receipts)
|
||||||
|
second := wheelRewardReceiptReference(receipts)
|
||||||
|
if first != second || !strings.HasPrefix(first, "batch:2:") || len(first) > 128 {
|
||||||
|
t.Fatalf("receipt reference must be stable and bounded: first=%q second=%q", first, second)
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
type fakeWheelRepository struct {
|
type fakeWheelRepository struct {
|
||||||
executeResult domain.DrawResult
|
executeResult domain.DrawResult
|
||||||
markGrantedDrawIDs []string
|
markGrantedDrawIDs []string
|
||||||
|
claimRecords []domain.RewardOutboxRecord
|
||||||
|
claimApps []string
|
||||||
|
deliveredOutboxID string
|
||||||
|
deliveredWorkerID string
|
||||||
|
deliveredDrawIDs []string
|
||||||
|
deliveredTransactionID string
|
||||||
|
deliverErr error
|
||||||
|
failedOutboxID string
|
||||||
|
failedReason string
|
||||||
}
|
}
|
||||||
|
|
||||||
func (r *fakeWheelRepository) PublishWheelRuleConfig(context.Context, domain.RuleConfig, int64) (domain.RuleConfig, error) {
|
func (r *fakeWheelRepository) PublishWheelRuleConfig(context.Context, domain.RuleConfig, int64) (domain.RuleConfig, error) {
|
||||||
@ -128,6 +154,27 @@ func (r *fakeWheelRepository) MarkWheelDrawsGranted(_ context.Context, _ string,
|
|||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func (r *fakeWheelRepository) ClaimWheelRewardOutbox(ctx context.Context, _ string, _ int64, _ int64, _ int, _ int) ([]domain.RewardOutboxRecord, error) {
|
||||||
|
r.claimApps = append(r.claimApps, appcode.FromContext(ctx))
|
||||||
|
records := append([]domain.RewardOutboxRecord(nil), r.claimRecords...)
|
||||||
|
r.claimRecords = nil
|
||||||
|
return records, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func (r *fakeWheelRepository) MarkWheelRewardOutboxDelivered(_ context.Context, outboxID string, workerID string, drawIDs []string, transactionID string, _ int64) error {
|
||||||
|
r.deliveredOutboxID = outboxID
|
||||||
|
r.deliveredWorkerID = workerID
|
||||||
|
r.deliveredDrawIDs = append([]string(nil), drawIDs...)
|
||||||
|
r.deliveredTransactionID = transactionID
|
||||||
|
return r.deliverErr
|
||||||
|
}
|
||||||
|
|
||||||
|
func (r *fakeWheelRepository) MarkWheelRewardOutboxFailed(_ context.Context, outboxID string, _ string, _ int64, _ int64, _ int, lastError string) error {
|
||||||
|
r.failedOutboxID = outboxID
|
||||||
|
r.failedReason = lastError
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
type fakeWheelWallet struct {
|
type fakeWheelWallet struct {
|
||||||
coinCredits []*walletv1.CreditWheelRewardRequest
|
coinCredits []*walletv1.CreditWheelRewardRequest
|
||||||
resourceGrants []*walletv1.GrantResourceRequest
|
resourceGrants []*walletv1.GrantResourceRequest
|
||||||
@ -146,3 +193,85 @@ func (w *fakeWheelWallet) GrantResource(_ context.Context, req *walletv1.GrantRe
|
|||||||
w.resourceGrants = append(w.resourceGrants, req)
|
w.resourceGrants = append(w.resourceGrants, req)
|
||||||
return &walletv1.ResourceGrantResponse{Grant: &walletv1.ResourceGrant{GrantId: "grant-" + req.GetCommandId()}}, nil
|
return &walletv1.ResourceGrantResponse{Grant: &walletv1.ResourceGrant{GrantId: "grant-" + req.GetCommandId()}}, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func TestProcessPendingRewardsReplaysStableWalletCommandAndMarksDelivered(t *testing.T) {
|
||||||
|
repository := &fakeWheelRepository{claimRecords: []domain.RewardOutboxRecord{{
|
||||||
|
AppCode: "fami",
|
||||||
|
OutboxID: "wheel_reward_wheel_draw_replay",
|
||||||
|
// 线上历史 payload 的 rewards 使用 Go 默认字段名;补偿解码必须兼容该不可变快照,不能改读当前奖档。
|
||||||
|
Payload: []byte(`{"app_code":"fami","draw_id":"wheel_draw_replay","draw_ids":["wheel_draw_replay"],"command_id":"draw-command","user_id":1001,"wheel_id":"classic","rewards":[{"SelectedTierID":"tier-1","RewardType":"coin","RewardCoins":200}],"visible_region_id":9}`),
|
||||||
|
}}}
|
||||||
|
wallet := &fakeWheelWallet{coinBalanceAfter: 1200}
|
||||||
|
service := New(repository, WithWallet(wallet))
|
||||||
|
service.SetClock(func() time.Time { return time.UnixMilli(1700000000000).UTC() })
|
||||||
|
|
||||||
|
result, err := service.ProcessPendingRewards(context.Background(), RewardWorkerOptions{
|
||||||
|
WorkerID: "activity-1:wheel-reward",
|
||||||
|
AppCodes: []string{"fami"},
|
||||||
|
})
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("ProcessPendingRewards returned error: %v", err)
|
||||||
|
}
|
||||||
|
if result.Claimed != 1 || result.Delivered != 1 || result.Failed != 0 {
|
||||||
|
t.Fatalf("unexpected batch result: %+v", result)
|
||||||
|
}
|
||||||
|
if len(wallet.coinCredits) != 1 {
|
||||||
|
t.Fatalf("wallet credit count = %d, want 1", len(wallet.coinCredits))
|
||||||
|
}
|
||||||
|
request := wallet.coinCredits[0]
|
||||||
|
if request.GetCommandId() != "wheel_reward:wheel_draw_replay" || request.GetAppCode() != "fami" || request.GetTargetUserId() != 1001 || request.GetAmount() != 200 {
|
||||||
|
t.Fatalf("wallet reconciliation command mismatch: %+v", request)
|
||||||
|
}
|
||||||
|
if repository.deliveredOutboxID != "wheel_reward_wheel_draw_replay" || repository.deliveredWorkerID != "activity-1:wheel-reward" || repository.deliveredTransactionID != "tx-wheel_reward:wheel_draw_replay" {
|
||||||
|
t.Fatalf("delivered marker mismatch: %+v", repository)
|
||||||
|
}
|
||||||
|
if len(repository.deliveredDrawIDs) != 1 || repository.deliveredDrawIDs[0] != "wheel_draw_replay" {
|
||||||
|
t.Fatalf("delivered draw ids mismatch: %+v", repository.deliveredDrawIDs)
|
||||||
|
}
|
||||||
|
if len(repository.claimApps) != 1 || repository.claimApps[0] != "fami" {
|
||||||
|
t.Fatalf("claim must use explicit app partition, got %+v", repository.claimApps)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestProcessPendingRewardsRejectsCrossAppPayloadBeforeWallet(t *testing.T) {
|
||||||
|
repository := &fakeWheelRepository{claimRecords: []domain.RewardOutboxRecord{{
|
||||||
|
AppCode: "fami",
|
||||||
|
OutboxID: "wheel_reward_cross_app",
|
||||||
|
Payload: []byte(`{"app_code":"huwaa","draw_id":"cross_app","command_id":"cmd","user_id":1001,"wheel_id":"classic","reward_type":"coin","reward_coins":200,"selected_tier_id":"tier-1"}`),
|
||||||
|
}}}
|
||||||
|
wallet := &fakeWheelWallet{}
|
||||||
|
service := New(repository, WithWallet(wallet))
|
||||||
|
service.SetClock(func() time.Time { return time.UnixMilli(1700000000000).UTC() })
|
||||||
|
|
||||||
|
result, err := service.ProcessPendingRewards(context.Background(), RewardWorkerOptions{WorkerID: "worker", AppCodes: []string{"fami"}})
|
||||||
|
if err == nil || result.Claimed != 1 || result.Failed != 1 || result.Delivered != 0 {
|
||||||
|
t.Fatalf("cross-app payload must fail closed, result=%+v err=%v", result, err)
|
||||||
|
}
|
||||||
|
if len(wallet.coinCredits) != 0 || len(wallet.resourceGrants) != 0 {
|
||||||
|
t.Fatalf("cross-app payload must not reach wallet: coin=%d resource=%d", len(wallet.coinCredits), len(wallet.resourceGrants))
|
||||||
|
}
|
||||||
|
if repository.failedOutboxID != "wheel_reward_cross_app" || repository.failedReason == "" {
|
||||||
|
t.Fatalf("failed outbox marker missing: %+v", repository)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestProcessPendingRewardsLeavesCompletedWalletCommandForLeaseRecoveryWhenDeliveryMarkFails(t *testing.T) {
|
||||||
|
repository := &fakeWheelRepository{
|
||||||
|
deliverErr: errors.New("mysql unavailable"),
|
||||||
|
claimRecords: []domain.RewardOutboxRecord{{
|
||||||
|
AppCode: "lalu",
|
||||||
|
OutboxID: "wheel_reward_mark_failure",
|
||||||
|
Payload: []byte(`{"app_code":"lalu","draw_id":"mark_failure","command_id":"cmd","user_id":1001,"wheel_id":"classic","reward_type":"coin","reward_coins":20,"selected_tier_id":"tier-1"}`),
|
||||||
|
}},
|
||||||
|
}
|
||||||
|
wallet := &fakeWheelWallet{}
|
||||||
|
service := New(repository, WithWallet(wallet))
|
||||||
|
|
||||||
|
result, err := service.ProcessPendingRewards(context.Background(), RewardWorkerOptions{WorkerID: "worker", AppCodes: []string{"lalu"}})
|
||||||
|
if err == nil || result.Failed != 1 || len(wallet.coinCredits) != 1 {
|
||||||
|
t.Fatalf("delivery mark failure mismatch: result=%+v wallet=%d err=%v", result, len(wallet.coinCredits), err)
|
||||||
|
}
|
||||||
|
if repository.failedOutboxID != "" {
|
||||||
|
t.Fatalf("lost lease/delivery failure must stay delivering for safe idempotent reclaim, got failed marker %q", repository.failedOutboxID)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|||||||
218
services/activity-service/internal/service/wheel/worker.go
Normal file
218
services/activity-service/internal/service/wheel/worker.go
Normal file
@ -0,0 +1,218 @@
|
|||||||
|
package wheel
|
||||||
|
|
||||||
|
import (
|
||||||
|
"context"
|
||||||
|
"encoding/json"
|
||||||
|
"errors"
|
||||||
|
"fmt"
|
||||||
|
"log/slog"
|
||||||
|
"strings"
|
||||||
|
"time"
|
||||||
|
|
||||||
|
"hyapp/pkg/appcode"
|
||||||
|
"hyapp/pkg/logx"
|
||||||
|
"hyapp/pkg/xerr"
|
||||||
|
domain "hyapp/services/activity-service/internal/domain/wheel"
|
||||||
|
)
|
||||||
|
|
||||||
|
// RewardWorkerOptions 定义补偿 worker 的租户、锁和重试边界。
|
||||||
|
type RewardWorkerOptions struct {
|
||||||
|
WorkerID string
|
||||||
|
AppCodes []string
|
||||||
|
PollInterval time.Duration
|
||||||
|
BatchSize int
|
||||||
|
LockTTL time.Duration
|
||||||
|
MaxRetry int
|
||||||
|
}
|
||||||
|
|
||||||
|
// RewardWorkerBatchResult 只统计本轮 claim 后的终态;失败项已经释放为 retryable/failed,不会静默丢失。
|
||||||
|
type RewardWorkerBatchResult struct {
|
||||||
|
Claimed int
|
||||||
|
Delivered int
|
||||||
|
Failed int
|
||||||
|
}
|
||||||
|
|
||||||
|
type wheelRewardOutboxPayload struct {
|
||||||
|
AppCode string `json:"app_code"`
|
||||||
|
DrawID string `json:"draw_id"`
|
||||||
|
DrawIDs []string `json:"draw_ids"`
|
||||||
|
CommandID string `json:"command_id"`
|
||||||
|
UserID int64 `json:"user_id"`
|
||||||
|
WheelID string `json:"wheel_id"`
|
||||||
|
SelectedTierID string `json:"selected_tier_id"`
|
||||||
|
RewardType string `json:"reward_type"`
|
||||||
|
RewardID string `json:"reward_id"`
|
||||||
|
RewardCount int64 `json:"reward_count"`
|
||||||
|
RewardCoins int64 `json:"reward_coins"`
|
||||||
|
Rewards []domain.DrawReward `json:"rewards"`
|
||||||
|
VisibleRegionID int64 `json:"visible_region_id"`
|
||||||
|
}
|
||||||
|
|
||||||
|
// RunRewardWorker 持续逐 App 分区补偿 WheelRewardSettlement。
|
||||||
|
// AppCode 必须写进 context 后再 claim 和调用钱包,避免后台根 context 被 appcode 默认值悄悄收敛成 lalu。
|
||||||
|
func (s *Service) RunRewardWorker(ctx context.Context, options RewardWorkerOptions) {
|
||||||
|
options = normalizeRewardWorkerOptions(options)
|
||||||
|
ticker := time.NewTicker(options.PollInterval)
|
||||||
|
defer ticker.Stop()
|
||||||
|
for {
|
||||||
|
if _, err := s.ProcessPendingRewards(ctx, options); err != nil && ctx.Err() == nil {
|
||||||
|
logx.Error(ctx, "wheel_reward_outbox_batch_failed", err, slog.String("worker_id", options.WorkerID))
|
||||||
|
}
|
||||||
|
select {
|
||||||
|
case <-ctx.Done():
|
||||||
|
return
|
||||||
|
case <-ticker.C:
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// ProcessPendingRewards 执行一轮有界补偿。
|
||||||
|
// 钱包 RPC 使用 draw_id 派生的稳定 command_id;钱包 owner 会在同一事务内先查既有交易/权益及 request hash,命中则返回原收据,未命中才创建,因此 activity 只在收齐所有收据后标 delivered。
|
||||||
|
func (s *Service) ProcessPendingRewards(ctx context.Context, options RewardWorkerOptions) (RewardWorkerBatchResult, error) {
|
||||||
|
if err := s.requireRepository(); err != nil {
|
||||||
|
return RewardWorkerBatchResult{}, err
|
||||||
|
}
|
||||||
|
options = normalizeRewardWorkerOptions(options)
|
||||||
|
var result RewardWorkerBatchResult
|
||||||
|
var batchErr error
|
||||||
|
for _, appCodeValue := range options.AppCodes {
|
||||||
|
if ctx.Err() != nil {
|
||||||
|
return result, errors.Join(batchErr, ctx.Err())
|
||||||
|
}
|
||||||
|
appCtx := appcode.WithContext(ctx, appCodeValue)
|
||||||
|
nowMS := s.now().UTC().UnixMilli()
|
||||||
|
records, err := s.repository.ClaimWheelRewardOutbox(appCtx, options.WorkerID, nowMS, nowMS+options.LockTTL.Milliseconds(), options.BatchSize, options.MaxRetry)
|
||||||
|
if err != nil {
|
||||||
|
batchErr = errors.Join(batchErr, fmt.Errorf("claim app=%s: %w", appCodeValue, err))
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
result.Claimed += len(records)
|
||||||
|
for _, record := range records {
|
||||||
|
if err := s.processRewardOutboxRecord(appCtx, record, options); err != nil {
|
||||||
|
result.Failed++
|
||||||
|
batchErr = errors.Join(batchErr, fmt.Errorf("process app=%s outbox=%s: %w", appCodeValue, record.OutboxID, err))
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
result.Delivered++
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return result, batchErr
|
||||||
|
}
|
||||||
|
|
||||||
|
func (s *Service) processRewardOutboxRecord(ctx context.Context, record domain.RewardOutboxRecord, options RewardWorkerOptions) error {
|
||||||
|
payload, err := decodeWheelRewardOutboxPayload(record)
|
||||||
|
if err == nil {
|
||||||
|
var fulfilled domain.DrawResult
|
||||||
|
fulfilled, err = s.fulfillWheelReward(ctx, domain.DrawCommand{
|
||||||
|
CommandID: payload.CommandID,
|
||||||
|
WheelID: payload.WheelID,
|
||||||
|
UserID: payload.UserID,
|
||||||
|
VisibleRegionID: payload.VisibleRegionID,
|
||||||
|
}, domain.DrawResult{
|
||||||
|
DrawID: payload.DrawID,
|
||||||
|
DrawIDs: payload.DrawIDs,
|
||||||
|
CommandID: payload.CommandID,
|
||||||
|
WheelID: payload.WheelID,
|
||||||
|
SelectedTierID: payload.SelectedTierID,
|
||||||
|
RewardType: payload.RewardType,
|
||||||
|
RewardID: payload.RewardID,
|
||||||
|
RewardCount: payload.RewardCount,
|
||||||
|
RewardCoins: payload.RewardCoins,
|
||||||
|
RewardStatus: domain.StatusPending,
|
||||||
|
Rewards: payload.Rewards,
|
||||||
|
})
|
||||||
|
if err == nil {
|
||||||
|
drawIDs := payload.DrawIDs
|
||||||
|
if len(drawIDs) == 0 {
|
||||||
|
drawIDs = []string{payload.DrawID}
|
||||||
|
}
|
||||||
|
// draw、统计和 outbox 的最终提交带 worker fencing token;若 lease 已被接管,这里会回滚而不是覆盖新 worker。
|
||||||
|
return s.repository.MarkWheelRewardOutboxDelivered(ctx, record.OutboxID, options.WorkerID, drawIDs, fulfilled.WalletTransactionID, s.now().UTC().UnixMilli())
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
nowMS := s.now().UTC().UnixMilli()
|
||||||
|
nextRetryAtMS := nowMS + wheelRewardRetryDelay(record.RetryCount+1).Milliseconds()
|
||||||
|
markErr := s.repository.MarkWheelRewardOutboxFailed(ctx, record.OutboxID, options.WorkerID, nowMS, nextRetryAtMS, options.MaxRetry, err.Error())
|
||||||
|
if markErr != nil {
|
||||||
|
return errors.Join(err, fmt.Errorf("mark wheel reward retryable: %w", markErr))
|
||||||
|
}
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
|
||||||
|
func decodeWheelRewardOutboxPayload(record domain.RewardOutboxRecord) (wheelRewardOutboxPayload, error) {
|
||||||
|
var payload wheelRewardOutboxPayload
|
||||||
|
if err := json.Unmarshal(record.Payload, &payload); err != nil {
|
||||||
|
return payload, fmt.Errorf("decode wheel reward payload: %w", err)
|
||||||
|
}
|
||||||
|
recordApp := appcode.Normalize(record.AppCode)
|
||||||
|
payloadApp := appcode.Normalize(payload.AppCode)
|
||||||
|
if payload.AppCode != "" && payloadApp != recordApp {
|
||||||
|
return payload, xerr.New(xerr.Conflict, "wheel reward payload app_code does not match outbox partition")
|
||||||
|
}
|
||||||
|
payload.AppCode = recordApp
|
||||||
|
payload.DrawID = strings.TrimSpace(payload.DrawID)
|
||||||
|
payload.CommandID = strings.TrimSpace(payload.CommandID)
|
||||||
|
payload.WheelID = strings.TrimSpace(payload.WheelID)
|
||||||
|
if payload.DrawID == "" || payload.CommandID == "" || payload.WheelID == "" || payload.UserID <= 0 {
|
||||||
|
return payload, xerr.New(xerr.InvalidArgument, "wheel reward payload identity is incomplete")
|
||||||
|
}
|
||||||
|
if len(payload.DrawIDs) == 0 {
|
||||||
|
payload.DrawIDs = []string{payload.DrawID}
|
||||||
|
}
|
||||||
|
for index := range payload.DrawIDs {
|
||||||
|
payload.DrawIDs[index] = strings.TrimSpace(payload.DrawIDs[index])
|
||||||
|
if payload.DrawIDs[index] == "" {
|
||||||
|
return payload, xerr.New(xerr.InvalidArgument, "wheel reward payload contains empty draw_id")
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return payload, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func normalizeRewardWorkerOptions(options RewardWorkerOptions) RewardWorkerOptions {
|
||||||
|
options.WorkerID = strings.TrimSpace(options.WorkerID)
|
||||||
|
if options.WorkerID == "" {
|
||||||
|
options.WorkerID = "activity-wheel-reward"
|
||||||
|
}
|
||||||
|
seen := make(map[string]struct{}, len(options.AppCodes))
|
||||||
|
appCodes := make([]string, 0, len(options.AppCodes))
|
||||||
|
for _, value := range options.AppCodes {
|
||||||
|
value = appcode.Normalize(value)
|
||||||
|
if _, exists := seen[value]; exists {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
seen[value] = struct{}{}
|
||||||
|
appCodes = append(appCodes, value)
|
||||||
|
}
|
||||||
|
if len(appCodes) == 0 {
|
||||||
|
appCodes = []string{"lalu", "fami", "huwaa"}
|
||||||
|
}
|
||||||
|
options.AppCodes = appCodes
|
||||||
|
if options.PollInterval <= 0 {
|
||||||
|
options.PollInterval = time.Second
|
||||||
|
}
|
||||||
|
if options.BatchSize <= 0 {
|
||||||
|
options.BatchSize = 50
|
||||||
|
}
|
||||||
|
if options.LockTTL <= 0 {
|
||||||
|
options.LockTTL = 30 * time.Second
|
||||||
|
}
|
||||||
|
if options.MaxRetry <= 0 {
|
||||||
|
options.MaxRetry = 16
|
||||||
|
}
|
||||||
|
return options
|
||||||
|
}
|
||||||
|
|
||||||
|
func wheelRewardRetryDelay(attempt int) time.Duration {
|
||||||
|
if attempt < 1 {
|
||||||
|
attempt = 1
|
||||||
|
}
|
||||||
|
if attempt > 7 {
|
||||||
|
attempt = 7
|
||||||
|
}
|
||||||
|
delay := time.Second * time.Duration(1<<(attempt-1))
|
||||||
|
if delay > time.Minute {
|
||||||
|
return time.Minute
|
||||||
|
}
|
||||||
|
return delay
|
||||||
|
}
|
||||||
@ -0,0 +1,212 @@
|
|||||||
|
package mysql
|
||||||
|
|
||||||
|
import (
|
||||||
|
"context"
|
||||||
|
"strings"
|
||||||
|
|
||||||
|
"hyapp/pkg/appcode"
|
||||||
|
"hyapp/pkg/xerr"
|
||||||
|
domain "hyapp/services/activity-service/internal/domain/wheel"
|
||||||
|
)
|
||||||
|
|
||||||
|
// ClaimWheelRewardOutbox 抢占一个 App 分区内可补偿的转盘发奖事实。
|
||||||
|
// 行锁和 SKIP LOCKED 允许双节点并行;过期 delivering 可被接管,而仍持有效 lease 的记录不会被第二个节点重复处理。
|
||||||
|
func (r *Repository) ClaimWheelRewardOutbox(ctx context.Context, workerID string, nowMS int64, lockUntilMS int64, limit int, maxRetry int) ([]domain.RewardOutboxRecord, error) {
|
||||||
|
if r == nil || r.db == nil {
|
||||||
|
return nil, xerr.New(xerr.Unavailable, "mysql repository is not configured")
|
||||||
|
}
|
||||||
|
workerID = strings.TrimSpace(workerID)
|
||||||
|
if workerID == "" {
|
||||||
|
return nil, xerr.New(xerr.InvalidArgument, "wheel reward worker_id is required")
|
||||||
|
}
|
||||||
|
if limit <= 0 {
|
||||||
|
limit = 50
|
||||||
|
}
|
||||||
|
if maxRetry <= 0 {
|
||||||
|
maxRetry = 16
|
||||||
|
}
|
||||||
|
tx, err := r.db.BeginTx(ctx, nil)
|
||||||
|
if err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
defer func() { _ = tx.Rollback() }()
|
||||||
|
appCode := appcode.FromContext(ctx)
|
||||||
|
rows, err := tx.QueryContext(ctx, `
|
||||||
|
SELECT app_code, outbox_id, payload, status, retry_count, next_retry_at_ms,
|
||||||
|
locked_by, lock_until_ms, last_error, created_at_ms, updated_at_ms
|
||||||
|
FROM activity_outbox
|
||||||
|
WHERE app_code = ?
|
||||||
|
AND event_type = ?
|
||||||
|
AND retry_count < ?
|
||||||
|
AND (
|
||||||
|
(status IN (?, ?) AND next_retry_at_ms <= ?)
|
||||||
|
OR (status = ? AND lock_until_ms <= ?)
|
||||||
|
)
|
||||||
|
ORDER BY created_at_ms, outbox_id
|
||||||
|
LIMIT ?
|
||||||
|
FOR UPDATE SKIP LOCKED`,
|
||||||
|
appCode,
|
||||||
|
domain.EventTypeWheelRewardSettlement,
|
||||||
|
maxRetry,
|
||||||
|
domain.OutboxStatusPending,
|
||||||
|
domain.OutboxStatusRetryable,
|
||||||
|
nowMS,
|
||||||
|
domain.OutboxStatusDelivering,
|
||||||
|
nowMS,
|
||||||
|
limit,
|
||||||
|
)
|
||||||
|
if err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
records := make([]domain.RewardOutboxRecord, 0, limit)
|
||||||
|
for rows.Next() {
|
||||||
|
var record domain.RewardOutboxRecord
|
||||||
|
if err := rows.Scan(
|
||||||
|
&record.AppCode,
|
||||||
|
&record.OutboxID,
|
||||||
|
&record.Payload,
|
||||||
|
&record.Status,
|
||||||
|
&record.RetryCount,
|
||||||
|
&record.NextRetryAtMS,
|
||||||
|
&record.LockedBy,
|
||||||
|
&record.LockUntilMS,
|
||||||
|
&record.LastError,
|
||||||
|
&record.CreatedAtMS,
|
||||||
|
&record.UpdatedAtMS,
|
||||||
|
); err != nil {
|
||||||
|
_ = rows.Close()
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
records = append(records, record)
|
||||||
|
}
|
||||||
|
if err := rows.Close(); err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
if err := rows.Err(); err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
for index := range records {
|
||||||
|
result, err := tx.ExecContext(ctx, `
|
||||||
|
UPDATE activity_outbox
|
||||||
|
SET status = ?, locked_by = ?, lock_until_ms = ?, updated_at_ms = ?
|
||||||
|
WHERE app_code = ? AND outbox_id = ?`,
|
||||||
|
domain.OutboxStatusDelivering,
|
||||||
|
workerID,
|
||||||
|
lockUntilMS,
|
||||||
|
nowMS,
|
||||||
|
records[index].AppCode,
|
||||||
|
records[index].OutboxID,
|
||||||
|
)
|
||||||
|
if err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
if affected, err := result.RowsAffected(); err != nil || affected != 1 {
|
||||||
|
if err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
return nil, xerr.New(xerr.Conflict, "wheel reward outbox claim lost")
|
||||||
|
}
|
||||||
|
records[index].Status = domain.OutboxStatusDelivering
|
||||||
|
records[index].LockedBy = workerID
|
||||||
|
records[index].LockUntilMS = lockUntilMS
|
||||||
|
}
|
||||||
|
if err := tx.Commit(); err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
return records, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// MarkWheelRewardOutboxDelivered 把钱包返回的既有或新建收据与 draw 状态、统计和 outbox 在一个 MySQL 事务中提交。
|
||||||
|
// locked_by 是 fencing token;worker lease 过期并被接管后,旧 worker 即使晚返回也不能覆盖新 owner 的处理结果。
|
||||||
|
func (r *Repository) MarkWheelRewardOutboxDelivered(ctx context.Context, outboxID string, workerID string, drawIDs []string, transactionID string, nowMS int64) error {
|
||||||
|
if r == nil || r.db == nil {
|
||||||
|
return xerr.New(xerr.Unavailable, "mysql repository is not configured")
|
||||||
|
}
|
||||||
|
outboxID = strings.TrimSpace(outboxID)
|
||||||
|
workerID = strings.TrimSpace(workerID)
|
||||||
|
if outboxID == "" || workerID == "" {
|
||||||
|
return xerr.New(xerr.InvalidArgument, "wheel reward outbox delivery identity is incomplete")
|
||||||
|
}
|
||||||
|
tx, err := r.db.BeginTx(ctx, nil)
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
defer func() { _ = tx.Rollback() }()
|
||||||
|
appCode := appcode.FromContext(ctx)
|
||||||
|
if err := markWheelDrawsGrantedTx(ctx, tx, appCode, drawIDs, transactionID, nowMS, true); err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
result, err := tx.ExecContext(ctx, `
|
||||||
|
UPDATE activity_outbox
|
||||||
|
SET status = ?, locked_by = '', lock_until_ms = 0, last_error = '', updated_at_ms = ?
|
||||||
|
WHERE app_code = ? AND outbox_id = ? AND event_type = ? AND status = ? AND locked_by = ?`,
|
||||||
|
domain.OutboxStatusDelivered,
|
||||||
|
nowMS,
|
||||||
|
appCode,
|
||||||
|
outboxID,
|
||||||
|
domain.EventTypeWheelRewardSettlement,
|
||||||
|
domain.OutboxStatusDelivering,
|
||||||
|
workerID,
|
||||||
|
)
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
if affected, err := result.RowsAffected(); err != nil || affected != 1 {
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
return xerr.New(xerr.Conflict, "wheel reward outbox delivery lock lost")
|
||||||
|
}
|
||||||
|
return tx.Commit()
|
||||||
|
}
|
||||||
|
|
||||||
|
// MarkWheelRewardOutboxFailed 释放失败 lease,并以有界重试次数决定 retryable 或 failed。
|
||||||
|
func (r *Repository) MarkWheelRewardOutboxFailed(ctx context.Context, outboxID string, workerID string, nowMS int64, nextRetryAtMS int64, maxRetry int, lastError string) error {
|
||||||
|
if r == nil || r.db == nil {
|
||||||
|
return xerr.New(xerr.Unavailable, "mysql repository is not configured")
|
||||||
|
}
|
||||||
|
if maxRetry <= 0 {
|
||||||
|
maxRetry = 16
|
||||||
|
}
|
||||||
|
result, err := r.db.ExecContext(ctx, `
|
||||||
|
UPDATE activity_outbox
|
||||||
|
SET status = CASE WHEN retry_count + 1 >= ? THEN ? ELSE ? END,
|
||||||
|
next_retry_at_ms = CASE WHEN retry_count + 1 >= ? THEN 0 ELSE ? END,
|
||||||
|
retry_count = retry_count + 1,
|
||||||
|
locked_by = '',
|
||||||
|
lock_until_ms = 0,
|
||||||
|
last_error = ?,
|
||||||
|
updated_at_ms = ?
|
||||||
|
WHERE app_code = ? AND outbox_id = ? AND event_type = ? AND status = ? AND locked_by = ?`,
|
||||||
|
maxRetry,
|
||||||
|
domain.OutboxStatusFailed,
|
||||||
|
domain.OutboxStatusRetryable,
|
||||||
|
maxRetry,
|
||||||
|
nextRetryAtMS,
|
||||||
|
truncateWheelOutboxError(lastError),
|
||||||
|
nowMS,
|
||||||
|
appcode.FromContext(ctx),
|
||||||
|
strings.TrimSpace(outboxID),
|
||||||
|
domain.EventTypeWheelRewardSettlement,
|
||||||
|
domain.OutboxStatusDelivering,
|
||||||
|
strings.TrimSpace(workerID),
|
||||||
|
)
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
if affected, err := result.RowsAffected(); err != nil || affected != 1 {
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
return xerr.New(xerr.Conflict, "wheel reward outbox failure lock lost")
|
||||||
|
}
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func truncateWheelOutboxError(message string) string {
|
||||||
|
message = strings.TrimSpace(message)
|
||||||
|
if len(message) <= 512 {
|
||||||
|
return message
|
||||||
|
}
|
||||||
|
return message[:512]
|
||||||
|
}
|
||||||
@ -0,0 +1,79 @@
|
|||||||
|
package mysql
|
||||||
|
|
||||||
|
import (
|
||||||
|
"context"
|
||||||
|
"testing"
|
||||||
|
|
||||||
|
"hyapp/internal/testutil/mysqlschema"
|
||||||
|
"hyapp/pkg/appcode"
|
||||||
|
domain "hyapp/services/activity-service/internal/domain/wheel"
|
||||||
|
)
|
||||||
|
|
||||||
|
func TestWheelRewardOutboxClaimAndDeliveryAreLeaseFencedAndAtomic(t *testing.T) {
|
||||||
|
schema := mysqlschema.New(t, mysqlschema.Config{
|
||||||
|
EnvVar: "ACTIVITY_SERVICE_MYSQL_TEST_DSN",
|
||||||
|
InitDBPath: mysqlschema.InitDBPath(t, mysqlschema.CallerFile(t, 1), "..", "..", "..", "deploy", "mysql", "initdb", "001_activity_service.sql"),
|
||||||
|
DatabasePrefix: "hyapp_activity_wheel_outbox_test",
|
||||||
|
})
|
||||||
|
repository, err := Open(context.Background(), schema.DSN)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("open repository: %v", err)
|
||||||
|
}
|
||||||
|
t.Cleanup(func() { _ = repository.Close() })
|
||||||
|
ctx := appcode.WithContext(context.Background(), "fami")
|
||||||
|
const (
|
||||||
|
drawID = "wheel_draw_outbox_test"
|
||||||
|
outboxID = "wheel_reward_wheel_draw_outbox_test"
|
||||||
|
)
|
||||||
|
if _, err := schema.DB.ExecContext(ctx, `
|
||||||
|
INSERT INTO wheel_draw_stats (
|
||||||
|
app_code, wheel_id, total_draws, unique_users, total_spent_coins, total_rtp_value_coins,
|
||||||
|
pending_draws, granted_draws, failed_draws, created_at_ms, updated_at_ms
|
||||||
|
) VALUES ('fami', 'classic', 1, 1, 100, 20, 1, 0, 0, 1000, 1000)`); err != nil {
|
||||||
|
t.Fatalf("seed wheel stats: %v", err)
|
||||||
|
}
|
||||||
|
if _, err := schema.DB.ExecContext(ctx, `
|
||||||
|
INSERT INTO wheel_draw_records (
|
||||||
|
app_code, draw_id, command_id, user_id, device_id, visible_region_id, wheel_id, coin_spent,
|
||||||
|
rule_version, rtp_window_index, selected_tier_id, reward_type, reward_id, reward_count,
|
||||||
|
reward_coins, rtp_value_coins, candidate_tiers_json, rtp_snapshot_json, prize_metadata_json,
|
||||||
|
reward_status, paid_at_ms, created_at_ms, updated_at_ms
|
||||||
|
) VALUES ('fami', ?, 'draw-command', 1001, 'device', 9, 'classic', 100,
|
||||||
|
1, 1, 'tier-1', 'coin', '', 1, 20, 20, '{}', '{}', '{}', 'pending', 1000, 1000, 1000)`, drawID); err != nil {
|
||||||
|
t.Fatalf("seed wheel draw: %v", err)
|
||||||
|
}
|
||||||
|
if _, err := schema.DB.ExecContext(ctx, `
|
||||||
|
INSERT INTO activity_outbox (
|
||||||
|
app_code, outbox_id, event_type, payload, status, retry_count, next_retry_at_ms,
|
||||||
|
locked_by, lock_until_ms, last_error, created_at_ms, updated_at_ms
|
||||||
|
) VALUES ('fami', ?, ?, '{}', 'pending', 0, 1000, '', 0, '', 1000, 1000)`, outboxID, domain.EventTypeWheelRewardSettlement); err != nil {
|
||||||
|
t.Fatalf("seed wheel outbox: %v", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
records, err := repository.ClaimWheelRewardOutbox(ctx, "worker-1", 1000, 31000, 10, 16)
|
||||||
|
if err != nil || len(records) != 1 || records[0].LockedBy != "worker-1" || records[0].Status != domain.OutboxStatusDelivering {
|
||||||
|
t.Fatalf("claim mismatch: records=%+v err=%v", records, err)
|
||||||
|
}
|
||||||
|
secondClaim, err := repository.ClaimWheelRewardOutbox(ctx, "worker-2", 2000, 32000, 10, 16)
|
||||||
|
if err != nil || len(secondClaim) != 0 {
|
||||||
|
t.Fatalf("active lease must block second worker: records=%+v err=%v", secondClaim, err)
|
||||||
|
}
|
||||||
|
if err := repository.MarkWheelRewardOutboxDelivered(ctx, outboxID, "worker-1", []string{drawID}, "wtx-existing", 3000); err != nil {
|
||||||
|
t.Fatalf("mark delivered: %v", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
var drawStatus, receiptRef, outboxStatus, lockedBy string
|
||||||
|
var pending, granted int64
|
||||||
|
if err := schema.DB.QueryRowContext(ctx, `SELECT reward_status, reward_transaction_id FROM wheel_draw_records WHERE app_code = 'fami' AND draw_id = ?`, drawID).Scan(&drawStatus, &receiptRef); err != nil {
|
||||||
|
t.Fatalf("read draw state: %v", err)
|
||||||
|
}
|
||||||
|
if err := schema.DB.QueryRowContext(ctx, `SELECT pending_draws, granted_draws FROM wheel_draw_stats WHERE app_code = 'fami' AND wheel_id = 'classic'`).Scan(&pending, &granted); err != nil {
|
||||||
|
t.Fatalf("read wheel stats: %v", err)
|
||||||
|
}
|
||||||
|
if err := schema.DB.QueryRowContext(ctx, `SELECT status, locked_by FROM activity_outbox WHERE app_code = 'fami' AND outbox_id = ?`, outboxID).Scan(&outboxStatus, &lockedBy); err != nil {
|
||||||
|
t.Fatalf("read outbox state: %v", err)
|
||||||
|
}
|
||||||
|
if drawStatus != domain.StatusGranted || receiptRef != "wtx-existing" || pending != 0 || granted != 1 || outboxStatus != domain.OutboxStatusDelivered || lockedBy != "" {
|
||||||
|
t.Fatalf("delivery state mismatch: draw=%s receipt=%s pending=%d granted=%d outbox=%s lock=%q", drawStatus, receiptRef, pending, granted, outboxStatus, lockedBy)
|
||||||
|
}
|
||||||
|
}
|
||||||
@ -317,11 +317,38 @@ func (r *Repository) MarkWheelDrawsGranted(ctx context.Context, appCode string,
|
|||||||
return err
|
return err
|
||||||
}
|
}
|
||||||
defer func() { _ = tx.Rollback() }()
|
defer func() { _ = tx.Rollback() }()
|
||||||
|
if err := markWheelDrawsGrantedTx(ctx, tx, appCode, drawIDs, transactionID, nowMS, false); err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
for _, drawID := range drawIDs {
|
for _, drawID := range drawIDs {
|
||||||
drawID = strings.TrimSpace(drawID)
|
drawID = strings.TrimSpace(drawID)
|
||||||
if drawID == "" {
|
if drawID == "" {
|
||||||
continue
|
continue
|
||||||
}
|
}
|
||||||
|
if _, err := tx.ExecContext(ctx, `
|
||||||
|
UPDATE activity_outbox
|
||||||
|
SET status = 'delivered', locked_by = '', lock_until_ms = 0, last_error = '', updated_at_ms = ?
|
||||||
|
WHERE app_code = ? AND outbox_id = ?`,
|
||||||
|
nowMS, appCode, "wheel_reward_"+drawID,
|
||||||
|
); err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return tx.Commit()
|
||||||
|
}
|
||||||
|
|
||||||
|
// markWheelDrawsGrantedTx 原子搬移 draw 和汇总状态;requireDraw 用于补偿 worker 防止损坏 payload 把 outbox 错标 delivered。
|
||||||
|
func markWheelDrawsGrantedTx(ctx context.Context, tx *sql.Tx, appCode string, drawIDs []string, transactionID string, nowMS int64, requireDraw bool) error {
|
||||||
|
seen := make(map[string]struct{}, len(drawIDs))
|
||||||
|
for _, drawID := range drawIDs {
|
||||||
|
drawID = strings.TrimSpace(drawID)
|
||||||
|
if drawID == "" {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
if _, exists := seen[drawID]; exists {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
seen[drawID] = struct{}{}
|
||||||
var wheelID, oldStatus string
|
var wheelID, oldStatus string
|
||||||
if err := tx.QueryRowContext(ctx, `
|
if err := tx.QueryRowContext(ctx, `
|
||||||
SELECT wheel_id, reward_status
|
SELECT wheel_id, reward_status
|
||||||
@ -331,6 +358,9 @@ func (r *Repository) MarkWheelDrawsGranted(ctx context.Context, appCode string,
|
|||||||
appCode, drawID,
|
appCode, drawID,
|
||||||
).Scan(&wheelID, &oldStatus); err != nil {
|
).Scan(&wheelID, &oldStatus); err != nil {
|
||||||
if errors.Is(err, sql.ErrNoRows) {
|
if errors.Is(err, sql.ErrNoRows) {
|
||||||
|
if requireDraw {
|
||||||
|
return xerr.New(xerr.NotFound, "wheel draw record not found")
|
||||||
|
}
|
||||||
continue
|
continue
|
||||||
}
|
}
|
||||||
return err
|
return err
|
||||||
@ -356,16 +386,11 @@ func (r *Repository) MarkWheelDrawsGranted(ctx context.Context, appCode string,
|
|||||||
return err
|
return err
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
if _, err := tx.ExecContext(ctx, `
|
|
||||||
UPDATE activity_outbox
|
|
||||||
SET status = 'delivered', locked_by = '', lock_until_ms = 0, last_error = '', updated_at_ms = ?
|
|
||||||
WHERE app_code = ? AND outbox_id = ?`,
|
|
||||||
nowMS, appCode, "wheel_reward_"+drawID,
|
|
||||||
); err != nil {
|
|
||||||
return err
|
|
||||||
}
|
|
||||||
}
|
}
|
||||||
return tx.Commit()
|
if requireDraw && len(seen) == 0 {
|
||||||
|
return xerr.New(xerr.InvalidArgument, "wheel draw_ids are empty")
|
||||||
|
}
|
||||||
|
return nil
|
||||||
}
|
}
|
||||||
|
|
||||||
func (r *Repository) executeSingleWheelDraw(cmd domain.DrawCommand, commandID string, unitSpent int64, config domain.RuleConfig, window *wheelRTPWindow, pool *wheelPool, nowMS int64) (domain.DrawResult, int64, error) {
|
func (r *Repository) executeSingleWheelDraw(cmd domain.DrawCommand, commandID string, unitSpent int64, config domain.RuleConfig, window *wheelRTPWindow, pool *wheelPool, nowMS int64) (domain.DrawResult, int64, error) {
|
||||||
|
|||||||
@ -34,6 +34,7 @@ rocketmq:
|
|||||||
producer_group: "hyapp-game-outbox-producer"
|
producer_group: "hyapp-game-outbox-producer"
|
||||||
outbox_worker:
|
outbox_worker:
|
||||||
enabled: true
|
enabled: true
|
||||||
|
app_codes: []
|
||||||
poll_interval: "1s"
|
poll_interval: "1s"
|
||||||
batch_size: 100
|
batch_size: 100
|
||||||
publish_timeout: "3s"
|
publish_timeout: "3s"
|
||||||
|
|||||||
@ -34,6 +34,8 @@ rocketmq:
|
|||||||
producer_group: "hyapp-game-outbox-producer"
|
producer_group: "hyapp-game-outbox-producer"
|
||||||
outbox_worker:
|
outbox_worker:
|
||||||
enabled: true
|
enabled: true
|
||||||
|
# 空值自动发现全部 App;历史积压回放时先设 ["lalu", "fami"],核对后再加入 "huwaa"。
|
||||||
|
app_codes: []
|
||||||
poll_interval: "60s"
|
poll_interval: "60s"
|
||||||
batch_size: 10
|
batch_size: 10
|
||||||
publish_timeout: "3s"
|
publish_timeout: "3s"
|
||||||
|
|||||||
@ -34,6 +34,7 @@ rocketmq:
|
|||||||
producer_group: "hyapp-game-outbox-producer"
|
producer_group: "hyapp-game-outbox-producer"
|
||||||
outbox_worker:
|
outbox_worker:
|
||||||
enabled: true
|
enabled: true
|
||||||
|
app_codes: []
|
||||||
poll_interval: "1s"
|
poll_interval: "1s"
|
||||||
batch_size: 100
|
batch_size: 100
|
||||||
publish_timeout: "3s"
|
publish_timeout: "3s"
|
||||||
|
|||||||
@ -11,10 +11,12 @@ import (
|
|||||||
"google.golang.org/grpc"
|
"google.golang.org/grpc"
|
||||||
"google.golang.org/grpc/credentials/insecure"
|
"google.golang.org/grpc/credentials/insecure"
|
||||||
gamev1 "hyapp.local/api/proto/game/v1"
|
gamev1 "hyapp.local/api/proto/game/v1"
|
||||||
|
"hyapp/pkg/appcode"
|
||||||
"hyapp/pkg/gamemq"
|
"hyapp/pkg/gamemq"
|
||||||
"hyapp/pkg/grpchealth"
|
"hyapp/pkg/grpchealth"
|
||||||
"hyapp/pkg/healthhttp"
|
"hyapp/pkg/healthhttp"
|
||||||
"hyapp/pkg/logx"
|
"hyapp/pkg/logx"
|
||||||
|
"hyapp/pkg/outboxpartition"
|
||||||
"hyapp/pkg/rocketmqx"
|
"hyapp/pkg/rocketmqx"
|
||||||
servicegrpc "hyapp/pkg/servicekit/grpcserver"
|
servicegrpc "hyapp/pkg/servicekit/grpcserver"
|
||||||
servicehealth "hyapp/pkg/servicekit/health"
|
servicehealth "hyapp/pkg/servicekit/health"
|
||||||
@ -40,11 +42,13 @@ type App struct {
|
|||||||
activityConn *grpc.ClientConn
|
activityConn *grpc.ClientConn
|
||||||
robotConn *grpc.ClientConn
|
robotConn *grpc.ClientConn
|
||||||
outboxProducer *rocketmqx.Producer
|
outboxProducer *rocketmqx.Producer
|
||||||
outboxCtx context.Context
|
// outboxPartitions 缓存真实 App 列表并让双实例从不同轮转起点公平 claim。
|
||||||
outboxCancel context.CancelFunc
|
outboxPartitions *outboxpartition.Partitions
|
||||||
outboxWG sync.WaitGroup
|
outboxCtx context.Context
|
||||||
cfg config.Config
|
outboxCancel context.CancelFunc
|
||||||
closeOnce sync.Once
|
outboxWG sync.WaitGroup
|
||||||
|
cfg config.Config
|
||||||
|
closeOnce sync.Once
|
||||||
}
|
}
|
||||||
|
|
||||||
func New(cfg config.Config) (*App, error) {
|
func New(cfg config.Config) (*App, error) {
|
||||||
@ -154,7 +158,7 @@ func New(cfg config.Config) (*App, error) {
|
|||||||
}
|
}
|
||||||
|
|
||||||
outboxCtx, outboxCancel := context.WithCancel(context.Background())
|
outboxCtx, outboxCancel := context.WithCancel(context.Background())
|
||||||
return &App{server: server, listener: listener, health: health, healthHTTP: healthHTTP, repo: repo, walletConn: walletConn, userConn: userConn, activityConn: activityConn, robotConn: robotConn, outboxProducer: outboxProducer, outboxCtx: outboxCtx, outboxCancel: outboxCancel, cfg: cfg}, nil
|
return &App{server: server, listener: listener, health: health, healthHTTP: healthHTTP, repo: repo, walletConn: walletConn, userConn: userConn, activityConn: activityConn, robotConn: robotConn, outboxProducer: outboxProducer, outboxPartitions: outboxpartition.New(cfg.OutboxWorker.AppCodes, outboxpartition.DefaultDiscoveryRefreshInterval), outboxCtx: outboxCtx, outboxCancel: outboxCancel, cfg: cfg}, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
func (a *App) Run() error {
|
func (a *App) Run() error {
|
||||||
@ -229,7 +233,17 @@ func (a *App) runOutboxWorker() {
|
|||||||
|
|
||||||
func (a *App) processOutboxBatch() (int, error) {
|
func (a *App) processOutboxBatch() (int, error) {
|
||||||
workerID := "game-outbox-" + a.cfg.NodeID
|
workerID := "game-outbox-" + a.cfg.NodeID
|
||||||
records, err := a.repo.ClaimPendingGameOutbox(a.outboxCtx, workerID, a.cfg.OutboxWorker.BatchSize)
|
appCodes, discoveryErr := a.outboxPartitions.Resolve(a.outboxCtx, a.repo.ListGameOutboxAppCodes)
|
||||||
|
if discoveryErr != nil {
|
||||||
|
if len(appCodes) == 0 {
|
||||||
|
return 0, discoveryErr
|
||||||
|
}
|
||||||
|
// 已缓存 App 继续发布;低频刷新失败不应把整个 game outbox worker 停掉。
|
||||||
|
logx.Warn(a.outboxCtx, "game_outbox_app_discovery_stale", slog.String("worker_id", workerID), slog.String("error", discoveryErr.Error()))
|
||||||
|
}
|
||||||
|
records, err := outboxpartition.ClaimFair(a.outboxCtx, a.outboxPartitions.Order(appCodes), a.cfg.OutboxWorker.BatchSize, func(claimCtx context.Context, appCode string, limit int) ([]mysqlstorage.GameOutboxRecord, error) {
|
||||||
|
return a.repo.ClaimPendingGameOutbox(appcode.WithContext(claimCtx, appCode), workerID, limit)
|
||||||
|
})
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return 0, err
|
return 0, err
|
||||||
}
|
}
|
||||||
|
|||||||
@ -9,6 +9,7 @@ import (
|
|||||||
|
|
||||||
"hyapp/pkg/configx"
|
"hyapp/pkg/configx"
|
||||||
"hyapp/pkg/logx"
|
"hyapp/pkg/logx"
|
||||||
|
"hyapp/pkg/outboxpartition"
|
||||||
"hyapp/pkg/tencentim"
|
"hyapp/pkg/tencentim"
|
||||||
)
|
)
|
||||||
|
|
||||||
@ -95,7 +96,9 @@ type GameOutboxConfig struct {
|
|||||||
}
|
}
|
||||||
|
|
||||||
type OutboxWorkerConfig struct {
|
type OutboxWorkerConfig struct {
|
||||||
Enabled bool `yaml:"enabled"`
|
Enabled bool `yaml:"enabled"`
|
||||||
|
// AppCodes 是可投递租户 allowlist;空值自动发现,非空用于分阶段回放历史积压。
|
||||||
|
AppCodes []string `yaml:"app_codes"`
|
||||||
PollInterval time.Duration `yaml:"poll_interval"`
|
PollInterval time.Duration `yaml:"poll_interval"`
|
||||||
BatchSize int `yaml:"batch_size"`
|
BatchSize int `yaml:"batch_size"`
|
||||||
PublishTimeout time.Duration `yaml:"publish_timeout"`
|
PublishTimeout time.Duration `yaml:"publish_timeout"`
|
||||||
@ -210,6 +213,7 @@ func (cfg *Config) Normalize() error {
|
|||||||
if cfg.OutboxWorker.PublishTimeout <= 0 {
|
if cfg.OutboxWorker.PublishTimeout <= 0 {
|
||||||
cfg.OutboxWorker.PublishTimeout = 3 * time.Second
|
cfg.OutboxWorker.PublishTimeout = 3 * time.Second
|
||||||
}
|
}
|
||||||
|
cfg.OutboxWorker.AppCodes = outboxpartition.Normalize(cfg.OutboxWorker.AppCodes, false)
|
||||||
if cfg.OutboxWorker.Enabled && !cfg.RocketMQ.GameOutbox.Enabled {
|
if cfg.OutboxWorker.Enabled && !cfg.RocketMQ.GameOutbox.Enabled {
|
||||||
return errors.New("outbox_worker requires rocketmq.game_outbox.enabled")
|
return errors.New("outbox_worker requires rocketmq.game_outbox.enabled")
|
||||||
}
|
}
|
||||||
|
|||||||
@ -2,6 +2,27 @@ package config
|
|||||||
|
|
||||||
import "testing"
|
import "testing"
|
||||||
|
|
||||||
|
func TestOutboxWorkerAppCodesNormalizeForStagedReplay(t *testing.T) {
|
||||||
|
cfg := Default()
|
||||||
|
cfg.OutboxWorker.AppCodes = []string{" LALU ", "fami", "HUWAA", "fami"}
|
||||||
|
if err := cfg.Normalize(); err != nil {
|
||||||
|
t.Fatalf("normalize staged outbox config: %v", err)
|
||||||
|
}
|
||||||
|
if len(cfg.OutboxWorker.AppCodes) != 3 || cfg.OutboxWorker.AppCodes[0] != "lalu" || cfg.OutboxWorker.AppCodes[1] != "fami" || cfg.OutboxWorker.AppCodes[2] != "huwaa" {
|
||||||
|
t.Fatalf("normalized outbox app_codes = %v", cfg.OutboxWorker.AppCodes)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestLocalOutboxWorkerEmptyAppCodesUsesDiscovery(t *testing.T) {
|
||||||
|
cfg, err := Load("../../configs/config.yaml")
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("load local config: %v", err)
|
||||||
|
}
|
||||||
|
if len(cfg.OutboxWorker.AppCodes) != 0 {
|
||||||
|
t.Fatalf("local empty app_codes must retain automatic discovery: %v", cfg.OutboxWorker.AppCodes)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
func TestLoadConfigsKeepActivityServiceAddr(t *testing.T) {
|
func TestLoadConfigsKeepActivityServiceAddr(t *testing.T) {
|
||||||
tests := []struct {
|
tests := []struct {
|
||||||
name string
|
name string
|
||||||
|
|||||||
@ -0,0 +1,30 @@
|
|||||||
|
package mysql
|
||||||
|
|
||||||
|
import (
|
||||||
|
"context"
|
||||||
|
"regexp"
|
||||||
|
"testing"
|
||||||
|
|
||||||
|
"github.com/DATA-DOG/go-sqlmock"
|
||||||
|
)
|
||||||
|
|
||||||
|
func TestListGameOutboxAppCodesForcesPrimaryPrefixIndex(t *testing.T) {
|
||||||
|
db, mock, err := sqlmock.New()
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("create sqlmock: %v", err)
|
||||||
|
}
|
||||||
|
defer db.Close()
|
||||||
|
mock.ExpectQuery(regexp.QuoteMeta(`SELECT DISTINCT app_code FROM game_outbox FORCE INDEX (PRIMARY) ORDER BY app_code`)).
|
||||||
|
WillReturnRows(sqlmock.NewRows([]string{"app_code"}).AddRow("fami").AddRow("huwaa").AddRow("lalu"))
|
||||||
|
|
||||||
|
apps, err := (&Repository{db: db}).ListGameOutboxAppCodes(context.Background())
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("list game outbox apps: %v", err)
|
||||||
|
}
|
||||||
|
if len(apps) != 3 || apps[0] != "fami" || apps[1] != "huwaa" || apps[2] != "lalu" {
|
||||||
|
t.Fatalf("game outbox apps mismatch: %v", apps)
|
||||||
|
}
|
||||||
|
if err := mock.ExpectationsWereMet(); err != nil {
|
||||||
|
t.Fatalf("game app discovery query mismatch: %v", err)
|
||||||
|
}
|
||||||
|
}
|
||||||
@ -1152,6 +1152,29 @@ type GameOutboxRecord struct {
|
|||||||
LockUntilMS int64
|
LockUntilMS int64
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// ListGameOutboxAppCodes enumerates real tenant keys from the game owner table.
|
||||||
|
// PRIMARY is ordered by (app_code,event_id), allowing cached discovery without
|
||||||
|
// weakening the app_code prefix used by every SKIP LOCKED claim transaction.
|
||||||
|
func (r *Repository) ListGameOutboxAppCodes(ctx context.Context) ([]string, error) {
|
||||||
|
if r == nil || r.db == nil {
|
||||||
|
return nil, xerr.New(xerr.Unavailable, "mysql repository is not configured")
|
||||||
|
}
|
||||||
|
rows, err := r.db.QueryContext(ctx, `SELECT DISTINCT app_code FROM game_outbox FORCE INDEX (PRIMARY) ORDER BY app_code`)
|
||||||
|
if err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
defer rows.Close()
|
||||||
|
appCodes := make([]string, 0, 4)
|
||||||
|
for rows.Next() {
|
||||||
|
var appCode string
|
||||||
|
if err := rows.Scan(&appCode); err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
appCodes = append(appCodes, appCode)
|
||||||
|
}
|
||||||
|
return appCodes, rows.Err()
|
||||||
|
}
|
||||||
|
|
||||||
// ClaimPendingGameOutbox locks pending game facts so one worker publishes them.
|
// ClaimPendingGameOutbox locks pending game facts so one worker publishes them.
|
||||||
func (r *Repository) ClaimPendingGameOutbox(ctx context.Context, workerID string, batchSize int) ([]GameOutboxRecord, error) {
|
func (r *Repository) ClaimPendingGameOutbox(ctx context.Context, workerID string, batchSize int) ([]GameOutboxRecord, error) {
|
||||||
if r == nil || r.db == nil {
|
if r == nil || r.db == nil {
|
||||||
@ -1356,23 +1379,34 @@ func (r *Repository) queryUnlockedLevelEventClaimKeys(ctx context.Context, tx *s
|
|||||||
}
|
}
|
||||||
|
|
||||||
func (r *Repository) queryExpiredLevelEventClaimKeys(ctx context.Context, tx *sql.Tx, appCode string, nowMS int64, batchSize int) ([]levelEventClaimKey, error) {
|
func (r *Repository) queryExpiredLevelEventClaimKeys(ctx context.Context, tx *sql.Tx, appCode string, nowMS int64, batchSize int) ([]levelEventClaimKey, error) {
|
||||||
// 过期锁接管仍使用 retry/lock 前缀缩小范围,但只锁主键,避免大 JSON 字段进入排序和慢日志。
|
// 过期接管只处理 running:pending/failed 已由 unlocked path 负责。拆开 legacy lock=0
|
||||||
return queryLevelEventClaimKeysByStatus(batchSize, func(status string) ([]levelEventClaimKey, error) {
|
// 与正常过期锁可继续利用 (app_code,status,locked_until_ms,...) 前缀,同时明确
|
||||||
|
// 排除未来锁,避免并发 cron 抢走仍由其它 worker 处理的事实。
|
||||||
|
query := func(lockCondition string, args ...any) ([]levelEventClaimKey, error) {
|
||||||
|
queryArgs := []any{appCode, "running"}
|
||||||
|
queryArgs = append(queryArgs, args...)
|
||||||
|
queryArgs = append(queryArgs, nowMS, batchSize)
|
||||||
rows, err := tx.QueryContext(ctx, `
|
rows, err := tx.QueryContext(ctx, `
|
||||||
SELECT event_id, created_at_ms
|
SELECT event_id, created_at_ms
|
||||||
FROM game_level_event_outbox FORCE INDEX (idx_game_level_event_claim)
|
FROM game_level_event_outbox FORCE INDEX (idx_game_level_event_claim)
|
||||||
WHERE app_code = ? AND status = ? AND next_retry_at_ms <= ?
|
WHERE app_code = ? AND status = ? AND `+lockCondition+` AND next_retry_at_ms <= ?
|
||||||
AND locked_until_ms > 0 AND locked_until_ms <= ?
|
|
||||||
ORDER BY created_at_ms ASC, event_id ASC
|
ORDER BY created_at_ms ASC, event_id ASC
|
||||||
LIMIT ? FOR UPDATE SKIP LOCKED`,
|
LIMIT ? FOR UPDATE SKIP LOCKED`, queryArgs...)
|
||||||
appCode, status, nowMS, nowMS, batchSize,
|
|
||||||
)
|
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return nil, err
|
return nil, err
|
||||||
}
|
}
|
||||||
defer rows.Close()
|
defer rows.Close()
|
||||||
return scanLevelEventClaimKeys(rows)
|
return scanLevelEventClaimKeys(rows)
|
||||||
})
|
}
|
||||||
|
legacyUnlocked, err := query("locked_until_ms = 0")
|
||||||
|
if err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
expired, err := query("locked_until_ms > 0 AND locked_until_ms <= ?", nowMS)
|
||||||
|
if err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
return mergeLevelEventClaimKeys(batchSize, legacyUnlocked, expired), nil
|
||||||
}
|
}
|
||||||
|
|
||||||
func queryLevelEventClaimKeysByStatus(batchSize int, query func(status string) ([]levelEventClaimKey, error)) ([]levelEventClaimKey, error) {
|
func queryLevelEventClaimKeysByStatus(batchSize int, query func(status string) ([]levelEventClaimKey, error)) ([]levelEventClaimKey, error) {
|
||||||
|
|||||||
@ -483,6 +483,65 @@ func TestClaimPendingLevelEventsPreservesCreatedOrderAcrossStatuses(t *testing.T
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func TestClaimPendingLevelEventsReclaimsLegacyAndExpiredRunningLocks(t *testing.T) {
|
||||||
|
caller := mysqlschema.CallerFile(t, 1)
|
||||||
|
schema := mysqlschema.New(t, mysqlschema.Config{
|
||||||
|
InitDBPath: mysqlschema.InitDBPath(t, caller, "../../../deploy/mysql/initdb/001_game_service.sql"),
|
||||||
|
DatabasePrefix: "hy_game_test",
|
||||||
|
})
|
||||||
|
|
||||||
|
ctx := appcode.WithContext(context.Background(), "lalu")
|
||||||
|
repo, err := Open(ctx, schema.DSN)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("open repository failed: %v", err)
|
||||||
|
}
|
||||||
|
t.Cleanup(func() { _ = repo.Close() })
|
||||||
|
|
||||||
|
insertRunning := func(eventID string, lockedUntilMS int64, createdAtMS int64) {
|
||||||
|
t.Helper()
|
||||||
|
if _, err := repo.db.ExecContext(ctx, `
|
||||||
|
INSERT INTO game_level_event_outbox (
|
||||||
|
app_code, event_id, order_id, user_id, game_id, provider_order_id,
|
||||||
|
provider_round_id, coin_amount, wallet_transaction_id, payload_json,
|
||||||
|
status, attempt_count, next_retry_at_ms, locked_by, locked_until_ms,
|
||||||
|
failure_reason, occurred_at_ms, created_at_ms, updated_at_ms
|
||||||
|
) VALUES ('lalu', ?, ?, 42, 'game_1', ?, '', 100, 'wallet_tx_1', '{}',
|
||||||
|
'running', 1, 0, 'old-worker', ?, '', ?, ?, ?)`,
|
||||||
|
eventID, "order_"+eventID, "provider_"+eventID, lockedUntilMS, createdAtMS, createdAtMS, createdAtMS,
|
||||||
|
); err != nil {
|
||||||
|
t.Fatalf("insert running level event %s failed: %v", eventID, err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
insertRunning("legacy_zero_lock", 0, 1000)
|
||||||
|
insertRunning("expired_lock", 2500, 1100)
|
||||||
|
insertRunning("future_lock", 4000, 1200)
|
||||||
|
|
||||||
|
events, err := repo.ClaimPendingLevelEvents(ctx, "replacement-worker", 3000, 30*time.Second, 10)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("claim expired running events failed: %v", err)
|
||||||
|
}
|
||||||
|
if len(events) != 2 || events[0].EventID != "legacy_zero_lock" || events[1].EventID != "expired_lock" {
|
||||||
|
t.Fatalf("reclaimed running events mismatch: %+v", events)
|
||||||
|
}
|
||||||
|
for _, event := range events {
|
||||||
|
if event.Status != "running" {
|
||||||
|
t.Fatalf("reclaimed event must expose its pre-claim running state: %+v", event)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
var lockedBy string
|
||||||
|
var lockedUntilMS int64
|
||||||
|
if err := repo.db.QueryRowContext(ctx, `
|
||||||
|
SELECT locked_by, locked_until_ms FROM game_level_event_outbox
|
||||||
|
WHERE app_code = 'lalu' AND event_id = 'future_lock'`,
|
||||||
|
).Scan(&lockedBy, &lockedUntilMS); err != nil {
|
||||||
|
t.Fatalf("query future running lock failed: %v", err)
|
||||||
|
}
|
||||||
|
if lockedBy != "old-worker" || lockedUntilMS != 4000 {
|
||||||
|
t.Fatalf("future lock was stolen: locked_by=%q locked_until_ms=%d", lockedBy, lockedUntilMS)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
func TestLevelEventClaimKeyQueryUsesOrderIndexWithoutFilesort(t *testing.T) {
|
func TestLevelEventClaimKeyQueryUsesOrderIndexWithoutFilesort(t *testing.T) {
|
||||||
caller := mysqlschema.CallerFile(t, 1)
|
caller := mysqlschema.CallerFile(t, 1)
|
||||||
schema := mysqlschema.New(t, mysqlschema.Config{
|
schema := mysqlschema.New(t, mysqlschema.Config{
|
||||||
|
|||||||
@ -1,7 +1,6 @@
|
|||||||
package appapi
|
package appapi
|
||||||
|
|
||||||
import (
|
import (
|
||||||
"bytes"
|
|
||||||
"encoding/json"
|
"encoding/json"
|
||||||
"net/http"
|
"net/http"
|
||||||
"strings"
|
"strings"
|
||||||
@ -9,16 +8,12 @@ import (
|
|||||||
|
|
||||||
userv1 "hyapp.local/api/proto/user/v1"
|
userv1 "hyapp.local/api/proto/user/v1"
|
||||||
"hyapp/pkg/appcode"
|
"hyapp/pkg/appcode"
|
||||||
|
"hyapp/pkg/apptracking"
|
||||||
"hyapp/services/gateway-service/internal/auth"
|
"hyapp/services/gateway-service/internal/auth"
|
||||||
"hyapp/services/gateway-service/internal/client"
|
"hyapp/services/gateway-service/internal/client"
|
||||||
"hyapp/services/gateway-service/internal/transport/http/httpkit"
|
"hyapp/services/gateway-service/internal/transport/http/httpkit"
|
||||||
)
|
)
|
||||||
|
|
||||||
const (
|
|
||||||
appEventMaxBatchSize = 50
|
|
||||||
appEventMaxPropertiesBytes = 4096
|
|
||||||
)
|
|
||||||
|
|
||||||
type appEventsRequest struct {
|
type appEventsRequest struct {
|
||||||
Events []appEventRequest `json:"events"`
|
Events []appEventRequest `json:"events"`
|
||||||
}
|
}
|
||||||
@ -44,26 +39,17 @@ type appEventRequest struct {
|
|||||||
}
|
}
|
||||||
|
|
||||||
func (h *Handler) reportAppEvents(writer http.ResponseWriter, request *http.Request) {
|
func (h *Handler) reportAppEvents(writer http.ResponseWriter, request *http.Request) {
|
||||||
if h.statisticsClient == nil {
|
|
||||||
// App 埋点事实由 statistics-service 持久化;gateway 未注入写入边界时不能假成功,否则客户端会丢掉重试机会。
|
|
||||||
httpkit.WriteError(writer, request, http.StatusBadGateway, httpkit.CodeUpstreamError, "upstream service error")
|
|
||||||
return
|
|
||||||
}
|
|
||||||
var body appEventsRequest
|
var body appEventsRequest
|
||||||
if !httpkit.Decode(writer, request, &body) {
|
if !httpkit.Decode(writer, request, &body) {
|
||||||
return
|
return
|
||||||
}
|
}
|
||||||
if len(body.Events) == 0 || len(body.Events) > appEventMaxBatchSize {
|
if !apptracking.ValidBatchSize(len(body.Events)) {
|
||||||
httpkit.WriteError(writer, request, http.StatusBadRequest, httpkit.CodeInvalidArgument, "invalid argument")
|
httpkit.WriteError(writer, request, http.StatusBadRequest, httpkit.CodeInvalidArgument, "invalid argument")
|
||||||
return
|
return
|
||||||
}
|
}
|
||||||
|
|
||||||
nowMS := time.Now().UTC().UnixMilli()
|
nowMS := time.Now().UTC().UnixMilli()
|
||||||
userID := auth.UserIDFromContext(request.Context())
|
userID := auth.UserIDFromContext(request.Context())
|
||||||
countryID, regionID, ok := h.appEventUserDimensions(writer, request, userID)
|
|
||||||
if !ok {
|
|
||||||
return
|
|
||||||
}
|
|
||||||
headerDeviceID := httpkit.FirstHeader(request, "X-Device-ID", "X-HY-Device-ID")
|
headerDeviceID := httpkit.FirstHeader(request, "X-Device-ID", "X-HY-Device-ID")
|
||||||
headerPlatform := httpkit.FirstHeader(request, "X-App-Platform", "X-Platform")
|
headerPlatform := httpkit.FirstHeader(request, "X-App-Platform", "X-Platform")
|
||||||
headerAppVersion := httpkit.FirstHeader(request, "X-App-Version", "X-Client-Version")
|
headerAppVersion := httpkit.FirstHeader(request, "X-App-Version", "X-Client-Version")
|
||||||
@ -75,8 +61,6 @@ func (h *Handler) reportAppEvents(writer http.ResponseWriter, request *http.Requ
|
|||||||
event, valid := appTrackingEventFromRequest(item, appTrackingEventDefaults{
|
event, valid := appTrackingEventFromRequest(item, appTrackingEventDefaults{
|
||||||
appCode: appcode.FromContext(request.Context()),
|
appCode: appcode.FromContext(request.Context()),
|
||||||
userID: userID,
|
userID: userID,
|
||||||
countryID: countryID,
|
|
||||||
regionID: regionID,
|
|
||||||
deviceID: headerDeviceID,
|
deviceID: headerDeviceID,
|
||||||
sessionID: auth.SessionIDFromContext(request.Context()),
|
sessionID: auth.SessionIDFromContext(request.Context()),
|
||||||
platform: headerPlatform,
|
platform: headerPlatform,
|
||||||
@ -91,6 +75,21 @@ func (h *Handler) reportAppEvents(writer http.ResponseWriter, request *http.Requ
|
|||||||
}
|
}
|
||||||
events = append(events, event)
|
events = append(events, event)
|
||||||
}
|
}
|
||||||
|
if h.statisticsClient == nil {
|
||||||
|
// App 埋点事实由 statistics-service 持久化;gateway 未注入写入边界时不能假成功,否则客户端会丢掉重试机会。
|
||||||
|
httpkit.WriteError(writer, request, http.StatusBadGateway, httpkit.CodeUpstreamError, "upstream service error")
|
||||||
|
return
|
||||||
|
}
|
||||||
|
// 先完成客户端可控字段的全部长度校验,再读取 user-service。这样超限
|
||||||
|
// 请求稳定返回 400,不会制造无意义的下游调用,更不会被包装成 502。
|
||||||
|
countryID, regionID, ok := h.appEventUserDimensions(writer, request, userID)
|
||||||
|
if !ok {
|
||||||
|
return
|
||||||
|
}
|
||||||
|
for index := range events {
|
||||||
|
events[index].CountryID = countryID
|
||||||
|
events[index].RegionID = regionID
|
||||||
|
}
|
||||||
|
|
||||||
result, err := h.statisticsClient.ReportAppTrackingEvents(request.Context(), events)
|
result, err := h.statisticsClient.ReportAppTrackingEvents(request.Context(), events)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
@ -128,8 +127,6 @@ func (h *Handler) appEventUserDimensions(writer http.ResponseWriter, request *ht
|
|||||||
type appTrackingEventDefaults struct {
|
type appTrackingEventDefaults struct {
|
||||||
appCode string
|
appCode string
|
||||||
userID int64
|
userID int64
|
||||||
countryID int64
|
|
||||||
regionID int64
|
|
||||||
deviceID string
|
deviceID string
|
||||||
sessionID string
|
sessionID string
|
||||||
platform string
|
platform string
|
||||||
@ -140,16 +137,7 @@ type appTrackingEventDefaults struct {
|
|||||||
}
|
}
|
||||||
|
|
||||||
func appTrackingEventFromRequest(item appEventRequest, defaults appTrackingEventDefaults) (client.AppTrackingEvent, bool) {
|
func appTrackingEventFromRequest(item appEventRequest, defaults appTrackingEventDefaults) (client.AppTrackingEvent, bool) {
|
||||||
item.EventID = strings.TrimSpace(item.EventID)
|
|
||||||
item.EventName = strings.TrimSpace(item.EventName)
|
|
||||||
if item.EventID == "" || item.EventName == "" {
|
|
||||||
return client.AppTrackingEvent{}, false
|
|
||||||
}
|
|
||||||
deviceID := firstTrimmed(item.DeviceID, defaults.deviceID)
|
deviceID := firstTrimmed(item.DeviceID, defaults.deviceID)
|
||||||
if defaults.userID <= 0 && deviceID == "" {
|
|
||||||
// 匿名埋点没有服务端 user_id,只能依赖稳定 device_id 做排障和后续粗粒度去重。
|
|
||||||
return client.AppTrackingEvent{}, false
|
|
||||||
}
|
|
||||||
properties, ok := normalizeAppEventProperties(item.Properties)
|
properties, ok := normalizeAppEventProperties(item.Properties)
|
||||||
if !ok {
|
if !ok {
|
||||||
return client.AppTrackingEvent{}, false
|
return client.AppTrackingEvent{}, false
|
||||||
@ -158,14 +146,14 @@ func appTrackingEventFromRequest(item appEventRequest, defaults appTrackingEvent
|
|||||||
if occurredAtMS <= 0 {
|
if occurredAtMS <= 0 {
|
||||||
occurredAtMS = defaults.receivedAtMS
|
occurredAtMS = defaults.receivedAtMS
|
||||||
}
|
}
|
||||||
return client.AppTrackingEvent{
|
event := client.AppTrackingEvent{
|
||||||
AppCode: defaults.appCode,
|
AppCode: defaults.appCode,
|
||||||
EventID: item.EventID,
|
EventID: item.EventID,
|
||||||
EventName: item.EventName,
|
EventName: item.EventName,
|
||||||
EventType: strings.TrimSpace(item.EventType),
|
EventType: item.EventType,
|
||||||
Screen: strings.TrimSpace(item.Screen),
|
Screen: item.Screen,
|
||||||
TargetType: strings.TrimSpace(item.TargetType),
|
TargetType: item.TargetType,
|
||||||
TargetID: strings.TrimSpace(item.TargetID),
|
TargetID: item.TargetID,
|
||||||
UserID: defaults.userID,
|
UserID: defaults.userID,
|
||||||
DeviceID: deviceID,
|
DeviceID: deviceID,
|
||||||
SessionID: firstTrimmed(defaults.sessionID, item.SessionID),
|
SessionID: firstTrimmed(defaults.sessionID, item.SessionID),
|
||||||
@ -173,25 +161,60 @@ func appTrackingEventFromRequest(item appEventRequest, defaults appTrackingEvent
|
|||||||
AppVersion: firstTrimmed(item.AppVersion, defaults.appVersion),
|
AppVersion: firstTrimmed(item.AppVersion, defaults.appVersion),
|
||||||
Language: firstTrimmed(item.Language, defaults.language),
|
Language: firstTrimmed(item.Language, defaults.language),
|
||||||
Timezone: firstTrimmed(item.Timezone, defaults.timezone),
|
Timezone: firstTrimmed(item.Timezone, defaults.timezone),
|
||||||
CountryID: defaults.countryID,
|
|
||||||
RegionID: defaults.regionID,
|
|
||||||
DurationMS: item.DurationMS,
|
DurationMS: item.DurationMS,
|
||||||
Success: item.Success,
|
Success: item.Success,
|
||||||
ErrorCode: strings.TrimSpace(item.ErrorCode),
|
ErrorCode: item.ErrorCode,
|
||||||
Properties: properties,
|
Properties: properties,
|
||||||
OccurredAtMS: occurredAtMS,
|
OccurredAtMS: occurredAtMS,
|
||||||
}, true
|
}
|
||||||
|
if !normalizeAppTrackingEventText(&event) || event.EventID == "" || event.EventName == "" {
|
||||||
|
return client.AppTrackingEvent{}, false
|
||||||
|
}
|
||||||
|
if defaults.userID <= 0 && event.DeviceID == "" {
|
||||||
|
// 匿名埋点没有服务端 user_id,只能依赖稳定 device_id 做排障和后续粗粒度去重。
|
||||||
|
return client.AppTrackingEvent{}, false
|
||||||
|
}
|
||||||
|
return event, true
|
||||||
}
|
}
|
||||||
|
|
||||||
func normalizeAppEventProperties(raw json.RawMessage) (json.RawMessage, bool) {
|
func normalizeAppEventProperties(raw json.RawMessage) (json.RawMessage, bool) {
|
||||||
trimmed := bytes.TrimSpace(raw)
|
return apptracking.NormalizeProperties(raw)
|
||||||
if len(trimmed) == 0 || bytes.Equal(trimmed, []byte("null")) {
|
}
|
||||||
return nil, true
|
|
||||||
|
func normalizeAppTrackingEventText(event *client.AppTrackingEvent) bool {
|
||||||
|
if event == nil {
|
||||||
|
return false
|
||||||
}
|
}
|
||||||
if len(trimmed) > appEventMaxPropertiesBytes || !json.Valid(trimmed) {
|
// Every field uses the same byte limit as statistics persistence. Returning
|
||||||
return nil, false
|
// false is deliberate: identifiers and diagnostic error codes must be
|
||||||
|
// accepted intact or rejected, never silently truncated into another fact.
|
||||||
|
fields := []struct {
|
||||||
|
value *string
|
||||||
|
maxBytes int
|
||||||
|
}{
|
||||||
|
{value: &event.AppCode, maxBytes: apptracking.MaxAppCodeBytes},
|
||||||
|
{value: &event.EventID, maxBytes: apptracking.MaxEventIDBytes},
|
||||||
|
{value: &event.EventName, maxBytes: apptracking.MaxEventNameBytes},
|
||||||
|
{value: &event.EventType, maxBytes: apptracking.MaxEventTypeBytes},
|
||||||
|
{value: &event.Screen, maxBytes: apptracking.MaxScreenBytes},
|
||||||
|
{value: &event.TargetType, maxBytes: apptracking.MaxTargetTypeBytes},
|
||||||
|
{value: &event.TargetID, maxBytes: apptracking.MaxTargetIDBytes},
|
||||||
|
{value: &event.DeviceID, maxBytes: apptracking.MaxDeviceIDBytes},
|
||||||
|
{value: &event.SessionID, maxBytes: apptracking.MaxSessionIDBytes},
|
||||||
|
{value: &event.Platform, maxBytes: apptracking.MaxPlatformBytes},
|
||||||
|
{value: &event.AppVersion, maxBytes: apptracking.MaxAppVersionBytes},
|
||||||
|
{value: &event.Language, maxBytes: apptracking.MaxLanguageBytes},
|
||||||
|
{value: &event.Timezone, maxBytes: apptracking.MaxTimezoneBytes},
|
||||||
|
{value: &event.ErrorCode, maxBytes: apptracking.MaxErrorCodeBytes},
|
||||||
}
|
}
|
||||||
return json.RawMessage(trimmed), true
|
for _, field := range fields {
|
||||||
|
normalized, ok := apptracking.NormalizeText(*field.value, field.maxBytes)
|
||||||
|
if !ok {
|
||||||
|
return false
|
||||||
|
}
|
||||||
|
*field.value = normalized
|
||||||
|
}
|
||||||
|
return true
|
||||||
}
|
}
|
||||||
|
|
||||||
func appEventRequestLanguage(request *http.Request) string {
|
func appEventRequestLanguage(request *http.Request) string {
|
||||||
|
|||||||
@ -4625,6 +4625,36 @@ func TestReportAppEventsRejectsInvalidRequests(t *testing.T) {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func TestReportAppEventsRejects185ByteErrorCodeBeforeDownstream(t *testing.T) {
|
||||||
|
statisticsClient := &fakeStatisticsClient{}
|
||||||
|
handler := NewHandlerWithClients(&fakeRoomClient{}, nil, nil, &fakeUserProfileClient{})
|
||||||
|
handler.SetStatisticsClient(statisticsClient)
|
||||||
|
router := handler.Routes(auth.NewVerifier("secret"))
|
||||||
|
body, err := json.Marshal(map[string]any{
|
||||||
|
"events": []map[string]any{{
|
||||||
|
"event_id": "evt-error-code-too-long",
|
||||||
|
"event_name": "pay_failed",
|
||||||
|
"device_id": "dev-1",
|
||||||
|
"error_code": strings.Repeat("e", 185),
|
||||||
|
}},
|
||||||
|
})
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("marshal app events request: %v", err)
|
||||||
|
}
|
||||||
|
request := httptest.NewRequest(http.MethodPost, "/api/v1/app/events", bytes.NewReader(body))
|
||||||
|
request.Header.Set("X-Request-ID", "req-app-events-error-code-limit")
|
||||||
|
recorder := httptest.NewRecorder()
|
||||||
|
|
||||||
|
router.ServeHTTP(recorder, request)
|
||||||
|
|
||||||
|
// Regression: statistics-service used to reject this after forwarding and
|
||||||
|
// gateway translated that downstream 400 into a misleading 502.
|
||||||
|
assertEnvelope(t, recorder, http.StatusBadRequest, httpkit.CodeInvalidArgument, "req-app-events-error-code-limit")
|
||||||
|
if len(statisticsClient.lastAppEvents) != 0 {
|
||||||
|
t.Fatalf("oversized error_code must not reach statistics-service: %+v", statisticsClient.lastAppEvents)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
func TestReportAppEventsUpstreamFailure(t *testing.T) {
|
func TestReportAppEventsUpstreamFailure(t *testing.T) {
|
||||||
statisticsClient := &fakeStatisticsClient{appErr: errors.New("statistics unavailable")}
|
statisticsClient := &fakeStatisticsClient{appErr: errors.New("statistics unavailable")}
|
||||||
handler := NewHandlerWithClients(&fakeRoomClient{}, nil, nil, &fakeUserProfileClient{})
|
handler := NewHandlerWithClients(&fakeRoomClient{}, nil, nil, &fakeUserProfileClient{})
|
||||||
|
|||||||
@ -79,6 +79,12 @@ func logRoomGiftPanelStage(ctx context.Context, requestID string, roomID string,
|
|||||||
slog.Bool("degraded", degraded),
|
slog.Bool("degraded", degraded),
|
||||||
}
|
}
|
||||||
switch {
|
switch {
|
||||||
|
case roomGiftPanelStageCanceled(err):
|
||||||
|
// 首个必要阶段失败后 handler 会取消仍在运行的 sibling RPC;这些 Canceled
|
||||||
|
// 只是请求内并发已被收敛,不是新的下游故障。保留 INFO 轨迹用于 request_id
|
||||||
|
// 关联,但不能为同一个主错误再制造多条 ERROR 告警。
|
||||||
|
attrs = append(attrs, slog.String("error", err.Error()), slog.Bool("canceled", true))
|
||||||
|
logx.Info(logCtx, "gateway_room_gift_panel_stage", attrs...)
|
||||||
case err != nil && degraded:
|
case err != nil && degraded:
|
||||||
attrs = append(attrs, slog.String("error", err.Error()))
|
attrs = append(attrs, slog.String("error", err.Error()))
|
||||||
logx.Warn(logCtx, "gateway_room_gift_panel_stage", attrs...)
|
logx.Warn(logCtx, "gateway_room_gift_panel_stage", attrs...)
|
||||||
@ -91,6 +97,10 @@ func logRoomGiftPanelStage(ctx context.Context, requestID string, roomID string,
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func roomGiftPanelStageCanceled(err error) bool {
|
||||||
|
return err != nil && (errors.Is(err, context.Canceled) || status.Code(err) == codes.Canceled)
|
||||||
|
}
|
||||||
|
|
||||||
// serveRoomGiftPanel 在单个明确预算内聚合 owner service 读模型。
|
// serveRoomGiftPanel 在单个明确预算内聚合 owner service 读模型。
|
||||||
// 第一阶段只并发互不依赖的 RPC;依赖 visible_region_id 的礼物配置在快照返回后启动,避免猜测区域。
|
// 第一阶段只并发互不依赖的 RPC;依赖 visible_region_id 的礼物配置在快照返回后启动,避免猜测区域。
|
||||||
func (h *Handler) serveRoomGiftPanel(writer http.ResponseWriter, request *http.Request) {
|
func (h *Handler) serveRoomGiftPanel(writer http.ResponseWriter, request *http.Request) {
|
||||||
|
|||||||
@ -4,6 +4,7 @@ import (
|
|||||||
"context"
|
"context"
|
||||||
"encoding/json"
|
"encoding/json"
|
||||||
"errors"
|
"errors"
|
||||||
|
"fmt"
|
||||||
"net/http"
|
"net/http"
|
||||||
"net/http/httptest"
|
"net/http/httptest"
|
||||||
"sync/atomic"
|
"sync/atomic"
|
||||||
@ -18,6 +19,9 @@ import (
|
|||||||
"hyapp/services/gateway-service/internal/auth"
|
"hyapp/services/gateway-service/internal/auth"
|
||||||
gatewayclient "hyapp/services/gateway-service/internal/client"
|
gatewayclient "hyapp/services/gateway-service/internal/client"
|
||||||
"hyapp/services/gateway-service/internal/transport/http/httpkit"
|
"hyapp/services/gateway-service/internal/transport/http/httpkit"
|
||||||
|
|
||||||
|
"google.golang.org/grpc/codes"
|
||||||
|
"google.golang.org/grpc/status"
|
||||||
)
|
)
|
||||||
|
|
||||||
type giftPanelTestRoomQueryClient struct {
|
type giftPanelTestRoomQueryClient struct {
|
||||||
@ -164,6 +168,21 @@ func TestRoomGiftPanelProductionBudgetsStayBelowFlutterTimeout(t *testing.T) {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func TestRoomGiftPanelStageCanceledRecognizesLocalAndGRPCCancellation(t *testing.T) {
|
||||||
|
for _, err := range []error{
|
||||||
|
context.Canceled,
|
||||||
|
fmt.Errorf("query interrupted: %w", context.Canceled),
|
||||||
|
status.Error(codes.Canceled, "context canceled"),
|
||||||
|
} {
|
||||||
|
if !roomGiftPanelStageCanceled(err) {
|
||||||
|
t.Fatalf("expected cancellation for %v", err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if roomGiftPanelStageCanceled(status.Error(codes.DeadlineExceeded, "deadline exceeded")) {
|
||||||
|
t.Fatal("deadline exhaustion is a real latency failure and must remain error-level")
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
func TestRoomGiftPanelRunsIndependentStagesConcurrentlyAndSkipsUnusedEnrichment(t *testing.T) {
|
func TestRoomGiftPanelRunsIndependentStagesConcurrentlyAndSkipsUnusedEnrichment(t *testing.T) {
|
||||||
phaseOne := newGiftPanelStageGate()
|
phaseOne := newGiftPanelStageGate()
|
||||||
phaseTwo := newGiftPanelStageGate()
|
phaseTwo := newGiftPanelStageGate()
|
||||||
|
|||||||
505
services/notice-service/cmd/replay-deliveries/main.go
Normal file
505
services/notice-service/cmd/replay-deliveries/main.go
Normal file
@ -0,0 +1,505 @@
|
|||||||
|
// Command replay-deliveries replays an explicitly selected set of expired
|
||||||
|
// notice delivery leases from immutable owner outbox facts.
|
||||||
|
package main
|
||||||
|
|
||||||
|
import (
|
||||||
|
"context"
|
||||||
|
"database/sql"
|
||||||
|
"encoding/json"
|
||||||
|
"errors"
|
||||||
|
"flag"
|
||||||
|
"fmt"
|
||||||
|
"log"
|
||||||
|
"os"
|
||||||
|
"sort"
|
||||||
|
"strings"
|
||||||
|
"time"
|
||||||
|
|
||||||
|
roomeventsv1 "hyapp.local/api/proto/events/room/v1"
|
||||||
|
"hyapp/pkg/appcode"
|
||||||
|
"hyapp/pkg/tencentim"
|
||||||
|
"hyapp/pkg/walletmq"
|
||||||
|
"hyapp/services/notice-service/internal/config"
|
||||||
|
"hyapp/services/notice-service/internal/modules/roomnotice"
|
||||||
|
"hyapp/services/notice-service/internal/modules/walletnotice"
|
||||||
|
mysqlplatform "hyapp/services/notice-service/internal/platform/mysql"
|
||||||
|
)
|
||||||
|
|
||||||
|
const (
|
||||||
|
walletSource = "wallet_outbox"
|
||||||
|
roomSource = "room_outbox"
|
||||||
|
c2cChannel = "tencent_im_c2c"
|
||||||
|
)
|
||||||
|
|
||||||
|
type repeatedFlag []string
|
||||||
|
|
||||||
|
func (values *repeatedFlag) String() string { return strings.Join(*values, ",") }
|
||||||
|
|
||||||
|
func (values *repeatedFlag) Set(value string) error {
|
||||||
|
value = strings.TrimSpace(value)
|
||||||
|
if value == "" {
|
||||||
|
return errors.New("flag value cannot be empty")
|
||||||
|
}
|
||||||
|
*values = append(*values, value)
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
type deliverySelector struct {
|
||||||
|
SourceName string
|
||||||
|
AppCode string
|
||||||
|
EventID string
|
||||||
|
}
|
||||||
|
|
||||||
|
func (selector deliverySelector) String() string {
|
||||||
|
return selector.SourceName + ":" + selector.AppCode + ":" + selector.EventID
|
||||||
|
}
|
||||||
|
|
||||||
|
type deliverySelectorFlag []deliverySelector
|
||||||
|
|
||||||
|
func (values *deliverySelectorFlag) String() string {
|
||||||
|
parts := make([]string, 0, len(*values))
|
||||||
|
for _, selector := range *values {
|
||||||
|
parts = append(parts, selector.String())
|
||||||
|
}
|
||||||
|
return strings.Join(parts, ",")
|
||||||
|
}
|
||||||
|
|
||||||
|
func (values *deliverySelectorFlag) Set(value string) error {
|
||||||
|
parts := strings.SplitN(strings.TrimSpace(value), ":", 3)
|
||||||
|
if len(parts) != 3 {
|
||||||
|
return errors.New("delivery must use source_name:app_code:event_id")
|
||||||
|
}
|
||||||
|
selector := deliverySelector{
|
||||||
|
SourceName: strings.TrimSpace(parts[0]),
|
||||||
|
AppCode: strings.TrimSpace(parts[1]),
|
||||||
|
EventID: strings.TrimSpace(parts[2]),
|
||||||
|
}
|
||||||
|
if selector.SourceName != walletSource && selector.SourceName != roomSource {
|
||||||
|
return fmt.Errorf("unsupported delivery source %q", selector.SourceName)
|
||||||
|
}
|
||||||
|
if selector.AppCode == "" || selector.EventID == "" {
|
||||||
|
return errors.New("delivery app_code and event_id are required")
|
||||||
|
}
|
||||||
|
selector.AppCode = appcode.Normalize(selector.AppCode)
|
||||||
|
*values = append(*values, selector)
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
type deliveryCandidate struct {
|
||||||
|
Selector string `json:"selector"`
|
||||||
|
SourceName string `json:"source_name"`
|
||||||
|
AppCode string `json:"app_code"`
|
||||||
|
EventID string `json:"event_id"`
|
||||||
|
Status string `json:"status"`
|
||||||
|
RetryCount int `json:"retry_count"`
|
||||||
|
UpdatedAtMS int64 `json:"updated_at_ms"`
|
||||||
|
}
|
||||||
|
|
||||||
|
type replayItem struct {
|
||||||
|
Candidate deliveryCandidate
|
||||||
|
WalletMessage *walletmq.WalletOutboxMessage
|
||||||
|
RoomEnvelope *roomeventsv1.EventEnvelope
|
||||||
|
}
|
||||||
|
|
||||||
|
func main() {
|
||||||
|
configPath := flag.String("config", "services/notice-service/configs/config.yaml", "path to notice-service config")
|
||||||
|
execute := flag.Bool("execute", false, "publish selected due deliveries; default is a read-only preview")
|
||||||
|
limit := flag.Int("limit", 100, "maximum candidate count, 1..500")
|
||||||
|
var eventIDs repeatedFlag
|
||||||
|
var appCodes repeatedFlag
|
||||||
|
var deliveries deliverySelectorFlag
|
||||||
|
flag.Var(&eventIDs, "event-id", "preview filter for a source event ID; repeat for multiple events")
|
||||||
|
flag.Var(&appCodes, "app-code", "required App partition for indexed preview; repeat for multiple Apps")
|
||||||
|
flag.Var(&deliveries, "delivery", "exact source_name:app_code:event_id selector copied from preview; repeat for multiple deliveries")
|
||||||
|
flag.Parse()
|
||||||
|
|
||||||
|
if *limit <= 0 || *limit > 500 {
|
||||||
|
log.Fatal("limit must be between 1 and 500")
|
||||||
|
}
|
||||||
|
if len(deliveries) > 0 && (len(eventIDs) > 0 || len(appCodes) > 0) {
|
||||||
|
log.Fatal("--delivery cannot be combined with --event-id or --app-code")
|
||||||
|
}
|
||||||
|
if *execute && len(deliveries) == 0 {
|
||||||
|
// A write run is pinned to full primary-key selectors copied from preview;
|
||||||
|
// separate App/event filters can form an accidental cross product.
|
||||||
|
log.Fatal("--execute requires at least one exact --delivery from a preview")
|
||||||
|
}
|
||||||
|
if len(deliveries) > *limit {
|
||||||
|
log.Fatal("exact delivery count exceeds --limit")
|
||||||
|
}
|
||||||
|
if len(deliveries) == 0 && len(appCodes) == 0 {
|
||||||
|
// notice_delivery_events indexes start with source_name,app_code. Refusing
|
||||||
|
// an unpartitioned preview prevents a LIMIT query from scanning a 20+ GiB table.
|
||||||
|
log.Fatal("preview requires at least one --app-code or exact --delivery")
|
||||||
|
}
|
||||||
|
|
||||||
|
cfg, err := config.Load(*configPath)
|
||||||
|
if err != nil {
|
||||||
|
log.Fatal(err)
|
||||||
|
}
|
||||||
|
ctx, cancel := context.WithTimeout(context.Background(), 2*time.Minute)
|
||||||
|
defer cancel()
|
||||||
|
store, err := mysqlplatform.Open(ctx, cfg.MySQLDSN)
|
||||||
|
if err != nil {
|
||||||
|
log.Fatal(err)
|
||||||
|
}
|
||||||
|
defer func() { _ = store.Close() }()
|
||||||
|
|
||||||
|
nowMS := time.Now().UnixMilli()
|
||||||
|
var candidates []deliveryCandidate
|
||||||
|
if len(deliveries) > 0 {
|
||||||
|
candidates, err = loadExactDueDeliveries(ctx, store.DB, deliveries, nowMS)
|
||||||
|
} else {
|
||||||
|
candidates, err = listDueDeliveries(ctx, store.DB, eventIDs, appCodes, *limit, nowMS)
|
||||||
|
}
|
||||||
|
if err != nil {
|
||||||
|
log.Fatal(err)
|
||||||
|
}
|
||||||
|
if !*execute {
|
||||||
|
encoder := json.NewEncoder(os.Stdout)
|
||||||
|
for _, candidate := range candidates {
|
||||||
|
if err := encoder.Encode(candidate); err != nil {
|
||||||
|
log.Fatal(err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
fmt.Fprintf(os.Stderr, "previewed %d due delivery rows; no writes performed\n", len(candidates))
|
||||||
|
return
|
||||||
|
}
|
||||||
|
if !cfg.TencentIM.Enabled {
|
||||||
|
log.Fatal("tencent_im.enabled is required for replay")
|
||||||
|
}
|
||||||
|
walletRepo, err := walletnotice.NewMySQLRepository(store.DB, cfg.WalletDatabase)
|
||||||
|
if err != nil {
|
||||||
|
log.Fatal(err)
|
||||||
|
}
|
||||||
|
roomRepo, err := roomnotice.NewMySQLRepository(store.DB, cfg.RoomDatabase)
|
||||||
|
if err != nil {
|
||||||
|
log.Fatal(err)
|
||||||
|
}
|
||||||
|
items, err := loadReplayItems(ctx, candidates, walletRepo, roomRepo)
|
||||||
|
if err != nil {
|
||||||
|
// Preload every immutable owner fact before creating an IM client or
|
||||||
|
// publishing the first item. One missing/corrupt fact therefore aborts the
|
||||||
|
// exact set without producing a partial operator repair.
|
||||||
|
log.Fatal(err)
|
||||||
|
}
|
||||||
|
publisher, err := tencentim.NewRESTClient(cfg.TencentIM.RESTConfig())
|
||||||
|
if err != nil {
|
||||||
|
log.Fatal(err)
|
||||||
|
}
|
||||||
|
walletService := walletnotice.New(walletnotice.Config{NodeID: cfg.NodeID}, walletRepo, publisher)
|
||||||
|
roomService := roomnotice.New(roomnotice.Config{NodeID: cfg.NodeID}, roomRepo, publisher)
|
||||||
|
walletOptions := walletWorkerOptions(cfg)
|
||||||
|
roomOptions := roomWorkerOptions(cfg)
|
||||||
|
|
||||||
|
failures := 0
|
||||||
|
for _, item := range items {
|
||||||
|
candidate := item.Candidate
|
||||||
|
candidateCtx := appcode.WithContext(ctx, candidate.AppCode)
|
||||||
|
var replayErr error
|
||||||
|
switch candidate.SourceName {
|
||||||
|
case walletSource:
|
||||||
|
if item.WalletMessage == nil {
|
||||||
|
replayErr = errors.New("wallet owner snapshot is missing")
|
||||||
|
} else {
|
||||||
|
_, replayErr = walletService.ProcessWalletOutboxMessage(candidateCtx, *item.WalletMessage, walletOptions)
|
||||||
|
}
|
||||||
|
case roomSource:
|
||||||
|
if item.RoomEnvelope == nil {
|
||||||
|
replayErr = errors.New("room owner snapshot is missing")
|
||||||
|
} else {
|
||||||
|
_, replayErr = roomService.ProcessRoomOutboxEnvelope(candidateCtx, item.RoomEnvelope, roomOptions)
|
||||||
|
}
|
||||||
|
default:
|
||||||
|
replayErr = fmt.Errorf("unsupported source %q", candidate.SourceName)
|
||||||
|
}
|
||||||
|
if replayErr == nil {
|
||||||
|
replayErr = requireDelivered(ctx, store.DB, candidate)
|
||||||
|
}
|
||||||
|
if replayErr != nil {
|
||||||
|
failures++
|
||||||
|
fmt.Fprintf(os.Stderr, "replay failed source=%s app_code=%s event_id=%s: %v\n", candidate.SourceName, candidate.AppCode, candidate.EventID, replayErr)
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
fmt.Printf("delivered source=%s app_code=%s event_id=%s\n", candidate.SourceName, candidate.AppCode, candidate.EventID)
|
||||||
|
}
|
||||||
|
if failures > 0 {
|
||||||
|
log.Fatalf("replay finished with %d failures out of %d selected rows", failures, len(candidates))
|
||||||
|
}
|
||||||
|
fmt.Printf("replay delivered %d selected rows\n", len(candidates))
|
||||||
|
}
|
||||||
|
|
||||||
|
func loadReplayItems(ctx context.Context, candidates []deliveryCandidate, walletRepo *walletnotice.MySQLRepository, roomRepo *roomnotice.MySQLRepository) ([]replayItem, error) {
|
||||||
|
items := make([]replayItem, 0, len(candidates))
|
||||||
|
for _, candidate := range candidates {
|
||||||
|
candidateCtx := appcode.WithContext(ctx, candidate.AppCode)
|
||||||
|
item := replayItem{Candidate: candidate}
|
||||||
|
switch candidate.SourceName {
|
||||||
|
case walletSource:
|
||||||
|
if walletRepo == nil {
|
||||||
|
return nil, errors.New("wallet owner repository is not configured")
|
||||||
|
}
|
||||||
|
message, err := walletRepo.LoadWalletOutboxMessage(candidateCtx, candidate.AppCode, candidate.EventID)
|
||||||
|
if err != nil {
|
||||||
|
return nil, fmt.Errorf("load owner snapshot source=%s app_code=%s event_id=%s: %w", candidate.SourceName, candidate.AppCode, candidate.EventID, err)
|
||||||
|
}
|
||||||
|
item.WalletMessage = &message
|
||||||
|
case roomSource:
|
||||||
|
if roomRepo == nil {
|
||||||
|
return nil, errors.New("room owner repository is not configured")
|
||||||
|
}
|
||||||
|
envelope, err := roomRepo.LoadRoomOutboxEnvelope(candidateCtx, candidate.AppCode, candidate.EventID)
|
||||||
|
if err != nil {
|
||||||
|
return nil, fmt.Errorf("load owner snapshot source=%s app_code=%s event_id=%s: %w", candidate.SourceName, candidate.AppCode, candidate.EventID, err)
|
||||||
|
}
|
||||||
|
item.RoomEnvelope = envelope
|
||||||
|
default:
|
||||||
|
return nil, fmt.Errorf("unsupported source %q", candidate.SourceName)
|
||||||
|
}
|
||||||
|
items = append(items, item)
|
||||||
|
}
|
||||||
|
return items, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func listDueDeliveries(ctx context.Context, db *sql.DB, eventIDs []string, appCodes []string, limit int, nowMS int64) ([]deliveryCandidate, error) {
|
||||||
|
if db == nil {
|
||||||
|
return nil, errors.New("notice database is not configured")
|
||||||
|
}
|
||||||
|
if limit <= 0 || limit > 500 {
|
||||||
|
return nil, errors.New("limit must be between 1 and 500")
|
||||||
|
}
|
||||||
|
apps := normalizeAppCodes(appCodes)
|
||||||
|
if len(apps) == 0 {
|
||||||
|
return nil, errors.New("at least one app_code is required for indexed preview")
|
||||||
|
}
|
||||||
|
if len(apps) > 32 {
|
||||||
|
return nil, errors.New("app_code filter count exceeds 32")
|
||||||
|
}
|
||||||
|
ids := normalizeEventIDs(eventIDs)
|
||||||
|
if len(ids) > 500 {
|
||||||
|
return nil, errors.New("event_id filter count exceeds 500")
|
||||||
|
}
|
||||||
|
candidates := make([]deliveryCandidate, 0, limit)
|
||||||
|
for _, sourceName := range []string{walletSource, roomSource} {
|
||||||
|
for _, appCodeValue := range apps {
|
||||||
|
branches := []struct {
|
||||||
|
indexName string
|
||||||
|
status string
|
||||||
|
dueColumn string
|
||||||
|
}{
|
||||||
|
{indexName: "idx_notice_delivery_retry", status: "retryable", dueColumn: "next_retry_at_ms"},
|
||||||
|
{indexName: "idx_notice_delivery_lock", status: "delivering", dueColumn: "lock_until_ms"},
|
||||||
|
}
|
||||||
|
for _, branch := range branches {
|
||||||
|
query, args := dueDeliveryBranch(sourceName, appCodeValue, branch.indexName, branch.status, branch.dueColumn, ids, limit, nowMS)
|
||||||
|
branchCandidates, err := queryDueDeliveryCandidates(ctx, db, query, args...)
|
||||||
|
if err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
candidates = append(candidates, branchCandidates...)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
sort.Slice(candidates, func(i, j int) bool {
|
||||||
|
if candidates[i].UpdatedAtMS != candidates[j].UpdatedAtMS {
|
||||||
|
return candidates[i].UpdatedAtMS < candidates[j].UpdatedAtMS
|
||||||
|
}
|
||||||
|
return candidates[i].Selector < candidates[j].Selector
|
||||||
|
})
|
||||||
|
if len(candidates) > limit {
|
||||||
|
candidates = candidates[:limit]
|
||||||
|
}
|
||||||
|
return candidates, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func loadExactDueDeliveries(ctx context.Context, db *sql.DB, selectors []deliverySelector, nowMS int64) ([]deliveryCandidate, error) {
|
||||||
|
if db == nil {
|
||||||
|
return nil, errors.New("notice database is not configured")
|
||||||
|
}
|
||||||
|
normalized, err := normalizeDeliverySelectors(selectors)
|
||||||
|
if err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
candidates := make([]deliveryCandidate, 0, len(selectors))
|
||||||
|
for _, selector := range normalized {
|
||||||
|
key := selector.String()
|
||||||
|
var candidate deliveryCandidate
|
||||||
|
err = db.QueryRowContext(ctx, `
|
||||||
|
SELECT source_name, app_code, source_event_id, status, retry_count, updated_at_ms
|
||||||
|
FROM notice_delivery_events FORCE INDEX (PRIMARY)
|
||||||
|
WHERE source_name = ? AND app_code = ? AND source_event_id = ? AND channel = ?
|
||||||
|
AND ((status = 'retryable' AND next_retry_at_ms <= ?)
|
||||||
|
OR (status = 'delivering' AND lock_until_ms <= ?))`,
|
||||||
|
selector.SourceName,
|
||||||
|
selector.AppCode,
|
||||||
|
selector.EventID,
|
||||||
|
c2cChannel,
|
||||||
|
nowMS,
|
||||||
|
nowMS,
|
||||||
|
).Scan(&candidate.SourceName, &candidate.AppCode, &candidate.EventID, &candidate.Status, &candidate.RetryCount, &candidate.UpdatedAtMS)
|
||||||
|
if errors.Is(err, sql.ErrNoRows) {
|
||||||
|
return nil, fmt.Errorf("delivery %q does not exist or is no longer due", key)
|
||||||
|
}
|
||||||
|
if err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
if candidate.SourceName != selector.SourceName || candidate.AppCode != selector.AppCode || candidate.EventID != selector.EventID {
|
||||||
|
return nil, fmt.Errorf("delivery %q returned a mismatched primary key", key)
|
||||||
|
}
|
||||||
|
candidate.Selector = key
|
||||||
|
candidates = append(candidates, candidate)
|
||||||
|
}
|
||||||
|
return candidates, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func normalizeDeliverySelectors(values []deliverySelector) ([]deliverySelector, error) {
|
||||||
|
normalized := make([]deliverySelector, 0, len(values))
|
||||||
|
seen := make(map[string]struct{}, len(values))
|
||||||
|
for _, value := range values {
|
||||||
|
selector := deliverySelector{
|
||||||
|
SourceName: strings.TrimSpace(value.SourceName),
|
||||||
|
AppCode: strings.TrimSpace(value.AppCode),
|
||||||
|
EventID: strings.TrimSpace(value.EventID),
|
||||||
|
}
|
||||||
|
if selector.SourceName != walletSource && selector.SourceName != roomSource {
|
||||||
|
return nil, fmt.Errorf("unsupported delivery source %q", selector.SourceName)
|
||||||
|
}
|
||||||
|
if selector.AppCode == "" || selector.EventID == "" {
|
||||||
|
return nil, errors.New("delivery app_code and event_id are required")
|
||||||
|
}
|
||||||
|
selector.AppCode = appcode.Normalize(selector.AppCode)
|
||||||
|
key := selector.String()
|
||||||
|
if _, exists := seen[key]; exists {
|
||||||
|
return nil, fmt.Errorf("duplicate delivery selector %q", key)
|
||||||
|
}
|
||||||
|
seen[key] = struct{}{}
|
||||||
|
normalized = append(normalized, selector)
|
||||||
|
}
|
||||||
|
return normalized, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func dueDeliveryBranch(sourceName string, appCodeValue string, indexName string, status string, dueColumn string, eventIDs []string, limit int, nowMS int64) (string, []any) {
|
||||||
|
// Callers pass only compile-time index/status/column tuples. Splitting status
|
||||||
|
// branches gives MySQL the complete (source,app,channel,status,due_time) prefix.
|
||||||
|
query := `
|
||||||
|
SELECT source_name, app_code, source_event_id, status, retry_count, updated_at_ms
|
||||||
|
FROM notice_delivery_events FORCE INDEX (` + indexName + `)
|
||||||
|
WHERE source_name = ? AND app_code = ? AND channel = ? AND status = ?
|
||||||
|
AND ` + dueColumn + ` <= ?`
|
||||||
|
args := []any{sourceName, appCodeValue, c2cChannel, status, nowMS}
|
||||||
|
if len(eventIDs) > 0 {
|
||||||
|
query += " AND source_event_id IN (" + placeholders(len(eventIDs)) + ")"
|
||||||
|
for _, eventID := range eventIDs {
|
||||||
|
args = append(args, eventID)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
query += " ORDER BY updated_at_ms ASC, source_event_id ASC LIMIT ?"
|
||||||
|
args = append(args, limit)
|
||||||
|
return query, args
|
||||||
|
}
|
||||||
|
|
||||||
|
func queryDueDeliveryCandidates(ctx context.Context, db *sql.DB, query string, args ...any) ([]deliveryCandidate, error) {
|
||||||
|
rows, err := db.QueryContext(ctx, query, args...)
|
||||||
|
if err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
defer rows.Close()
|
||||||
|
candidates := make([]deliveryCandidate, 0)
|
||||||
|
for rows.Next() {
|
||||||
|
var candidate deliveryCandidate
|
||||||
|
if err := rows.Scan(&candidate.SourceName, &candidate.AppCode, &candidate.EventID, &candidate.Status, &candidate.RetryCount, &candidate.UpdatedAtMS); err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
candidate.Selector = deliverySelector{SourceName: candidate.SourceName, AppCode: candidate.AppCode, EventID: candidate.EventID}.String()
|
||||||
|
candidates = append(candidates, candidate)
|
||||||
|
}
|
||||||
|
return candidates, rows.Err()
|
||||||
|
}
|
||||||
|
|
||||||
|
func normalizeAppCodes(values []string) []string {
|
||||||
|
normalized := make([]string, 0, len(values))
|
||||||
|
seen := make(map[string]struct{}, len(values))
|
||||||
|
for _, value := range values {
|
||||||
|
if strings.TrimSpace(value) == "" {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
value = appcode.Normalize(value)
|
||||||
|
if _, exists := seen[value]; exists {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
seen[value] = struct{}{}
|
||||||
|
normalized = append(normalized, value)
|
||||||
|
}
|
||||||
|
return normalized
|
||||||
|
}
|
||||||
|
|
||||||
|
func normalizeEventIDs(values []string) []string {
|
||||||
|
normalized := make([]string, 0, len(values))
|
||||||
|
seen := make(map[string]struct{}, len(values))
|
||||||
|
for _, value := range values {
|
||||||
|
value = strings.TrimSpace(value)
|
||||||
|
if value == "" {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
if _, exists := seen[value]; exists {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
seen[value] = struct{}{}
|
||||||
|
normalized = append(normalized, value)
|
||||||
|
}
|
||||||
|
return normalized
|
||||||
|
}
|
||||||
|
|
||||||
|
func requireDelivered(ctx context.Context, db *sql.DB, candidate deliveryCandidate) error {
|
||||||
|
var status string
|
||||||
|
err := db.QueryRowContext(ctx, `
|
||||||
|
SELECT status
|
||||||
|
FROM notice_delivery_events
|
||||||
|
WHERE source_name = ? AND app_code = ? AND source_event_id = ? AND channel = ?`,
|
||||||
|
candidate.SourceName,
|
||||||
|
candidate.AppCode,
|
||||||
|
candidate.EventID,
|
||||||
|
c2cChannel,
|
||||||
|
).Scan(&status)
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
if status != "delivered" {
|
||||||
|
return fmt.Errorf("delivery status is %q after replay", status)
|
||||||
|
}
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func walletWorkerOptions(cfg config.Config) walletnotice.WalletNoticeWorkerOptions {
|
||||||
|
worker := cfg.WalletNoticeWorker
|
||||||
|
return walletnotice.WalletNoticeWorkerOptions{
|
||||||
|
WorkerID: "wallet-notice-replay-" + cfg.NodeID,
|
||||||
|
BatchSize: worker.BatchSize,
|
||||||
|
LockTTL: worker.LockTTL,
|
||||||
|
PublishTimeout: worker.PublishTimeout,
|
||||||
|
MaxRetryCount: worker.MaxRetryCount,
|
||||||
|
InitialBackoff: worker.InitialBackoff,
|
||||||
|
MaxBackoff: worker.MaxBackoff,
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func roomWorkerOptions(cfg config.Config) roomnotice.RoomNoticeWorkerOptions {
|
||||||
|
worker := cfg.RoomNoticeWorker
|
||||||
|
return roomnotice.RoomNoticeWorkerOptions{
|
||||||
|
WorkerID: "room-notice-replay-" + cfg.NodeID,
|
||||||
|
BatchSize: worker.BatchSize,
|
||||||
|
LockTTL: worker.LockTTL,
|
||||||
|
PublishTimeout: worker.PublishTimeout,
|
||||||
|
MaxRetryCount: worker.MaxRetryCount,
|
||||||
|
InitialBackoff: worker.InitialBackoff,
|
||||||
|
MaxBackoff: worker.MaxBackoff,
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func placeholders(count int) string {
|
||||||
|
values := make([]string, count)
|
||||||
|
for index := range values {
|
||||||
|
values[index] = "?"
|
||||||
|
}
|
||||||
|
return strings.Join(values, ",")
|
||||||
|
}
|
||||||
112
services/notice-service/cmd/replay-deliveries/main_test.go
Normal file
112
services/notice-service/cmd/replay-deliveries/main_test.go
Normal file
@ -0,0 +1,112 @@
|
|||||||
|
package main
|
||||||
|
|
||||||
|
import (
|
||||||
|
"context"
|
||||||
|
"database/sql"
|
||||||
|
"regexp"
|
||||||
|
"strings"
|
||||||
|
"testing"
|
||||||
|
|
||||||
|
"github.com/DATA-DOG/go-sqlmock"
|
||||||
|
)
|
||||||
|
|
||||||
|
func TestDeliverySelectorFlagKeepsColonInsideEventID(t *testing.T) {
|
||||||
|
var values deliverySelectorFlag
|
||||||
|
if err := values.Set("wallet_outbox: FAMI :wallet:balance:evt-1"); err != nil {
|
||||||
|
t.Fatalf("parse exact delivery selector: %v", err)
|
||||||
|
}
|
||||||
|
if len(values) != 1 || values[0].SourceName != walletSource || values[0].AppCode != "fami" || values[0].EventID != "wallet:balance:evt-1" {
|
||||||
|
t.Fatalf("parsed selector mismatch: %+v", values)
|
||||||
|
}
|
||||||
|
if err := values.Set("unknown:lalu:event-2"); err == nil {
|
||||||
|
t.Fatal("unsupported source must be rejected before any database access")
|
||||||
|
}
|
||||||
|
if err := values.Set("room_outbox::event-3"); err == nil {
|
||||||
|
t.Fatal("empty app_code must not default to lalu")
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestLoadExactDueDeliveriesUsesFullPrimaryKey(t *testing.T) {
|
||||||
|
db, mock, err := sqlmock.New()
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("create sqlmock: %v", err)
|
||||||
|
}
|
||||||
|
defer db.Close()
|
||||||
|
nowMS := int64(1_700_000_000_000)
|
||||||
|
query := regexp.QuoteMeta(`
|
||||||
|
SELECT source_name, app_code, source_event_id, status, retry_count, updated_at_ms
|
||||||
|
FROM notice_delivery_events FORCE INDEX (PRIMARY)
|
||||||
|
WHERE source_name = ? AND app_code = ? AND source_event_id = ? AND channel = ?
|
||||||
|
AND ((status = 'retryable' AND next_retry_at_ms <= ?)
|
||||||
|
OR (status = 'delivering' AND lock_until_ms <= ?))`)
|
||||||
|
mock.ExpectQuery(query).
|
||||||
|
WithArgs(walletSource, "fami", "wallet:evt-1", c2cChannel, nowMS, nowMS).
|
||||||
|
WillReturnRows(sqlmock.NewRows([]string{"source_name", "app_code", "source_event_id", "status", "retry_count", "updated_at_ms"}).
|
||||||
|
AddRow(walletSource, "fami", "wallet:evt-1", "retryable", 2, nowMS-1000))
|
||||||
|
|
||||||
|
candidates, err := loadExactDueDeliveries(context.Background(), db, []deliverySelector{{SourceName: walletSource, AppCode: " FAMI ", EventID: "wallet:evt-1"}}, nowMS)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("load exact due delivery: %v", err)
|
||||||
|
}
|
||||||
|
if len(candidates) != 1 || candidates[0].Selector != "wallet_outbox:fami:wallet:evt-1" || candidates[0].RetryCount != 2 {
|
||||||
|
t.Fatalf("exact candidate mismatch: %+v", candidates)
|
||||||
|
}
|
||||||
|
if err := mock.ExpectationsWereMet(); err != nil {
|
||||||
|
t.Fatalf("exact query did not use the expected primary key: %v", err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestLoadExactDueDeliveriesRejectsDuplicateBeforeQuery(t *testing.T) {
|
||||||
|
db, mock, err := sqlmock.New()
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("create sqlmock: %v", err)
|
||||||
|
}
|
||||||
|
defer db.Close()
|
||||||
|
selectors := []deliverySelector{
|
||||||
|
{SourceName: roomSource, AppCode: "lalu", EventID: "evt-1"},
|
||||||
|
{SourceName: roomSource, AppCode: " LALU ", EventID: "evt-1"},
|
||||||
|
}
|
||||||
|
if _, err := loadExactDueDeliveries(context.Background(), db, selectors, 1000); err == nil || !strings.Contains(err.Error(), "duplicate") {
|
||||||
|
t.Fatalf("duplicate selector error = %v", err)
|
||||||
|
}
|
||||||
|
if err := mock.ExpectationsWereMet(); err != nil {
|
||||||
|
t.Fatalf("duplicate validation unexpectedly queried MySQL: %v", err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestLoadExactDueDeliveriesRejectsNotDueRow(t *testing.T) {
|
||||||
|
db, mock, err := sqlmock.New()
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("create sqlmock: %v", err)
|
||||||
|
}
|
||||||
|
defer db.Close()
|
||||||
|
mock.ExpectQuery("FROM notice_delivery_events FORCE INDEX \\(PRIMARY\\)").
|
||||||
|
WillReturnError(sql.ErrNoRows)
|
||||||
|
_, err = loadExactDueDeliveries(context.Background(), db, []deliverySelector{{SourceName: roomSource, AppCode: "lalu", EventID: "evt-1"}}, 1000)
|
||||||
|
if err == nil || !strings.Contains(err.Error(), "no longer due") {
|
||||||
|
t.Fatalf("not-due selector error = %v", err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestDueDeliveryBranchUsesCompleteRetryIndexPrefix(t *testing.T) {
|
||||||
|
query, args := dueDeliveryBranch(walletSource, "huwaa", "idx_notice_delivery_retry", "retryable", "next_retry_at_ms", []string{"evt-1", "evt-2"}, 20, 1234)
|
||||||
|
for _, fragment := range []string{
|
||||||
|
"FORCE INDEX (idx_notice_delivery_retry)",
|
||||||
|
"source_name = ? AND app_code = ? AND channel = ? AND status = ?",
|
||||||
|
"next_retry_at_ms <= ?",
|
||||||
|
"source_event_id IN (?,?)",
|
||||||
|
} {
|
||||||
|
if !strings.Contains(query, fragment) {
|
||||||
|
t.Fatalf("indexed preview query is missing %q:\n%s", fragment, query)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
wantArgs := []any{walletSource, "huwaa", c2cChannel, "retryable", int64(1234), "evt-1", "evt-2", 20}
|
||||||
|
if len(args) != len(wantArgs) {
|
||||||
|
t.Fatalf("preview args = %v, want %v", args, wantArgs)
|
||||||
|
}
|
||||||
|
for index := range args {
|
||||||
|
if args[index] != wantArgs[index] {
|
||||||
|
t.Fatalf("preview arg[%d] = %#v, want %#v", index, args[index], wantArgs[index])
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
@ -251,6 +251,47 @@ func (r *MySQLRepository) ClaimRoomKickEnvelope(ctx context.Context, workerID st
|
|||||||
return event, true, nil
|
return event, true, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// LoadRoomOutboxEnvelope loads one immutable owner event for an exact offline
|
||||||
|
// replay. Requiring the full primary key prevents repair tooling from scanning or
|
||||||
|
// claiming room-service state that was not selected by an operator.
|
||||||
|
func (r *MySQLRepository) LoadRoomOutboxEnvelope(ctx context.Context, appCode string, eventID string) (*roomeventsv1.EventEnvelope, error) {
|
||||||
|
if r == nil || r.db == nil {
|
||||||
|
return nil, xerr.New(xerr.Unavailable, "notice repository is not configured")
|
||||||
|
}
|
||||||
|
appCode = strings.TrimSpace(appCode)
|
||||||
|
eventID = strings.TrimSpace(eventID)
|
||||||
|
if appCode == "" || eventID == "" {
|
||||||
|
return nil, xerr.New(xerr.InvalidArgument, "app_code and event_id are required")
|
||||||
|
}
|
||||||
|
appCode = appcode.Normalize(appCode)
|
||||||
|
|
||||||
|
var (
|
||||||
|
ownerRoomID string
|
||||||
|
envelopeBytes []byte
|
||||||
|
)
|
||||||
|
query := fmt.Sprintf(`
|
||||||
|
SELECT room_id, envelope
|
||||||
|
FROM %s
|
||||||
|
WHERE app_code = ? AND event_id = ? AND event_type = ?`, r.roomOutboxTable)
|
||||||
|
if err := r.db.QueryRowContext(ctx, query, appCode, eventID, eventRoomUserKicked).Scan(&ownerRoomID, &envelopeBytes); err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
var envelope roomeventsv1.EventEnvelope
|
||||||
|
if err := proto.Unmarshal(envelopeBytes, &envelope); err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
// Validate both the requested identity and the current consumer contract. A
|
||||||
|
// mismatched historical blob must never be delivered under another event's
|
||||||
|
// notice_delivery_events key.
|
||||||
|
if envelope.GetAppCode() != appCode || envelope.GetEventId() != eventID || envelope.GetRoomId() != ownerRoomID {
|
||||||
|
return nil, fmt.Errorf("room outbox envelope identity does not match requested key")
|
||||||
|
}
|
||||||
|
if _, err := roomKickEventFromEnvelope(&envelope, envelope.GetOccurredAtMs()); err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
return &envelope, nil
|
||||||
|
}
|
||||||
|
|
||||||
func (r *MySQLRepository) lockRoomKickEvent(ctx context.Context, tx *sql.Tx, candidate roomKickCandidate) (RoomKickEvent, error) {
|
func (r *MySQLRepository) lockRoomKickEvent(ctx context.Context, tx *sql.Tx, candidate roomKickCandidate) (RoomKickEvent, error) {
|
||||||
var (
|
var (
|
||||||
event RoomKickEvent
|
event RoomKickEvent
|
||||||
|
|||||||
@ -0,0 +1,68 @@
|
|||||||
|
package roomnotice
|
||||||
|
|
||||||
|
import (
|
||||||
|
"context"
|
||||||
|
"regexp"
|
||||||
|
"testing"
|
||||||
|
|
||||||
|
"github.com/DATA-DOG/go-sqlmock"
|
||||||
|
"google.golang.org/protobuf/proto"
|
||||||
|
roomeventsv1 "hyapp.local/api/proto/events/room/v1"
|
||||||
|
)
|
||||||
|
|
||||||
|
func TestLoadRoomOutboxEnvelopeRejectsEmptyAppInsteadOfDefaultingToLalu(t *testing.T) {
|
||||||
|
db, mock, err := sqlmock.New()
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("create sqlmock: %v", err)
|
||||||
|
}
|
||||||
|
defer db.Close()
|
||||||
|
repository := &MySQLRepository{db: db, roomOutboxTable: "`hyapp_room`.`room_outbox`"}
|
||||||
|
if _, err := repository.LoadRoomOutboxEnvelope(context.Background(), "", "evt-1"); err == nil {
|
||||||
|
t.Fatal("empty app_code must be rejected before owner table access")
|
||||||
|
}
|
||||||
|
if err := mock.ExpectationsWereMet(); err != nil {
|
||||||
|
t.Fatalf("empty key unexpectedly queried room owner table: %v", err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestLoadRoomOutboxEnvelopeUsesExactImmutableOwnerEnvelope(t *testing.T) {
|
||||||
|
db, mock, err := sqlmock.New()
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("create sqlmock: %v", err)
|
||||||
|
}
|
||||||
|
defer db.Close()
|
||||||
|
body, err := proto.Marshal(&roomeventsv1.RoomUserKicked{ActorUserId: 7, TargetUserId: 42})
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("marshal room kick body: %v", err)
|
||||||
|
}
|
||||||
|
envelopeBytes, err := proto.Marshal(&roomeventsv1.EventEnvelope{
|
||||||
|
AppCode: "huwaa",
|
||||||
|
EventId: "room:kick:evt-1",
|
||||||
|
EventType: eventRoomUserKicked,
|
||||||
|
RoomId: "room-1",
|
||||||
|
RoomVersion: 9,
|
||||||
|
OccurredAtMs: 1700000000000,
|
||||||
|
Body: body,
|
||||||
|
})
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("marshal room envelope: %v", err)
|
||||||
|
}
|
||||||
|
repository := &MySQLRepository{db: db, roomOutboxTable: "`hyapp_room`.`room_outbox`"}
|
||||||
|
mock.ExpectQuery(regexp.QuoteMeta(`
|
||||||
|
SELECT room_id, envelope
|
||||||
|
FROM `+"`hyapp_room`.`room_outbox`"+`
|
||||||
|
WHERE app_code = ? AND event_id = ? AND event_type = ?`)).
|
||||||
|
WithArgs("huwaa", "room:kick:evt-1", eventRoomUserKicked).
|
||||||
|
WillReturnRows(sqlmock.NewRows([]string{"room_id", "envelope"}).AddRow("room-1", envelopeBytes))
|
||||||
|
|
||||||
|
envelope, err := repository.LoadRoomOutboxEnvelope(context.Background(), " HUWAA ", "room:kick:evt-1")
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("load exact room owner envelope: %v", err)
|
||||||
|
}
|
||||||
|
if envelope.GetAppCode() != "huwaa" || envelope.GetEventId() != "room:kick:evt-1" || envelope.GetRoomId() != "room-1" {
|
||||||
|
t.Fatalf("room owner snapshot mismatch: %+v", envelope)
|
||||||
|
}
|
||||||
|
if err := mock.ExpectationsWereMet(); err != nil {
|
||||||
|
t.Fatalf("room replay query mismatch: %v", err)
|
||||||
|
}
|
||||||
|
}
|
||||||
@ -256,6 +256,61 @@ func (r *MySQLRepository) ClaimWalletBalanceMessage(ctx context.Context, workerI
|
|||||||
return event, true, nil
|
return event, true, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// LoadWalletOutboxMessage loads one immutable owner event for an operator-selected
|
||||||
|
// delivery replay. The exact (app_code, event_id) key is mandatory so an offline
|
||||||
|
// repair cannot accidentally turn into an unbounded wallet_outbox scan.
|
||||||
|
func (r *MySQLRepository) LoadWalletOutboxMessage(ctx context.Context, appCode string, eventID string) (walletmq.WalletOutboxMessage, error) {
|
||||||
|
if r == nil || r.db == nil {
|
||||||
|
return walletmq.WalletOutboxMessage{}, xerr.New(xerr.Unavailable, "notice repository is not configured")
|
||||||
|
}
|
||||||
|
appCode = strings.TrimSpace(appCode)
|
||||||
|
eventID = strings.TrimSpace(eventID)
|
||||||
|
if appCode == "" || eventID == "" {
|
||||||
|
return walletmq.WalletOutboxMessage{}, xerr.New(xerr.InvalidArgument, "app_code and event_id are required")
|
||||||
|
}
|
||||||
|
appCode = appcode.Normalize(appCode)
|
||||||
|
|
||||||
|
message := walletmq.WalletOutboxMessage{MessageType: walletmq.MessageTypeWalletOutboxEvent}
|
||||||
|
query := fmt.Sprintf(`
|
||||||
|
SELECT app_code, event_id, event_type, transaction_id, command_id, user_id,
|
||||||
|
asset_type, available_delta, frozen_delta,
|
||||||
|
COALESCE(CAST(payload AS CHAR), '{}'), created_at_ms
|
||||||
|
FROM %s
|
||||||
|
WHERE app_code = ? AND event_id = ?
|
||||||
|
AND event_type IN (?, ?) AND asset_type <> ?`, r.walletOutboxTable)
|
||||||
|
err := r.db.QueryRowContext(ctx, query, appCode, eventID, eventWalletBalance, eventVIPActivated, robotCoinAssetType).Scan(
|
||||||
|
&message.AppCode,
|
||||||
|
&message.EventID,
|
||||||
|
&message.EventType,
|
||||||
|
&message.TransactionID,
|
||||||
|
&message.CommandID,
|
||||||
|
&message.UserID,
|
||||||
|
&message.AssetType,
|
||||||
|
&message.AvailableDelta,
|
||||||
|
&message.FrozenDelta,
|
||||||
|
&message.PayloadJSON,
|
||||||
|
&message.OccurredAtMS,
|
||||||
|
)
|
||||||
|
if err != nil {
|
||||||
|
return walletmq.WalletOutboxMessage{}, err
|
||||||
|
}
|
||||||
|
// Reuse the live MQ decoder boundary before any external side effect. A
|
||||||
|
// malformed historical row must stay retryable instead of being sent with a
|
||||||
|
// subtly different repair-only payload.
|
||||||
|
encoded, err := walletmq.EncodeWalletOutboxMessage(message)
|
||||||
|
if err != nil {
|
||||||
|
return walletmq.WalletOutboxMessage{}, err
|
||||||
|
}
|
||||||
|
decoded, err := walletmq.DecodeWalletOutboxMessage(encoded)
|
||||||
|
if err != nil {
|
||||||
|
return walletmq.WalletOutboxMessage{}, err
|
||||||
|
}
|
||||||
|
if decoded.AppCode != appCode || decoded.EventID != eventID {
|
||||||
|
return walletmq.WalletOutboxMessage{}, fmt.Errorf("wallet outbox message identity does not match requested key")
|
||||||
|
}
|
||||||
|
return decoded, nil
|
||||||
|
}
|
||||||
|
|
||||||
func walletBalanceEventFromMessage(message walletmq.WalletOutboxMessage) (WalletBalanceEvent, error) {
|
func walletBalanceEventFromMessage(message walletmq.WalletOutboxMessage) (WalletBalanceEvent, error) {
|
||||||
event := WalletBalanceEvent{
|
event := WalletBalanceEvent{
|
||||||
AppCode: appcode.Normalize(message.AppCode),
|
AppCode: appcode.Normalize(message.AppCode),
|
||||||
|
|||||||
@ -0,0 +1,56 @@
|
|||||||
|
package walletnotice
|
||||||
|
|
||||||
|
import (
|
||||||
|
"context"
|
||||||
|
"regexp"
|
||||||
|
"testing"
|
||||||
|
|
||||||
|
"github.com/DATA-DOG/go-sqlmock"
|
||||||
|
)
|
||||||
|
|
||||||
|
func TestLoadWalletOutboxMessageRejectsEmptyAppInsteadOfDefaultingToLalu(t *testing.T) {
|
||||||
|
db, mock, err := sqlmock.New()
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("create sqlmock: %v", err)
|
||||||
|
}
|
||||||
|
defer db.Close()
|
||||||
|
repository := &MySQLRepository{db: db, walletOutboxTable: "`hyapp_wallet`.`wallet_outbox`"}
|
||||||
|
if _, err := repository.LoadWalletOutboxMessage(context.Background(), " ", "evt-1"); err == nil {
|
||||||
|
t.Fatal("empty app_code must be rejected before owner table access")
|
||||||
|
}
|
||||||
|
if err := mock.ExpectationsWereMet(); err != nil {
|
||||||
|
t.Fatalf("empty key unexpectedly queried wallet owner table: %v", err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestLoadWalletOutboxMessageUsesExactImmutablePrivateFact(t *testing.T) {
|
||||||
|
db, mock, err := sqlmock.New()
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("create sqlmock: %v", err)
|
||||||
|
}
|
||||||
|
defer db.Close()
|
||||||
|
repository := &MySQLRepository{db: db, walletOutboxTable: "`hyapp_wallet`.`wallet_outbox`"}
|
||||||
|
mock.ExpectQuery(regexp.QuoteMeta(`
|
||||||
|
SELECT app_code, event_id, event_type, transaction_id, command_id, user_id,
|
||||||
|
asset_type, available_delta, frozen_delta,
|
||||||
|
COALESCE(CAST(payload AS CHAR), '{}'), created_at_ms
|
||||||
|
FROM `+"`hyapp_wallet`.`wallet_outbox`"+`
|
||||||
|
WHERE app_code = ? AND event_id = ?
|
||||||
|
AND event_type IN (?, ?) AND asset_type <> ?`)).
|
||||||
|
WithArgs("fami", "wallet:evt-1", eventWalletBalance, eventVIPActivated, robotCoinAssetType).
|
||||||
|
WillReturnRows(sqlmock.NewRows([]string{
|
||||||
|
"app_code", "event_id", "event_type", "transaction_id", "command_id", "user_id",
|
||||||
|
"asset_type", "available_delta", "frozen_delta", "payload", "created_at_ms",
|
||||||
|
}).AddRow("fami", "wallet:evt-1", eventWalletBalance, "tx-1", "cmd-1", int64(42), "COIN", int64(100), int64(0), `{"balance_after":100}`, int64(1700000000000)))
|
||||||
|
|
||||||
|
message, err := repository.LoadWalletOutboxMessage(context.Background(), " FAMI ", "wallet:evt-1")
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("load exact wallet owner fact: %v", err)
|
||||||
|
}
|
||||||
|
if message.AppCode != "fami" || message.EventID != "wallet:evt-1" || message.UserID != 42 || message.PayloadJSON != `{"balance_after":100}` {
|
||||||
|
t.Fatalf("wallet owner snapshot mismatch: %+v", message)
|
||||||
|
}
|
||||||
|
if err := mock.ExpectationsWereMet(); err != nil {
|
||||||
|
t.Fatalf("wallet replay query mismatch: %v", err)
|
||||||
|
}
|
||||||
|
}
|
||||||
@ -516,12 +516,26 @@ func (s *Service) ensureRobotRoomParticipants(ctx context.Context, config RobotR
|
|||||||
activeRobots := onlineRobotIDs(config.RobotUserIDs, online)
|
activeRobots := onlineRobotIDs(config.RobotUserIDs, online)
|
||||||
missing := activeTarget - len(activeRobots)
|
missing := activeTarget - len(activeRobots)
|
||||||
if missing > 0 {
|
if missing > 0 {
|
||||||
|
// 首次补回缺席机器人也必须从当前 Room Cell 快照选择真实麦位。候选池下标可能
|
||||||
|
// 大于 seat_count;同一轮成功补位后立刻占记,避免后续候选复用尚未写回旧快照的麦位。
|
||||||
|
occupiedSeats := make(map[int32]bool, len(snapshot.GetMicSeats()))
|
||||||
|
for _, seat := range snapshot.GetMicSeats() {
|
||||||
|
if seat.GetUserId() > 0 {
|
||||||
|
occupiedSeats[seat.GetSeatNo()] = true
|
||||||
|
}
|
||||||
|
}
|
||||||
for _, userID := range robotRoomFillCandidates(config, online, missing) {
|
for _, userID := range robotRoomFillCandidates(config, online, missing) {
|
||||||
// active 机器人房从持久化配置恢复时,旧进程可能已经把机器人 stale 成 left;
|
// active 机器人房从持久化配置恢复时,旧进程可能已经把机器人 stale 成 left;
|
||||||
// 这里只补齐后台配置的活跃人数,不再把整个候选池一次性塞回房间。
|
// 这里只补齐后台配置的活跃人数,不再把整个候选池一次性塞回房间。
|
||||||
if err := s.joinRobotRoomUser(roomCtx, config, userID, 0, fmt.Sprintf("runtime-join:%d:%d", userID, time.Now().UTC().UnixNano())); err != nil {
|
seatNo := robotRestoreSeatNo(config, userID, snapshot.GetMicSeats(), occupiedSeats)
|
||||||
|
if seatNo <= 0 {
|
||||||
|
logx.Warn(roomCtx, "robot_room_runtime_join_restore_skipped", slog.String("room_id", config.RoomID), slog.Int64("robot_user_id", userID), slog.String("reason", "no_free_seat"))
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
if err := s.joinRobotRoomUser(roomCtx, config, userID, seatNo, fmt.Sprintf("runtime-join:%d:%d", userID, time.Now().UTC().UnixNano())); err != nil {
|
||||||
return err
|
return err
|
||||||
}
|
}
|
||||||
|
occupiedSeats[seatNo] = true
|
||||||
online[userID] = true
|
online[userID] = true
|
||||||
activeRobots = append(activeRobots, userID)
|
activeRobots = append(activeRobots, userID)
|
||||||
}
|
}
|
||||||
@ -544,13 +558,10 @@ func (s *Service) ensureRobotRoomParticipants(ctx context.Context, config RobotR
|
|||||||
if userID <= 0 || seated[userID] {
|
if userID <= 0 || seated[userID] {
|
||||||
continue
|
continue
|
||||||
}
|
}
|
||||||
seatNo := preferredRobotSeatNo(config, userID)
|
seatNo := robotRestoreSeatNo(config, userID, snapshot.GetMicSeats(), occupiedSeats)
|
||||||
if occupiedSeats[seatNo] {
|
if seatNo <= 0 {
|
||||||
seatNo = firstFreeRobotSeatNo(snapshot.GetMicSeats(), activeTarget)
|
logx.Warn(roomCtx, "robot_room_runtime_mic_restore_skipped", slog.String("room_id", config.RoomID), slog.Int64("robot_user_id", userID), slog.String("reason", "no_free_seat"))
|
||||||
if seatNo <= 0 {
|
continue
|
||||||
logx.Warn(roomCtx, "robot_room_runtime_mic_restore_skipped", slog.String("room_id", config.RoomID), slog.Int64("robot_user_id", userID), slog.String("reason", "no_free_seat"))
|
|
||||||
continue
|
|
||||||
}
|
|
||||||
}
|
}
|
||||||
// 虚拟麦位只服务机器人房展示;若并发恢复导致机器人已上麦,Conflict 可视为达成目标。
|
// 虚拟麦位只服务机器人房展示;若并发恢复导致机器人已上麦,Conflict 可视为达成目标。
|
||||||
if _, err := s.RobotVirtualMicUp(roomCtx, RobotMicUpInput{
|
if _, err := s.RobotVirtualMicUp(roomCtx, RobotMicUpInput{
|
||||||
@ -568,6 +579,39 @@ func (s *Service) ensureRobotRoomParticipants(ctx context.Context, config RobotR
|
|||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// robotRestoreSeatNo 只接受当前 Room Cell 快照里真实存在且空闲的麦位。
|
||||||
|
// robot_user_ids 是候选池而不是麦位表:历史配置可包含数十个轮换机器人,因此其下标
|
||||||
|
// 可能远大于当前 10/15/20 个麦位;这种 preferred 必须回退到快照中的首个空麦,不能
|
||||||
|
// 把不存在的 seat_no 发给 RobotVirtualMicUp 后让每 10 秒恢复扫描永久失败。
|
||||||
|
func robotRestoreSeatNo(config RobotRoomConfig, userID int64, seats []*roomv1.SeatState, occupied map[int32]bool) int32 {
|
||||||
|
preferred := preferredRobotSeatNo(config, userID)
|
||||||
|
if preferred > 0 && robotSnapshotSeatFree(seats, preferred) && !occupied[preferred] {
|
||||||
|
return preferred
|
||||||
|
}
|
||||||
|
// snapshot 是本轮恢复开始时的只读副本;occupied 还包含本轮刚成功分配、尚未反映到
|
||||||
|
// 该副本的麦位。回退时同时检查二者,避免多个待恢复机器人重复选择同一个空麦。
|
||||||
|
var firstFree int32
|
||||||
|
for _, seat := range seats {
|
||||||
|
seatNo := seat.GetSeatNo()
|
||||||
|
if seatNo <= 0 || seat.GetUserId() > 0 || seat.GetLocked() || occupied[seatNo] {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
if firstFree == 0 || seatNo < firstFree {
|
||||||
|
firstFree = seatNo
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return firstFree
|
||||||
|
}
|
||||||
|
|
||||||
|
func robotSnapshotSeatFree(seats []*roomv1.SeatState, seatNo int32) bool {
|
||||||
|
for _, seat := range seats {
|
||||||
|
if seat.GetSeatNo() == seatNo {
|
||||||
|
return seat.GetUserId() == 0 && !seat.GetLocked()
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return false
|
||||||
|
}
|
||||||
|
|
||||||
func (s *Service) runRobotRoomRuntime(ctx context.Context, config RobotRoomConfig) {
|
func (s *Service) runRobotRoomRuntime(ctx context.Context, config RobotRoomConfig) {
|
||||||
activity := newRobotRoomActivity(config.RobotUserIDs)
|
activity := newRobotRoomActivity(config.RobotUserIDs)
|
||||||
if snapshot, err := s.currentSnapshot(appcode.WithContext(ctx, config.AppCode), config.RoomID); err == nil {
|
if snapshot, err := s.currentSnapshot(appcode.WithContext(ctx, config.AppCode), config.RoomID); err == nil {
|
||||||
|
|||||||
@ -89,6 +89,101 @@ func TestRobotRoomRuntimeStartRestoresLeftRobots(t *testing.T) {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func TestRobotRoomRuntimeStartRestoresHighIndexOwnerToRealSeat(t *testing.T) {
|
||||||
|
ctx := appcode.WithContext(context.Background(), appcode.Default)
|
||||||
|
repository := mysqltest.NewRepository(t)
|
||||||
|
svc := roomservice.New(roomservice.Config{
|
||||||
|
NodeID: "node-robot-runtime-high-index-restore-test",
|
||||||
|
LeaseTTL: 10 * time.Second,
|
||||||
|
RankLimit: 20,
|
||||||
|
SnapshotEveryN: 1,
|
||||||
|
}, router.NewMemoryDirectory(), repository, followTestWallet{}, integration.NewNoopRoomEventPublisher(), integration.NewNoopOutboxPublisher())
|
||||||
|
|
||||||
|
// 候选池有 12 个机器人而房间只有 10 个麦位;owner 排序后位于第 12 位。
|
||||||
|
// bootstrap 会把 active owner 放到真实 seat 1,运行时恢复不能再按候选池下标猜 seat 12。
|
||||||
|
candidates := make([]int64, 12)
|
||||||
|
for index := range candidates {
|
||||||
|
candidates[index] = int64(2001 + index)
|
||||||
|
}
|
||||||
|
ownerID := candidates[len(candidates)-1]
|
||||||
|
created, err := svc.AdminCreateRobotRoom(ctx, &roomv1.AdminCreateRobotRoomRequest{
|
||||||
|
Meta: roomservice.NewRequestMeta("", 9001),
|
||||||
|
OwnerRobotUserId: ownerID,
|
||||||
|
CandidateRobotUserIds: candidates,
|
||||||
|
MinRobotCount: 2,
|
||||||
|
MaxRobotCount: 2,
|
||||||
|
SeatCount: 10,
|
||||||
|
RoomName: "Robot Runtime High Index Restore",
|
||||||
|
RoomAvatar: testRoomCoverURL,
|
||||||
|
VisibleRegionId: 686,
|
||||||
|
GiftRule: &roomv1.AdminRobotRoomGiftRule{
|
||||||
|
GiftIds: []string{"normal_gift"},
|
||||||
|
LuckyGiftIds: []string{"lucky_gift"},
|
||||||
|
NormalGiftIntervalMs: 10000,
|
||||||
|
LuckyComboMin: 1,
|
||||||
|
LuckyComboMax: 1,
|
||||||
|
LuckyPauseMinMs: 5000,
|
||||||
|
LuckyPauseMaxMs: 5000,
|
||||||
|
},
|
||||||
|
AdminId: 9001,
|
||||||
|
})
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("create high-index-owner robot room failed: %v", err)
|
||||||
|
}
|
||||||
|
roomID := created.GetRoom().GetRoomId()
|
||||||
|
robotIDs := created.GetRoom().GetRobotUserIds()
|
||||||
|
if len(robotIDs) != len(candidates) || robotIDs[len(robotIDs)-1] != ownerID {
|
||||||
|
t.Fatalf("test requires owner at preferred seat 12, got owner=%d robots=%v", ownerID, robotIDs)
|
||||||
|
}
|
||||||
|
|
||||||
|
initial, err := svc.GetRoomSnapshot(ctx, &roomv1.GetRoomSnapshotRequest{
|
||||||
|
Meta: &roomv1.RequestMeta{AppCode: appcode.Default, RoomId: roomID},
|
||||||
|
RoomId: roomID,
|
||||||
|
ViewerUserId: ownerID,
|
||||||
|
})
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("initial high-index-owner snapshot failed: %v", err)
|
||||||
|
}
|
||||||
|
if _, err := svc.AdminSetRobotRoomStatus(ctx, &roomv1.AdminSetRobotRoomStatusRequest{
|
||||||
|
Meta: roomservice.NewRequestMeta(roomID, 9001),
|
||||||
|
Status: "stopped",
|
||||||
|
AdminId: 9001,
|
||||||
|
}); err != nil {
|
||||||
|
t.Fatalf("stop high-index-owner robot room failed: %v", err)
|
||||||
|
}
|
||||||
|
for _, user := range initial.GetRoom().GetOnlineUsers() {
|
||||||
|
if _, err := svc.LeaveRoom(ctx, &roomv1.LeaveRoomRequest{Meta: roomservice.NewRequestMeta(roomID, user.GetUserId())}); err != nil {
|
||||||
|
t.Fatalf("leave initial robot %d failed: %v", user.GetUserId(), err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
if _, err := svc.AdminSetRobotRoomStatus(ctx, &roomv1.AdminSetRobotRoomStatusRequest{
|
||||||
|
Meta: roomservice.NewRequestMeta(roomID, 9001),
|
||||||
|
Status: "active",
|
||||||
|
AdminId: 9001,
|
||||||
|
}); err != nil {
|
||||||
|
t.Fatalf("restart high-index-owner robot room failed: %v", err)
|
||||||
|
}
|
||||||
|
restored, err := svc.GetRoomSnapshot(ctx, &roomv1.GetRoomSnapshotRequest{
|
||||||
|
Meta: &roomv1.RequestMeta{AppCode: appcode.Default, RoomId: roomID},
|
||||||
|
RoomId: roomID,
|
||||||
|
ViewerUserId: ownerID,
|
||||||
|
})
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("snapshot after high-index-owner restore failed: %v", err)
|
||||||
|
}
|
||||||
|
ownerSeat := int32(0)
|
||||||
|
for _, seat := range restored.GetRoom().GetMicSeats() {
|
||||||
|
if seat.GetUserId() == ownerID {
|
||||||
|
ownerSeat = seat.GetSeatNo()
|
||||||
|
break
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if ownerSeat <= 0 || ownerSeat > int32(len(restored.GetRoom().GetMicSeats())) {
|
||||||
|
t.Fatalf("high-index owner must be restored to a real snapshot seat, got seat=%d", ownerSeat)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
func snapshotContainsUser(snapshot *roomv1.RoomSnapshot, userID int64) bool {
|
func snapshotContainsUser(snapshot *roomv1.RoomSnapshot, userID int64) bool {
|
||||||
for _, user := range snapshot.GetOnlineUsers() {
|
for _, user := range snapshot.GetOnlineUsers() {
|
||||||
if user.GetUserId() == userID {
|
if user.GetUserId() == userID {
|
||||||
|
|||||||
@ -123,6 +123,51 @@ func TestNormalizeRobotRoomGiftRuleAppliesBehaviorDefaults(t *testing.T) {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func TestRobotRestoreSeatNoFallsBackWhenPreferredExceedsSnapshot(t *testing.T) {
|
||||||
|
robots := make([]int64, 12)
|
||||||
|
for index := range robots {
|
||||||
|
robots[index] = int64(1001 + index)
|
||||||
|
}
|
||||||
|
seats := make([]*roomv1.SeatState, 10)
|
||||||
|
for index := range seats {
|
||||||
|
seats[index] = &roomv1.SeatState{SeatNo: int32(index + 1)}
|
||||||
|
}
|
||||||
|
seats[0].UserId = robots[0]
|
||||||
|
|
||||||
|
// 候选池第 12 个机器人历史 preferred=12,但当前房间只有 10 个麦位;
|
||||||
|
// 恢复必须选择真实存在的首个空麦 2,而不是把 seat=12 交给 Room Cell。
|
||||||
|
got := robotRestoreSeatNo(
|
||||||
|
RobotRoomConfig{RobotUserIDs: robots, SeatCount: 10},
|
||||||
|
robots[11],
|
||||||
|
seats,
|
||||||
|
map[int32]bool{1: true},
|
||||||
|
)
|
||||||
|
if got != 2 {
|
||||||
|
t.Fatalf("out-of-range preferred seat must fall back to first free snapshot seat: got %d want 2", got)
|
||||||
|
}
|
||||||
|
|
||||||
|
got = robotRestoreSeatNo(
|
||||||
|
RobotRoomConfig{RobotUserIDs: robots, SeatCount: 10},
|
||||||
|
robots[11],
|
||||||
|
seats,
|
||||||
|
map[int32]bool{1: true, 2: true},
|
||||||
|
)
|
||||||
|
if got != 3 {
|
||||||
|
t.Fatalf("same-pass occupied seat must not be selected twice: got %d want 3", got)
|
||||||
|
}
|
||||||
|
|
||||||
|
seats[2].Locked = true
|
||||||
|
got = robotRestoreSeatNo(
|
||||||
|
RobotRoomConfig{RobotUserIDs: robots, SeatCount: 10},
|
||||||
|
robots[11],
|
||||||
|
seats,
|
||||||
|
map[int32]bool{1: true, 2: true},
|
||||||
|
)
|
||||||
|
if got != 4 {
|
||||||
|
t.Fatalf("locked snapshot seat is not available for restore: got %d want 4", got)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
func TestRobotRoomActivityPrefersOtherReplacement(t *testing.T) {
|
func TestRobotRoomActivityPrefersOtherReplacement(t *testing.T) {
|
||||||
activity := newRobotRoomActivity([]int64{1001, 1002, 1003})
|
activity := newRobotRoomActivity([]int64{1001, 1002, 1003})
|
||||||
activity.markActive(1001)
|
activity.markActive(1001)
|
||||||
|
|||||||
@ -11,6 +11,7 @@ import (
|
|||||||
"time"
|
"time"
|
||||||
|
|
||||||
"hyapp/pkg/appcode"
|
"hyapp/pkg/appcode"
|
||||||
|
"hyapp/pkg/apptracking"
|
||||||
mysqlstorage "hyapp/services/statistics-service/internal/storage/mysql"
|
mysqlstorage "hyapp/services/statistics-service/internal/storage/mysql"
|
||||||
)
|
)
|
||||||
|
|
||||||
@ -185,13 +186,13 @@ func (s *queryHTTPServer) socialRequirements(w http.ResponseWriter, r *http.Requ
|
|||||||
endMS = startMS + 24*time.Hour.Milliseconds()
|
endMS = startMS + 24*time.Hour.Milliseconds()
|
||||||
}
|
}
|
||||||
requirements, err := s.repo.QuerySocialRequirements(r.Context(), mysqlstorage.SocialRequirementsQuery{
|
requirements, err := s.repo.QuerySocialRequirements(r.Context(), mysqlstorage.SocialRequirementsQuery{
|
||||||
AppCode: query.Get("app_code"),
|
AppCode: query.Get("app_code"),
|
||||||
StatTZ: query.Get("stat_tz"),
|
StatTZ: query.Get("stat_tz"),
|
||||||
StartMS: startMS,
|
StartMS: startMS,
|
||||||
EndMS: endMS,
|
EndMS: endMS,
|
||||||
CountryID: parseInt64(query.Get("country_id")),
|
CountryID: parseInt64(query.Get("country_id")),
|
||||||
RegionID: parseInt64(query.Get("region_id")),
|
RegionID: parseInt64(query.Get("region_id")),
|
||||||
RegionIDs: parseInt64CSV(query.Get("region_ids")),
|
RegionIDs: parseInt64CSV(query.Get("region_ids")),
|
||||||
Section: query.Get("section"),
|
Section: query.Get("section"),
|
||||||
UserRole: query.Get("user_role"),
|
UserRole: query.Get("user_role"),
|
||||||
PayerType: query.Get("payer_type"),
|
PayerType: query.Get("payer_type"),
|
||||||
@ -326,7 +327,7 @@ func (s *queryHTTPServer) appEvents(w http.ResponseWriter, r *http.Request) {
|
|||||||
writeJSON(w, http.StatusBadRequest, map[string]any{"error": "invalid json"})
|
writeJSON(w, http.StatusBadRequest, map[string]any{"error": "invalid json"})
|
||||||
return
|
return
|
||||||
}
|
}
|
||||||
if len(body.Events) == 0 || len(body.Events) > 50 {
|
if !apptracking.ValidBatchSize(len(body.Events)) {
|
||||||
writeJSON(w, http.StatusBadRequest, map[string]any{"error": "events batch is invalid"})
|
writeJSON(w, http.StatusBadRequest, map[string]any{"error": "events batch is invalid"})
|
||||||
return
|
return
|
||||||
}
|
}
|
||||||
|
|||||||
@ -13,6 +13,7 @@ import (
|
|||||||
"github.com/DATA-DOG/go-sqlmock"
|
"github.com/DATA-DOG/go-sqlmock"
|
||||||
"hyapp/internal/testutil/mysqlschema"
|
"hyapp/internal/testutil/mysqlschema"
|
||||||
"hyapp/pkg/appcode"
|
"hyapp/pkg/appcode"
|
||||||
|
"hyapp/pkg/apptracking"
|
||||||
)
|
)
|
||||||
|
|
||||||
func TestGiftDebitConsumptionOwnersDoNotOverlap(t *testing.T) {
|
func TestGiftDebitConsumptionOwnersDoNotOverlap(t *testing.T) {
|
||||||
@ -627,7 +628,7 @@ func TestConsumeAppTrackingEventsRejectsInvalidProperties(t *testing.T) {
|
|||||||
properties json.RawMessage
|
properties json.RawMessage
|
||||||
}{
|
}{
|
||||||
{name: "invalid json", properties: json.RawMessage(`{"bad"`)},
|
{name: "invalid json", properties: json.RawMessage(`{"bad"`)},
|
||||||
{name: "too large json", properties: json.RawMessage(`"` + strings.Repeat("a", appTrackingMaxPropertiesBytes+1) + `"`)},
|
{name: "too large json", properties: json.RawMessage(`"` + strings.Repeat("a", apptracking.MaxPropertiesBytes+1) + `"`)},
|
||||||
} {
|
} {
|
||||||
t.Run(test.name, func(t *testing.T) {
|
t.Run(test.name, func(t *testing.T) {
|
||||||
_, err := repository.ConsumeAppTrackingEvents(appcode.WithContext(ctx, "lalu"), []AppTrackingEvent{
|
_, err := repository.ConsumeAppTrackingEvents(appcode.WithContext(ctx, "lalu"), []AppTrackingEvent{
|
||||||
@ -640,6 +641,29 @@ func TestConsumeAppTrackingEventsRejectsInvalidProperties(t *testing.T) {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func TestNormalizeAppTrackingEventRejects185ByteErrorCodeWithoutTruncation(t *testing.T) {
|
||||||
|
base := AppTrackingEvent{
|
||||||
|
AppCode: "lalu",
|
||||||
|
EventID: "evt-error-code-limit",
|
||||||
|
EventName: "pay_failed",
|
||||||
|
DeviceID: "dev-1",
|
||||||
|
OccurredAtMS: time.Date(2026, 7, 13, 0, 0, 0, 0, time.UTC).UnixMilli(),
|
||||||
|
}
|
||||||
|
base.ErrorCode = strings.Repeat("e", apptracking.MaxErrorCodeBytes)
|
||||||
|
normalized, _, err := normalizeAppTrackingEvent(context.Background(), base, base.OccurredAtMS)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("error_code at shared byte limit must remain valid: %v", err)
|
||||||
|
}
|
||||||
|
if normalized.ErrorCode != base.ErrorCode {
|
||||||
|
t.Fatalf("valid error_code must be stored intact: got=%q want=%q", normalized.ErrorCode, base.ErrorCode)
|
||||||
|
}
|
||||||
|
|
||||||
|
base.ErrorCode = strings.Repeat("e", 185)
|
||||||
|
if _, _, err := normalizeAppTrackingEvent(context.Background(), base, base.OccurredAtMS); err == nil {
|
||||||
|
t.Fatal("185-byte error_code must be rejected instead of truncated or deferred to MySQL")
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
func TestConsumeUserRegisteredCountsOnlyNewRegistration(t *testing.T) {
|
func TestConsumeUserRegisteredCountsOnlyNewRegistration(t *testing.T) {
|
||||||
ctx := context.Background()
|
ctx := context.Background()
|
||||||
_, file, _, _ := runtime.Caller(0)
|
_, file, _, _ := runtime.Caller(0)
|
||||||
|
|||||||
@ -1,7 +1,6 @@
|
|||||||
package mysql
|
package mysql
|
||||||
|
|
||||||
import (
|
import (
|
||||||
"bytes"
|
|
||||||
"context"
|
"context"
|
||||||
"database/sql"
|
"database/sql"
|
||||||
"encoding/json"
|
"encoding/json"
|
||||||
@ -11,6 +10,7 @@ import (
|
|||||||
|
|
||||||
mysqlerr "github.com/go-sql-driver/mysql"
|
mysqlerr "github.com/go-sql-driver/mysql"
|
||||||
"hyapp/pkg/appcode"
|
"hyapp/pkg/appcode"
|
||||||
|
"hyapp/pkg/apptracking"
|
||||||
"hyapp/pkg/xerr"
|
"hyapp/pkg/xerr"
|
||||||
)
|
)
|
||||||
|
|
||||||
@ -23,11 +23,6 @@ const (
|
|||||||
SourceFinance = "finance"
|
SourceFinance = "finance"
|
||||||
)
|
)
|
||||||
|
|
||||||
const (
|
|
||||||
appTrackingMaxBatchSize = 50
|
|
||||||
appTrackingMaxPropertiesBytes = 4096
|
|
||||||
)
|
|
||||||
|
|
||||||
// Repository owns the statistics read model. Online queries must use aggregate
|
// Repository owns the statistics read model. Online queries must use aggregate
|
||||||
// tables; the optional activity connection only reads lucky-gift day snapshots,
|
// tables; the optional activity connection only reads lucky-gift day snapshots,
|
||||||
// never the high-cardinality draw fact table.
|
// never the high-cardinality draw fact table.
|
||||||
@ -1670,7 +1665,7 @@ func (r *Repository) ConsumeAppTrackingEvents(ctx context.Context, events []AppT
|
|||||||
}
|
}
|
||||||
nowMS := time.Now().UTC().UnixMilli()
|
nowMS := time.Now().UTC().UnixMilli()
|
||||||
result := AppTrackingReportResult{Accepted: true, Received: len(events), ServerTimeMS: nowMS}
|
result := AppTrackingReportResult{Accepted: true, Received: len(events), ServerTimeMS: nowMS}
|
||||||
if len(events) == 0 || len(events) > appTrackingMaxBatchSize {
|
if !apptracking.ValidBatchSize(len(events)) {
|
||||||
return AppTrackingReportResult{}, xerr.New(xerr.InvalidArgument, "app tracking events batch is invalid")
|
return AppTrackingReportResult{}, xerr.New(xerr.InvalidArgument, "app tracking events batch is invalid")
|
||||||
}
|
}
|
||||||
tx, err := r.db.BeginTx(ctx, nil)
|
tx, err := r.db.BeginTx(ctx, nil)
|
||||||
@ -2329,62 +2324,65 @@ func normalizeAppTrackingEvent(ctx context.Context, event AppTrackingEvent, rece
|
|||||||
if strings.TrimSpace(event.AppCode) == "" {
|
if strings.TrimSpace(event.AppCode) == "" {
|
||||||
event.AppCode = appcode.FromContext(ctx)
|
event.AppCode = appcode.FromContext(ctx)
|
||||||
}
|
}
|
||||||
event.AppCode = appcode.Normalize(event.AppCode)
|
event.AppCode, err = normalizeAppTrackingText(appcode.Normalize(event.AppCode), apptracking.MaxAppCodeBytes)
|
||||||
event.EventID, err = normalizeAppTrackingText(event.EventID, 160)
|
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return AppTrackingEvent{}, "", err
|
return AppTrackingEvent{}, "", err
|
||||||
}
|
}
|
||||||
event.EventName, err = normalizeAppTrackingText(event.EventName, 96)
|
event.EventID, err = normalizeAppTrackingText(event.EventID, apptracking.MaxEventIDBytes)
|
||||||
|
if err != nil {
|
||||||
|
return AppTrackingEvent{}, "", err
|
||||||
|
}
|
||||||
|
event.EventName, err = normalizeAppTrackingText(event.EventName, apptracking.MaxEventNameBytes)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return AppTrackingEvent{}, "", err
|
return AppTrackingEvent{}, "", err
|
||||||
}
|
}
|
||||||
if event.EventID == "" || event.EventName == "" {
|
if event.EventID == "" || event.EventName == "" {
|
||||||
return AppTrackingEvent{}, "", xerr.New(xerr.InvalidArgument, "app tracking event_id and event_name are required")
|
return AppTrackingEvent{}, "", xerr.New(xerr.InvalidArgument, "app tracking event_id and event_name are required")
|
||||||
}
|
}
|
||||||
event.EventType, err = normalizeAppTrackingText(event.EventType, 64)
|
event.EventType, err = normalizeAppTrackingText(event.EventType, apptracking.MaxEventTypeBytes)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return AppTrackingEvent{}, "", err
|
return AppTrackingEvent{}, "", err
|
||||||
}
|
}
|
||||||
event.Screen, err = normalizeAppTrackingText(event.Screen, 128)
|
event.Screen, err = normalizeAppTrackingText(event.Screen, apptracking.MaxScreenBytes)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return AppTrackingEvent{}, "", err
|
return AppTrackingEvent{}, "", err
|
||||||
}
|
}
|
||||||
event.TargetType, err = normalizeAppTrackingText(event.TargetType, 64)
|
event.TargetType, err = normalizeAppTrackingText(event.TargetType, apptracking.MaxTargetTypeBytes)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return AppTrackingEvent{}, "", err
|
return AppTrackingEvent{}, "", err
|
||||||
}
|
}
|
||||||
event.TargetID, err = normalizeAppTrackingText(event.TargetID, 128)
|
event.TargetID, err = normalizeAppTrackingText(event.TargetID, apptracking.MaxTargetIDBytes)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return AppTrackingEvent{}, "", err
|
return AppTrackingEvent{}, "", err
|
||||||
}
|
}
|
||||||
event.DeviceID, err = normalizeAppTrackingText(event.DeviceID, 128)
|
event.DeviceID, err = normalizeAppTrackingText(event.DeviceID, apptracking.MaxDeviceIDBytes)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return AppTrackingEvent{}, "", err
|
return AppTrackingEvent{}, "", err
|
||||||
}
|
}
|
||||||
if event.UserID <= 0 && event.DeviceID == "" {
|
if event.UserID <= 0 && event.DeviceID == "" {
|
||||||
return AppTrackingEvent{}, "", xerr.New(xerr.InvalidArgument, "anonymous app tracking event requires device_id")
|
return AppTrackingEvent{}, "", xerr.New(xerr.InvalidArgument, "anonymous app tracking event requires device_id")
|
||||||
}
|
}
|
||||||
event.SessionID, err = normalizeAppTrackingText(event.SessionID, 160)
|
event.SessionID, err = normalizeAppTrackingText(event.SessionID, apptracking.MaxSessionIDBytes)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return AppTrackingEvent{}, "", err
|
return AppTrackingEvent{}, "", err
|
||||||
}
|
}
|
||||||
event.Platform, err = normalizeAppTrackingText(strings.ToLower(event.Platform), 32)
|
event.Platform, err = normalizeAppTrackingText(strings.ToLower(event.Platform), apptracking.MaxPlatformBytes)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return AppTrackingEvent{}, "", err
|
return AppTrackingEvent{}, "", err
|
||||||
}
|
}
|
||||||
event.AppVersion, err = normalizeAppTrackingText(event.AppVersion, 64)
|
event.AppVersion, err = normalizeAppTrackingText(event.AppVersion, apptracking.MaxAppVersionBytes)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return AppTrackingEvent{}, "", err
|
return AppTrackingEvent{}, "", err
|
||||||
}
|
}
|
||||||
event.Language, err = normalizeAppTrackingText(event.Language, 32)
|
event.Language, err = normalizeAppTrackingText(event.Language, apptracking.MaxLanguageBytes)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return AppTrackingEvent{}, "", err
|
return AppTrackingEvent{}, "", err
|
||||||
}
|
}
|
||||||
event.Timezone, err = normalizeAppTrackingText(event.Timezone, 64)
|
event.Timezone, err = normalizeAppTrackingText(event.Timezone, apptracking.MaxTimezoneBytes)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return AppTrackingEvent{}, "", err
|
return AppTrackingEvent{}, "", err
|
||||||
}
|
}
|
||||||
event.ErrorCode, err = normalizeAppTrackingText(event.ErrorCode, 64)
|
event.ErrorCode, err = normalizeAppTrackingText(event.ErrorCode, apptracking.MaxErrorCodeBytes)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return AppTrackingEvent{}, "", err
|
return AppTrackingEvent{}, "", err
|
||||||
}
|
}
|
||||||
@ -2402,21 +2400,21 @@ func normalizeAppTrackingEvent(ctx context.Context, event AppTrackingEvent, rece
|
|||||||
}
|
}
|
||||||
|
|
||||||
func normalizeAppTrackingText(value string, maxBytes int) (string, error) {
|
func normalizeAppTrackingText(value string, maxBytes int) (string, error) {
|
||||||
value = strings.TrimSpace(value)
|
value, ok := apptracking.NormalizeText(value, maxBytes)
|
||||||
if maxBytes > 0 && len(value) > maxBytes {
|
if !ok {
|
||||||
return "", xerr.New(xerr.InvalidArgument, "app tracking text field is too long")
|
return "", xerr.New(xerr.InvalidArgument, "app tracking text field is too long")
|
||||||
}
|
}
|
||||||
return value, nil
|
return value, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
func normalizeAppTrackingProperties(raw json.RawMessage) (string, error) {
|
func normalizeAppTrackingProperties(raw json.RawMessage) (string, error) {
|
||||||
trimmed := bytes.TrimSpace(raw)
|
trimmed, ok := apptracking.NormalizeProperties(raw)
|
||||||
if len(trimmed) == 0 || bytes.Equal(trimmed, []byte("null")) {
|
if !ok {
|
||||||
return "{}", nil
|
|
||||||
}
|
|
||||||
if len(trimmed) > appTrackingMaxPropertiesBytes || !json.Valid(trimmed) {
|
|
||||||
return "", xerr.New(xerr.InvalidArgument, "app tracking properties_json is invalid")
|
return "", xerr.New(xerr.InvalidArgument, "app tracking properties_json is invalid")
|
||||||
}
|
}
|
||||||
|
if len(trimmed) == 0 {
|
||||||
|
return "{}", nil
|
||||||
|
}
|
||||||
return string(trimmed), nil
|
return string(trimmed), nil
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|||||||
@ -50,6 +50,7 @@ cp_intimacy_leaderboard:
|
|||||||
key_prefix: "user:cp_intimacy_leaderboard"
|
key_prefix: "user:cp_intimacy_leaderboard"
|
||||||
outbox_worker:
|
outbox_worker:
|
||||||
enabled: true
|
enabled: true
|
||||||
|
app_codes: []
|
||||||
poll_interval: "1s"
|
poll_interval: "1s"
|
||||||
batch_size: 100
|
batch_size: 100
|
||||||
publish_timeout: "3s"
|
publish_timeout: "3s"
|
||||||
|
|||||||
@ -50,6 +50,8 @@ cp_intimacy_leaderboard:
|
|||||||
key_prefix: "user:cp_intimacy_leaderboard"
|
key_prefix: "user:cp_intimacy_leaderboard"
|
||||||
outbox_worker:
|
outbox_worker:
|
||||||
enabled: true
|
enabled: true
|
||||||
|
# 空值自动发现全部 App;历史积压回放时先设 ["lalu", "fami"],核对后再加入 "huwaa"。
|
||||||
|
app_codes: []
|
||||||
poll_interval: "1s"
|
poll_interval: "1s"
|
||||||
batch_size: 100
|
batch_size: 100
|
||||||
publish_timeout: "3s"
|
publish_timeout: "3s"
|
||||||
|
|||||||
@ -51,6 +51,7 @@ cp_intimacy_leaderboard:
|
|||||||
key_prefix: "user:cp_intimacy_leaderboard"
|
key_prefix: "user:cp_intimacy_leaderboard"
|
||||||
outbox_worker:
|
outbox_worker:
|
||||||
enabled: true
|
enabled: true
|
||||||
|
app_codes: []
|
||||||
poll_interval: "1s"
|
poll_interval: "1s"
|
||||||
batch_size: 100
|
batch_size: 100
|
||||||
publish_timeout: "3s"
|
publish_timeout: "3s"
|
||||||
|
|||||||
@ -20,6 +20,7 @@ import (
|
|||||||
"hyapp/pkg/healthhttp"
|
"hyapp/pkg/healthhttp"
|
||||||
"hyapp/pkg/idgen"
|
"hyapp/pkg/idgen"
|
||||||
"hyapp/pkg/logx"
|
"hyapp/pkg/logx"
|
||||||
|
"hyapp/pkg/outboxpartition"
|
||||||
"hyapp/pkg/rocketmqx"
|
"hyapp/pkg/rocketmqx"
|
||||||
"hyapp/pkg/roommq"
|
"hyapp/pkg/roommq"
|
||||||
serviceapp "hyapp/pkg/servicekit/app"
|
serviceapp "hyapp/pkg/servicekit/app"
|
||||||
@ -66,6 +67,8 @@ type App struct {
|
|||||||
mqConsumers []*rocketmqx.Consumer
|
mqConsumers []*rocketmqx.Consumer
|
||||||
// userOutboxProducer 把 user_outbox 事实发布到 RocketMQ,供统计等读模型消费。
|
// userOutboxProducer 把 user_outbox 事实发布到 RocketMQ,供统计等读模型消费。
|
||||||
userOutboxProducer *rocketmqx.Producer
|
userOutboxProducer *rocketmqx.Producer
|
||||||
|
// userOutboxPartitions 缓存真实 App 列表并提供跨 worker 的公平轮转游标。
|
||||||
|
userOutboxPartitions *outboxpartition.Partitions
|
||||||
// roomConn 是低频平台治理调用 room-service 的 gRPC 连接。
|
// roomConn 是低频平台治理调用 room-service 的 gRPC 连接。
|
||||||
roomConn *grpc.ClientConn
|
roomConn *grpc.ClientConn
|
||||||
// activityConn 是注册完成后触发注册奖励的 gRPC 连接。
|
// activityConn 是注册完成后触发注册奖励的 gRPC 连接。
|
||||||
@ -435,6 +438,7 @@ func New(cfg config.Config) (*App, error) {
|
|||||||
mysqlRepo: mysqlRepo,
|
mysqlRepo: mysqlRepo,
|
||||||
mqConsumers: mqConsumers,
|
mqConsumers: mqConsumers,
|
||||||
userOutboxProducer: userOutboxProducer,
|
userOutboxProducer: userOutboxProducer,
|
||||||
|
userOutboxPartitions: outboxpartition.New(cfg.OutboxWorker.AppCodes, outboxpartition.DefaultDiscoveryRefreshInterval),
|
||||||
roomConn: roomConn,
|
roomConn: roomConn,
|
||||||
activityConn: activityConn,
|
activityConn: activityConn,
|
||||||
walletConn: walletConn,
|
walletConn: walletConn,
|
||||||
@ -539,7 +543,18 @@ func (a *App) runUserOutboxWorker() {
|
|||||||
}
|
}
|
||||||
|
|
||||||
func (a *App) processUserOutboxBatch(workerID string) (int, error) {
|
func (a *App) processUserOutboxBatch(workerID string) (int, error) {
|
||||||
records, err := a.mysqlRepo.ClaimPendingUserOutbox(a.workers.Context(), workerID, a.cfg.OutboxWorker.BatchSize)
|
ctx := a.workers.Context()
|
||||||
|
appCodes, discoveryErr := a.userOutboxPartitions.Resolve(ctx, a.mysqlRepo.ListUserOutboxAppCodes)
|
||||||
|
if discoveryErr != nil {
|
||||||
|
if len(appCodes) == 0 {
|
||||||
|
return 0, discoveryErr
|
||||||
|
}
|
||||||
|
// 发现刷新失败时沿用旧列表;缓存会在五分钟后再试,不阻断既有 App 的事实发布。
|
||||||
|
logx.Warn(ctx, "user_outbox_app_discovery_stale", slog.String("worker_id", workerID), slog.String("error", discoveryErr.Error()))
|
||||||
|
}
|
||||||
|
records, err := outboxpartition.ClaimFair(ctx, a.userOutboxPartitions.Order(appCodes), a.cfg.OutboxWorker.BatchSize, func(claimCtx context.Context, appCode string, limit int) ([]mysqlstorage.UserOutboxRecord, error) {
|
||||||
|
return a.mysqlRepo.ClaimPendingUserOutbox(appcode.WithContext(claimCtx, appCode), workerID, limit)
|
||||||
|
})
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return 0, err
|
return 0, err
|
||||||
}
|
}
|
||||||
|
|||||||
@ -10,6 +10,7 @@ import (
|
|||||||
"hyapp/pkg/appcode"
|
"hyapp/pkg/appcode"
|
||||||
"hyapp/pkg/configx"
|
"hyapp/pkg/configx"
|
||||||
"hyapp/pkg/logx"
|
"hyapp/pkg/logx"
|
||||||
|
"hyapp/pkg/outboxpartition"
|
||||||
"hyapp/pkg/tencentim"
|
"hyapp/pkg/tencentim"
|
||||||
)
|
)
|
||||||
|
|
||||||
@ -215,7 +216,9 @@ type CPIntimacyLeaderboardConfig struct {
|
|||||||
|
|
||||||
// OutboxWorkerConfig 保存 user_outbox MQ 发布策略。
|
// OutboxWorkerConfig 保存 user_outbox MQ 发布策略。
|
||||||
type OutboxWorkerConfig struct {
|
type OutboxWorkerConfig struct {
|
||||||
Enabled bool `yaml:"enabled"`
|
Enabled bool `yaml:"enabled"`
|
||||||
|
// AppCodes 是可投递租户 allowlist;空值使用低频数据库发现,非空支持按 App 灰度回放。
|
||||||
|
AppCodes []string `yaml:"app_codes"`
|
||||||
PollInterval time.Duration `yaml:"poll_interval"`
|
PollInterval time.Duration `yaml:"poll_interval"`
|
||||||
BatchSize int `yaml:"batch_size"`
|
BatchSize int `yaml:"batch_size"`
|
||||||
PublishTimeout time.Duration `yaml:"publish_timeout"`
|
PublishTimeout time.Duration `yaml:"publish_timeout"`
|
||||||
@ -492,6 +495,7 @@ func Load(path string) (Config, error) {
|
|||||||
if cfg.OutboxWorker.PublishTimeout <= 0 {
|
if cfg.OutboxWorker.PublishTimeout <= 0 {
|
||||||
cfg.OutboxWorker.PublishTimeout = 3 * time.Second
|
cfg.OutboxWorker.PublishTimeout = 3 * time.Second
|
||||||
}
|
}
|
||||||
|
cfg.OutboxWorker.AppCodes = outboxpartition.Normalize(cfg.OutboxWorker.AppCodes, false)
|
||||||
if cfg.OutboxWorker.Enabled && !cfg.RocketMQ.UserOutbox.Enabled {
|
if cfg.OutboxWorker.Enabled && !cfg.RocketMQ.UserOutbox.Enabled {
|
||||||
return Config{}, errors.New("outbox_worker requires rocketmq.user_outbox.enabled")
|
return Config{}, errors.New("outbox_worker requires rocketmq.user_outbox.enabled")
|
||||||
}
|
}
|
||||||
|
|||||||
32
services/user-service/internal/config/outbox_worker_test.go
Normal file
32
services/user-service/internal/config/outbox_worker_test.go
Normal file
@ -0,0 +1,32 @@
|
|||||||
|
package config
|
||||||
|
|
||||||
|
import (
|
||||||
|
"os"
|
||||||
|
"path/filepath"
|
||||||
|
"strings"
|
||||||
|
"testing"
|
||||||
|
)
|
||||||
|
|
||||||
|
func TestOutboxWorkerAppCodesNormalizeForStagedReplay(t *testing.T) {
|
||||||
|
path := filepath.Join(t.TempDir(), "config.yaml")
|
||||||
|
if err := os.WriteFile(path, []byte("outbox_worker:\n app_codes: [' LALU ', fami, HUWAA, fami]\n"), 0o600); err != nil {
|
||||||
|
t.Fatalf("write config: %v", err)
|
||||||
|
}
|
||||||
|
cfg, err := Load(path)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("load staged outbox config: %v", err)
|
||||||
|
}
|
||||||
|
if got := strings.Join(cfg.OutboxWorker.AppCodes, ","); got != "lalu,fami,huwaa" {
|
||||||
|
t.Fatalf("normalized outbox app_codes = %q", got)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestLocalOutboxWorkerEmptyAppCodesUsesDiscovery(t *testing.T) {
|
||||||
|
cfg, err := Load("../../configs/config.yaml")
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("load local config: %v", err)
|
||||||
|
}
|
||||||
|
if len(cfg.OutboxWorker.AppCodes) != 0 {
|
||||||
|
t.Fatalf("local empty app_codes must retain automatic discovery: %v", cfg.OutboxWorker.AppCodes)
|
||||||
|
}
|
||||||
|
}
|
||||||
@ -25,6 +25,29 @@ type UserOutboxRecord struct {
|
|||||||
LockUntilMS int64
|
LockUntilMS int64
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// ListUserOutboxAppCodes enumerates real tenant keys from the owner table.
|
||||||
|
// PRIMARY begins with app_code, and the worker caches this discovery for five
|
||||||
|
// minutes; normal claims continue to use status/lock indexes within one App.
|
||||||
|
func (r *Repository) ListUserOutboxAppCodes(ctx context.Context) ([]string, error) {
|
||||||
|
if r == nil || r.db == nil {
|
||||||
|
return nil, xerr.New(xerr.Unavailable, "mysql repository is not configured")
|
||||||
|
}
|
||||||
|
rows, err := r.db.QueryContext(ctx, `SELECT DISTINCT app_code FROM user_outbox FORCE INDEX (PRIMARY) ORDER BY app_code`)
|
||||||
|
if err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
defer rows.Close()
|
||||||
|
appCodes := make([]string, 0, 4)
|
||||||
|
for rows.Next() {
|
||||||
|
var appCode string
|
||||||
|
if err := rows.Scan(&appCode); err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
appCodes = append(appCodes, appCode)
|
||||||
|
}
|
||||||
|
return appCodes, rows.Err()
|
||||||
|
}
|
||||||
|
|
||||||
// ClaimPendingUserOutbox locks pending user facts so one worker can publish them.
|
// ClaimPendingUserOutbox locks pending user facts so one worker can publish them.
|
||||||
func (r *Repository) ClaimPendingUserOutbox(ctx context.Context, workerID string, batchSize int) ([]UserOutboxRecord, error) {
|
func (r *Repository) ClaimPendingUserOutbox(ctx context.Context, workerID string, batchSize int) ([]UserOutboxRecord, error) {
|
||||||
if r == nil || r.db == nil {
|
if r == nil || r.db == nil {
|
||||||
|
|||||||
@ -0,0 +1,30 @@
|
|||||||
|
package mysql
|
||||||
|
|
||||||
|
import (
|
||||||
|
"context"
|
||||||
|
"regexp"
|
||||||
|
"testing"
|
||||||
|
|
||||||
|
"github.com/DATA-DOG/go-sqlmock"
|
||||||
|
)
|
||||||
|
|
||||||
|
func TestListUserOutboxAppCodesForcesPrimaryPrefixIndex(t *testing.T) {
|
||||||
|
db, mock, err := sqlmock.New()
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("create sqlmock: %v", err)
|
||||||
|
}
|
||||||
|
defer db.Close()
|
||||||
|
mock.ExpectQuery(regexp.QuoteMeta(`SELECT DISTINCT app_code FROM user_outbox FORCE INDEX (PRIMARY) ORDER BY app_code`)).
|
||||||
|
WillReturnRows(sqlmock.NewRows([]string{"app_code"}).AddRow("fami").AddRow("huwaa").AddRow("lalu"))
|
||||||
|
|
||||||
|
apps, err := (&Repository{db: db}).ListUserOutboxAppCodes(context.Background())
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("list user outbox apps: %v", err)
|
||||||
|
}
|
||||||
|
if len(apps) != 3 || apps[0] != "fami" || apps[1] != "huwaa" || apps[2] != "lalu" {
|
||||||
|
t.Fatalf("user outbox apps mismatch: %v", apps)
|
||||||
|
}
|
||||||
|
if err := mock.ExpectationsWereMet(); err != nil {
|
||||||
|
t.Fatalf("user app discovery query mismatch: %v", err)
|
||||||
|
}
|
||||||
|
}
|
||||||
@ -150,6 +150,7 @@ external_recharge:
|
|||||||
v5pay_return_url: "https://h5.global-interaction.com/recharge/index.html?env=test"
|
v5pay_return_url: "https://h5.global-interaction.com/recharge/index.html?env=test"
|
||||||
outbox_worker:
|
outbox_worker:
|
||||||
enabled: true
|
enabled: true
|
||||||
|
app_codes: []
|
||||||
poll_interval: "1s"
|
poll_interval: "1s"
|
||||||
batch_size: 200
|
batch_size: 200
|
||||||
concurrency: 4
|
concurrency: 4
|
||||||
@ -159,6 +160,7 @@ outbox_worker:
|
|||||||
max_backoff: "5m"
|
max_backoff: "5m"
|
||||||
realtime_outbox_worker:
|
realtime_outbox_worker:
|
||||||
enabled: true
|
enabled: true
|
||||||
|
app_codes: []
|
||||||
poll_interval: "500ms"
|
poll_interval: "500ms"
|
||||||
batch_size: 50
|
batch_size: 50
|
||||||
concurrency: 2
|
concurrency: 2
|
||||||
|
|||||||
@ -150,6 +150,8 @@ external_recharge:
|
|||||||
v5pay_return_url: "https://h5.global-interaction.com/recharge/index.html"
|
v5pay_return_url: "https://h5.global-interaction.com/recharge/index.html"
|
||||||
outbox_worker:
|
outbox_worker:
|
||||||
enabled: true
|
enabled: true
|
||||||
|
# 空值自动发现全部 App;历史积压回放时先设 ["lalu", "fami"],核对后再加入 "huwaa"。
|
||||||
|
app_codes: []
|
||||||
poll_interval: "1s"
|
poll_interval: "1s"
|
||||||
batch_size: 200
|
batch_size: 200
|
||||||
concurrency: 8
|
concurrency: 8
|
||||||
@ -159,6 +161,8 @@ outbox_worker:
|
|||||||
max_backoff: "5m"
|
max_backoff: "5m"
|
||||||
realtime_outbox_worker:
|
realtime_outbox_worker:
|
||||||
enabled: true
|
enabled: true
|
||||||
|
# 实时 lane 必须与普通 lane 使用同一阶段 allowlist,避免同表出现跨 App 抢占差异。
|
||||||
|
app_codes: []
|
||||||
poll_interval: "500ms"
|
poll_interval: "500ms"
|
||||||
batch_size: 50
|
batch_size: 50
|
||||||
concurrency: 4
|
concurrency: 4
|
||||||
|
|||||||
@ -153,6 +153,7 @@ external_recharge_reconcile_worker:
|
|||||||
batch_size: 50
|
batch_size: 50
|
||||||
outbox_worker:
|
outbox_worker:
|
||||||
enabled: true
|
enabled: true
|
||||||
|
app_codes: []
|
||||||
poll_interval: "1s"
|
poll_interval: "1s"
|
||||||
batch_size: 100
|
batch_size: 100
|
||||||
concurrency: 1
|
concurrency: 1
|
||||||
@ -162,6 +163,7 @@ outbox_worker:
|
|||||||
max_backoff: "5m"
|
max_backoff: "5m"
|
||||||
realtime_outbox_worker:
|
realtime_outbox_worker:
|
||||||
enabled: true
|
enabled: true
|
||||||
|
app_codes: []
|
||||||
poll_interval: "500ms"
|
poll_interval: "500ms"
|
||||||
batch_size: 50
|
batch_size: 50
|
||||||
concurrency: 2
|
concurrency: 2
|
||||||
|
|||||||
@ -13,6 +13,7 @@ import (
|
|||||||
"hyapp/pkg/grpchealth"
|
"hyapp/pkg/grpchealth"
|
||||||
"hyapp/pkg/healthhttp"
|
"hyapp/pkg/healthhttp"
|
||||||
"hyapp/pkg/logx"
|
"hyapp/pkg/logx"
|
||||||
|
"hyapp/pkg/outboxpartition"
|
||||||
"hyapp/pkg/rocketmqx"
|
"hyapp/pkg/rocketmqx"
|
||||||
servicegrpc "hyapp/pkg/servicekit/grpcserver"
|
servicegrpc "hyapp/pkg/servicekit/grpcserver"
|
||||||
servicehealth "hyapp/pkg/servicekit/health"
|
servicehealth "hyapp/pkg/servicekit/health"
|
||||||
@ -31,18 +32,21 @@ import (
|
|||||||
|
|
||||||
// App 装配 wallet-service gRPC 入口。
|
// App 装配 wallet-service gRPC 入口。
|
||||||
type App struct {
|
type App struct {
|
||||||
server *grpc.Server
|
server *grpc.Server
|
||||||
listener net.Listener
|
listener net.Listener
|
||||||
health *grpchealth.ServingChecker
|
health *grpchealth.ServingChecker
|
||||||
healthHTTP *healthhttp.Server
|
healthHTTP *healthhttp.Server
|
||||||
mysqlRepo *mysqlstorage.Repository
|
mysqlRepo *mysqlstorage.Repository
|
||||||
walletSvc *walletservice.Service
|
walletSvc *walletservice.Service
|
||||||
activityConn *grpc.ClientConn
|
activityConn *grpc.ClientConn
|
||||||
outboxProducer *rocketmqx.Producer
|
outboxProducer *rocketmqx.Producer
|
||||||
realtimeOutboxProducer *rocketmqx.Producer
|
realtimeOutboxProducer *rocketmqx.Producer
|
||||||
projectionConsumer *rocketmqx.Consumer
|
projectionConsumer *rocketmqx.Consumer
|
||||||
outboxWorkerCfg config.OutboxWorkerConfig
|
outboxWorkerCfg config.OutboxWorkerConfig
|
||||||
realtimeOutboxWorkerCfg config.OutboxWorkerConfig
|
realtimeOutboxWorkerCfg config.OutboxWorkerConfig
|
||||||
|
// 普通与实时 lane 共享同一张表但可独立配置事件类型;各自维护 App 轮转游标,避免一个 lane 的吞吐改变另一个 lane 的公平性。
|
||||||
|
outboxPartitions *outboxpartition.Partitions
|
||||||
|
realtimeOutboxPartitions *outboxpartition.Partitions
|
||||||
projectionWorkerCfg config.ProjectionWorkerConfig
|
projectionWorkerCfg config.ProjectionWorkerConfig
|
||||||
googlePaidSyncWorkerCfg config.GooglePaidSyncWorkerConfig
|
googlePaidSyncWorkerCfg config.GooglePaidSyncWorkerConfig
|
||||||
externalRechargeReconcileWorkerCfg config.ExternalRechargeReconcileWorkerConfig
|
externalRechargeReconcileWorkerCfg config.ExternalRechargeReconcileWorkerConfig
|
||||||
@ -204,6 +208,8 @@ func New(cfg config.Config) (*App, error) {
|
|||||||
projectionConsumer: projectionConsumer,
|
projectionConsumer: projectionConsumer,
|
||||||
outboxWorkerCfg: cfg.OutboxWorker,
|
outboxWorkerCfg: cfg.OutboxWorker,
|
||||||
realtimeOutboxWorkerCfg: cfg.RealtimeOutboxWorker,
|
realtimeOutboxWorkerCfg: cfg.RealtimeOutboxWorker,
|
||||||
|
outboxPartitions: outboxpartition.New(cfg.OutboxWorker.AppCodes, outboxpartition.DefaultDiscoveryRefreshInterval),
|
||||||
|
realtimeOutboxPartitions: outboxpartition.New(cfg.RealtimeOutboxWorker.AppCodes, outboxpartition.DefaultDiscoveryRefreshInterval),
|
||||||
projectionWorkerCfg: cfg.ProjectionWorker,
|
projectionWorkerCfg: cfg.ProjectionWorker,
|
||||||
googlePaidSyncWorkerCfg: googlePaidSyncWorkerCfg,
|
googlePaidSyncWorkerCfg: googlePaidSyncWorkerCfg,
|
||||||
externalRechargeReconcileWorkerCfg: cfg.ExternalRechargeReconcileWorker,
|
externalRechargeReconcileWorkerCfg: cfg.ExternalRechargeReconcileWorker,
|
||||||
|
|||||||
@ -14,10 +14,10 @@ func (a *App) runBackgroundWorkers() {
|
|||||||
// 普通账务 worker 在实时通道可用时跳过红包 UI 事件,避免两个 worker 抢同一行。
|
// 普通账务 worker 在实时通道可用时跳过红包 UI 事件,避免两个 worker 抢同一行。
|
||||||
excludedRealtimeTypes = a.realtimeOutboxWorkerCfg.EventTypes
|
excludedRealtimeTypes = a.realtimeOutboxWorkerCfg.EventTypes
|
||||||
}
|
}
|
||||||
a.startWalletOutboxWorkers(ctx, "wallet-outbox", a.outboxWorkerCfg, a.outboxProducer, a.walletOutboxTopic, nil, excludedRealtimeTypes)
|
a.startWalletOutboxWorkers(ctx, "wallet-outbox", a.outboxWorkerCfg, a.outboxPartitions, a.outboxProducer, a.walletOutboxTopic, nil, excludedRealtimeTypes)
|
||||||
}
|
}
|
||||||
if a.realtimeOutboxWorkerCfg.Enabled && a.realtimeOutboxProducer != nil {
|
if a.realtimeOutboxWorkerCfg.Enabled && a.realtimeOutboxProducer != nil {
|
||||||
a.startWalletOutboxWorkers(ctx, "wallet-realtime-outbox", a.realtimeOutboxWorkerCfg, a.realtimeOutboxProducer, a.realtimeWalletOutboxTopic, a.realtimeOutboxWorkerCfg.EventTypes, nil)
|
a.startWalletOutboxWorkers(ctx, "wallet-realtime-outbox", a.realtimeOutboxWorkerCfg, a.realtimeOutboxPartitions, a.realtimeOutboxProducer, a.realtimeWalletOutboxTopic, a.realtimeOutboxWorkerCfg.EventTypes, nil)
|
||||||
}
|
}
|
||||||
if a.externalRechargeReconcileWorkerCfg.Enabled {
|
if a.externalRechargeReconcileWorkerCfg.Enabled {
|
||||||
a.startExternalRechargeReconcileWorker(ctx)
|
a.startExternalRechargeReconcileWorker(ctx)
|
||||||
|
|||||||
@ -10,13 +10,14 @@ import (
|
|||||||
|
|
||||||
"hyapp/pkg/appcode"
|
"hyapp/pkg/appcode"
|
||||||
"hyapp/pkg/logx"
|
"hyapp/pkg/logx"
|
||||||
|
"hyapp/pkg/outboxpartition"
|
||||||
"hyapp/pkg/rocketmqx"
|
"hyapp/pkg/rocketmqx"
|
||||||
"hyapp/pkg/walletmq"
|
"hyapp/pkg/walletmq"
|
||||||
"hyapp/services/wallet-service/internal/config"
|
"hyapp/services/wallet-service/internal/config"
|
||||||
mysqlstorage "hyapp/services/wallet-service/internal/storage/mysql"
|
mysqlstorage "hyapp/services/wallet-service/internal/storage/mysql"
|
||||||
)
|
)
|
||||||
|
|
||||||
func (a *App) startWalletOutboxWorkers(ctx context.Context, workerName string, options config.OutboxWorkerConfig, producer *rocketmqx.Producer, topic string, includeEventTypes []string, excludeEventTypes []string) {
|
func (a *App) startWalletOutboxWorkers(ctx context.Context, workerName string, options config.OutboxWorkerConfig, partitions *outboxpartition.Partitions, producer *rocketmqx.Producer, topic string, includeEventTypes []string, excludeEventTypes []string) {
|
||||||
if options.Concurrency <= 0 {
|
if options.Concurrency <= 0 {
|
||||||
options.Concurrency = 1
|
options.Concurrency = 1
|
||||||
}
|
}
|
||||||
@ -25,16 +26,16 @@ func (a *App) startWalletOutboxWorkers(ctx context.Context, workerName string, o
|
|||||||
a.workers.Add(1)
|
a.workers.Add(1)
|
||||||
go func() {
|
go func() {
|
||||||
defer a.workers.Done()
|
defer a.workers.Done()
|
||||||
a.runWalletOutboxWorker(ctx, workerID, options, producer, topic, includeEventTypes, excludeEventTypes)
|
a.runWalletOutboxWorker(ctx, workerID, options, partitions, producer, topic, includeEventTypes, excludeEventTypes)
|
||||||
}()
|
}()
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
func (a *App) runWalletOutboxWorker(ctx context.Context, workerID string, options config.OutboxWorkerConfig, producer *rocketmqx.Producer, topic string, includeEventTypes []string, excludeEventTypes []string) {
|
func (a *App) runWalletOutboxWorker(ctx context.Context, workerID string, options config.OutboxWorkerConfig, partitions *outboxpartition.Partitions, producer *rocketmqx.Producer, topic string, includeEventTypes []string, excludeEventTypes []string) {
|
||||||
ticker := time.NewTicker(options.PollInterval)
|
ticker := time.NewTicker(options.PollInterval)
|
||||||
defer ticker.Stop()
|
defer ticker.Stop()
|
||||||
for {
|
for {
|
||||||
processed, err := a.processWalletOutboxBatch(ctx, workerID, options, producer, topic, includeEventTypes, excludeEventTypes)
|
processed, err := a.processWalletOutboxBatch(ctx, workerID, options, partitions, producer, topic, includeEventTypes, excludeEventTypes)
|
||||||
if err != nil && !errors.Is(err, context.Canceled) {
|
if err != nil && !errors.Is(err, context.Canceled) {
|
||||||
logx.Error(ctx, "wallet_outbox_publish_failed", err, slog.String("worker_id", workerID))
|
logx.Error(ctx, "wallet_outbox_publish_failed", err, slog.String("worker_id", workerID))
|
||||||
}
|
}
|
||||||
@ -49,18 +50,31 @@ func (a *App) runWalletOutboxWorker(ctx context.Context, workerID string, option
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
func (a *App) processWalletOutboxBatch(ctx context.Context, workerID string, options config.OutboxWorkerConfig, producer *rocketmqx.Producer, topic string, includeEventTypes []string, excludeEventTypes []string) (int, error) {
|
func (a *App) processWalletOutboxBatch(ctx context.Context, workerID string, options config.OutboxWorkerConfig, partitions *outboxpartition.Partitions, producer *rocketmqx.Producer, topic string, includeEventTypes []string, excludeEventTypes []string) (int, error) {
|
||||||
if a.mysqlRepo == nil || producer == nil {
|
if a.mysqlRepo == nil || producer == nil {
|
||||||
return 0, nil
|
return 0, nil
|
||||||
}
|
}
|
||||||
|
appCodes, discoveryErr := partitions.Resolve(ctx, a.mysqlRepo.ListWalletOutboxAppCodes)
|
||||||
|
if discoveryErr != nil {
|
||||||
|
if len(appCodes) == 0 {
|
||||||
|
return 0, discoveryErr
|
||||||
|
}
|
||||||
|
// 已知 App 继续投递,刷新失败只影响新租户发现;缓存层会延后下一次查询,避免热循环打 MySQL。
|
||||||
|
logx.Warn(ctx, "wallet_outbox_app_discovery_stale", slog.String("worker_id", workerID), slog.String("error", discoveryErr.Error()))
|
||||||
|
}
|
||||||
|
if len(appCodes) == 0 {
|
||||||
|
return 0, nil
|
||||||
|
}
|
||||||
processed := 0
|
processed := 0
|
||||||
for processed < options.BatchSize {
|
for processed < options.BatchSize {
|
||||||
// Sync publish is intentionally claimed one record at a time. Claiming an entire
|
// Sync publish is intentionally claimed one record at a time. Claiming an entire
|
||||||
// batch with one short lock lets later rows expire while earlier SendSync calls
|
// batch with one short lock lets later rows expire while earlier SendSync calls
|
||||||
// are still running, so another replica can publish the same tail concurrently.
|
// are still running, so another replica can publish the same tail concurrently.
|
||||||
records, err := a.mysqlRepo.ClaimPendingWalletOutboxFiltered(ctx, workerID, 1, time.Now().UTC().Add(walletOutboxSingleRecordLease(options.PublishTimeout)).UnixMilli(), mysqlstorage.WalletOutboxClaimFilter{
|
records, err := outboxpartition.ClaimFair(ctx, partitions.Order(appCodes), 1, func(claimCtx context.Context, appCode string, limit int) ([]mysqlstorage.WalletOutboxRecord, error) {
|
||||||
IncludeEventTypes: includeEventTypes,
|
return a.mysqlRepo.ClaimPendingWalletOutboxFiltered(appcode.WithContext(claimCtx, appCode), workerID, limit, time.Now().UTC().Add(walletOutboxSingleRecordLease(options.PublishTimeout)).UnixMilli(), mysqlstorage.WalletOutboxClaimFilter{
|
||||||
ExcludeEventTypes: excludeEventTypes,
|
IncludeEventTypes: includeEventTypes,
|
||||||
|
ExcludeEventTypes: excludeEventTypes,
|
||||||
|
})
|
||||||
})
|
})
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return processed, err
|
return processed, err
|
||||||
|
|||||||
@ -9,6 +9,7 @@ import (
|
|||||||
|
|
||||||
"hyapp/pkg/configx"
|
"hyapp/pkg/configx"
|
||||||
"hyapp/pkg/logx"
|
"hyapp/pkg/logx"
|
||||||
|
"hyapp/pkg/outboxpartition"
|
||||||
)
|
)
|
||||||
|
|
||||||
// Config 描述 wallet-service 启动配置。
|
// Config 描述 wallet-service 启动配置。
|
||||||
@ -102,7 +103,9 @@ type WalletOutboxMQConfig struct {
|
|||||||
|
|
||||||
// OutboxWorkerConfig 控制 wallet_outbox pending 事件的 MQ fanout。
|
// OutboxWorkerConfig 控制 wallet_outbox pending 事件的 MQ fanout。
|
||||||
type OutboxWorkerConfig struct {
|
type OutboxWorkerConfig struct {
|
||||||
Enabled bool `yaml:"enabled"`
|
Enabled bool `yaml:"enabled"`
|
||||||
|
// AppCodes 是可投递租户 allowlist;空值才自动发现,便于积压回放按 App 分阶段放量。
|
||||||
|
AppCodes []string `yaml:"app_codes"`
|
||||||
PollInterval time.Duration `yaml:"poll_interval"`
|
PollInterval time.Duration `yaml:"poll_interval"`
|
||||||
BatchSize int `yaml:"batch_size"`
|
BatchSize int `yaml:"batch_size"`
|
||||||
Concurrency int `yaml:"concurrency"`
|
Concurrency int `yaml:"concurrency"`
|
||||||
@ -820,6 +823,7 @@ func normalizeRocketMQConfig(cfg RocketMQConfig) (RocketMQConfig, error) {
|
|||||||
}
|
}
|
||||||
|
|
||||||
func normalizeOutboxWorkerConfig(cfg OutboxWorkerConfig, defaults OutboxWorkerConfig) OutboxWorkerConfig {
|
func normalizeOutboxWorkerConfig(cfg OutboxWorkerConfig, defaults OutboxWorkerConfig) OutboxWorkerConfig {
|
||||||
|
cfg.AppCodes = outboxpartition.Normalize(cfg.AppCodes, false)
|
||||||
if cfg.PollInterval <= 0 {
|
if cfg.PollInterval <= 0 {
|
||||||
cfg.PollInterval = defaults.PollInterval
|
cfg.PollInterval = defaults.PollInterval
|
||||||
}
|
}
|
||||||
|
|||||||
@ -31,11 +31,21 @@ func TestLoadLocalEnablesOutboxMQ(t *testing.T) {
|
|||||||
if cfg.OutboxWorker.Concurrency != 1 || cfg.RealtimeOutboxWorker.Concurrency != 2 {
|
if cfg.OutboxWorker.Concurrency != 1 || cfg.RealtimeOutboxWorker.Concurrency != 2 {
|
||||||
t.Fatalf("local outbox worker defaults mismatch: outbox=%+v realtime=%+v", cfg.OutboxWorker, cfg.RealtimeOutboxWorker)
|
t.Fatalf("local outbox worker defaults mismatch: outbox=%+v realtime=%+v", cfg.OutboxWorker, cfg.RealtimeOutboxWorker)
|
||||||
}
|
}
|
||||||
|
if len(cfg.OutboxWorker.AppCodes) != 0 || len(cfg.RealtimeOutboxWorker.AppCodes) != 0 {
|
||||||
|
t.Fatalf("empty app_codes must retain automatic discovery: outbox=%v realtime=%v", cfg.OutboxWorker.AppCodes, cfg.RealtimeOutboxWorker.AppCodes)
|
||||||
|
}
|
||||||
if cfg.BSCUSDT.Enabled || cfg.ExternalRecharge.USDTBEP20Enabled {
|
if cfg.BSCUSDT.Enabled || cfg.ExternalRecharge.USDTBEP20Enabled {
|
||||||
t.Fatalf("bsc usdt receipt verification must stay disabled by default: bsc=%+v external=%+v", cfg.BSCUSDT, cfg.ExternalRecharge)
|
t.Fatalf("bsc usdt receipt verification must stay disabled by default: bsc=%+v external=%+v", cfg.BSCUSDT, cfg.ExternalRecharge)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func TestOutboxWorkerAppCodesNormalizeForStagedReplay(t *testing.T) {
|
||||||
|
cfg := normalizeOutboxWorkerConfig(OutboxWorkerConfig{AppCodes: []string{" LALU ", "fami", "HUWAA", "fami"}}, Default().OutboxWorker)
|
||||||
|
if got := strings.Join(cfg.AppCodes, ","); got != "lalu,fami,huwaa" {
|
||||||
|
t.Fatalf("normalized outbox app_codes = %q", got)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
func TestDefaultWalletEventTagModeIsLegacy(t *testing.T) {
|
func TestDefaultWalletEventTagModeIsLegacy(t *testing.T) {
|
||||||
if got := Default().RocketMQ.WalletEventTagMode; got != WalletEventTagModeLegacy {
|
if got := Default().RocketMQ.WalletEventTagMode; got != WalletEventTagModeLegacy {
|
||||||
t.Fatalf("default wallet event tag mode = %q, want %q", got, WalletEventTagModeLegacy)
|
t.Fatalf("default wallet event tag mode = %q, want %q", got, WalletEventTagModeLegacy)
|
||||||
|
|||||||
@ -2385,6 +2385,45 @@ func TestCreditLuckyGiftRewardIsIdempotent(t *testing.T) {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// TestCreditWheelRewardReconciliationReturnsExistingTransaction 验证 Activity 补偿重复提交稳定 draw command 时,
|
||||||
|
// 钱包先按 command_id + request hash 找回既有交易和原回执,不会再次增加余额或重复生成 outbox。
|
||||||
|
func TestCreditWheelRewardReconciliationReturnsExistingTransaction(t *testing.T) {
|
||||||
|
repository := mysqltest.NewRepository(t)
|
||||||
|
svc := walletservice.New(repository)
|
||||||
|
command := ledger.WheelRewardCommand{
|
||||||
|
AppCode: "fami",
|
||||||
|
CommandID: "wheel_reward:wheel_draw_reconcile_1",
|
||||||
|
TargetUserID: 21102,
|
||||||
|
Amount: 288,
|
||||||
|
DrawID: "wheel_draw_reconcile_1",
|
||||||
|
WheelID: "classic",
|
||||||
|
SelectedTierID: "classic-coin-288",
|
||||||
|
VisibleRegionID: 10,
|
||||||
|
Reason: "wheel_reward",
|
||||||
|
}
|
||||||
|
|
||||||
|
first, err := svc.CreditWheelReward(context.Background(), command)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("CreditWheelReward failed: %v", err)
|
||||||
|
}
|
||||||
|
second, err := svc.CreditWheelReward(context.Background(), command)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("CreditWheelReward reconciliation retry failed: %v", err)
|
||||||
|
}
|
||||||
|
if first.TransactionID == "" || first.TransactionID != second.TransactionID || second.Balance.AvailableAmount != 288 {
|
||||||
|
t.Fatalf("wheel reward reconciliation receipt mismatch: first=%+v second=%+v", first, second)
|
||||||
|
}
|
||||||
|
if got := repository.CountRows("wallet_transactions", "app_code = ? AND command_id = ? AND biz_type = ?", "fami", command.CommandID, "wheel_reward"); got != 1 {
|
||||||
|
t.Fatalf("wheel reward reconciliation must keep one transaction, got %d", got)
|
||||||
|
}
|
||||||
|
if got := repository.CountRows("wallet_outbox", "app_code = ? AND transaction_id = ? AND event_type = ?", "fami", first.TransactionID, "WalletBalanceChanged"); got != 1 {
|
||||||
|
t.Fatalf("wheel reward reconciliation must keep one balance event, got %d", got)
|
||||||
|
}
|
||||||
|
if got := repository.CountRows("wallet_outbox", "app_code = ? AND transaction_id = ? AND event_type = ?", "fami", first.TransactionID, "WalletWheelRewardCredited"); got != 1 {
|
||||||
|
t.Fatalf("wheel reward reconciliation must keep one reward fact, got %d", got)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
// TestTransferCoinFromSellerMovesDedicatedAssetToPlayerCoin 验证币商专用金币和玩家普通金币不会混账。
|
// TestTransferCoinFromSellerMovesDedicatedAssetToPlayerCoin 验证币商专用金币和玩家普通金币不会混账。
|
||||||
func TestTransferCoinFromSellerMovesDedicatedAssetToPlayerCoin(t *testing.T) {
|
func TestTransferCoinFromSellerMovesDedicatedAssetToPlayerCoin(t *testing.T) {
|
||||||
repository := mysqltest.NewRepository(t)
|
repository := mysqltest.NewRepository(t)
|
||||||
|
|||||||
@ -84,6 +84,29 @@ func (r *Repository) insertWalletOutbox(ctx context.Context, tx *sql.Tx, events
|
|||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// ListWalletOutboxAppCodes enumerates the tenant key stored in this owner table.
|
||||||
|
// PRIMARY starts with app_code, so MySQL can use a loose ordered index scan;
|
||||||
|
// callers cache the result and never run this discovery query on every poll.
|
||||||
|
func (r *Repository) ListWalletOutboxAppCodes(ctx context.Context) ([]string, error) {
|
||||||
|
if r == nil || r.db == nil {
|
||||||
|
return nil, xerr.New(xerr.Unavailable, "mysql repository is not configured")
|
||||||
|
}
|
||||||
|
rows, err := r.db.QueryContext(ctx, `SELECT DISTINCT app_code FROM wallet_outbox FORCE INDEX (PRIMARY) ORDER BY app_code`)
|
||||||
|
if err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
defer rows.Close()
|
||||||
|
appCodes := make([]string, 0, 4)
|
||||||
|
for rows.Next() {
|
||||||
|
var appCode string
|
||||||
|
if err := rows.Scan(&appCode); err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
appCodes = append(appCodes, appCode)
|
||||||
|
}
|
||||||
|
return appCodes, rows.Err()
|
||||||
|
}
|
||||||
|
|
||||||
// ClaimPendingWalletOutbox atomically claims committed wallet facts for MQ fanout.
|
// ClaimPendingWalletOutbox atomically claims committed wallet facts for MQ fanout.
|
||||||
func (r *Repository) ClaimPendingWalletOutbox(ctx context.Context, workerID string, limit int, lockUntilMS int64) ([]WalletOutboxRecord, error) {
|
func (r *Repository) ClaimPendingWalletOutbox(ctx context.Context, workerID string, limit int, lockUntilMS int64) ([]WalletOutboxRecord, error) {
|
||||||
return r.ClaimPendingWalletOutboxFiltered(ctx, workerID, limit, lockUntilMS, WalletOutboxClaimFilter{})
|
return r.ClaimPendingWalletOutboxFiltered(ctx, workerID, limit, lockUntilMS, WalletOutboxClaimFilter{})
|
||||||
|
|||||||
@ -0,0 +1,30 @@
|
|||||||
|
package mysql
|
||||||
|
|
||||||
|
import (
|
||||||
|
"context"
|
||||||
|
"regexp"
|
||||||
|
"testing"
|
||||||
|
|
||||||
|
"github.com/DATA-DOG/go-sqlmock"
|
||||||
|
)
|
||||||
|
|
||||||
|
func TestListWalletOutboxAppCodesForcesPrimaryPrefixIndex(t *testing.T) {
|
||||||
|
db, mock, err := sqlmock.New()
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("create sqlmock: %v", err)
|
||||||
|
}
|
||||||
|
defer db.Close()
|
||||||
|
mock.ExpectQuery(regexp.QuoteMeta(`SELECT DISTINCT app_code FROM wallet_outbox FORCE INDEX (PRIMARY) ORDER BY app_code`)).
|
||||||
|
WillReturnRows(sqlmock.NewRows([]string{"app_code"}).AddRow("fami").AddRow("huwaa").AddRow("lalu"))
|
||||||
|
|
||||||
|
apps, err := (&Repository{db: db}).ListWalletOutboxAppCodes(context.Background())
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("list wallet outbox apps: %v", err)
|
||||||
|
}
|
||||||
|
if len(apps) != 3 || apps[0] != "fami" || apps[1] != "huwaa" || apps[2] != "lalu" {
|
||||||
|
t.Fatalf("wallet outbox apps mismatch: %v", apps)
|
||||||
|
}
|
||||||
|
if err := mock.ExpectationsWereMet(); err != nil {
|
||||||
|
t.Fatalf("wallet app discovery query mismatch: %v", err)
|
||||||
|
}
|
||||||
|
}
|
||||||
Loading…
x
Reference in New Issue
Block a user