fix: stabilize activity room event recovery
This commit is contained in:
zhx
parent
23376ab87f
commit
ba8907f90b
@ -4,6 +4,7 @@ import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"time"
|
||||
|
||||
"hyapp/pkg/appcode"
|
||||
@ -129,22 +130,24 @@ func newRoomOutboxConsumer(cfg config.Config, services *serviceBundle) (*rocketm
|
||||
// 任何模块失败都返回错误给 MQ，让同一个 envelope 重试，避免“播报成功但活动漏计分”的半完成状态。
|
||||
eventCtx := appcode.WithContext(ctx, envelope.GetAppCode())
|
||||
if _, err := services.broadcast.HandleRoomEvent(eventCtx, envelope); err != nil {
|
||||
return err
|
||||
return fmt.Errorf("room outbox broadcast projection: %w", err)
|
||||
}
|
||||
if _, err := services.growth.HandleRoomEvent(eventCtx, envelope); err != nil {
|
||||
return err
|
||||
return fmt.Errorf("room outbox growth projection: %w", err)
|
||||
}
|
||||
if _, err := services.weeklyStar.HandleRoomEvent(eventCtx, envelope); err != nil {
|
||||
return err
|
||||
return fmt.Errorf("room outbox weekly star projection: %w", err)
|
||||
}
|
||||
if _, err := services.agencyOpening.HandleRoomEvent(eventCtx, envelope); err != nil {
|
||||
return err
|
||||
return fmt.Errorf("room outbox agency opening projection: %w", err)
|
||||
}
|
||||
if _, err := services.task.HandleRoomEvent(eventCtx, envelope); err != nil {
|
||||
return err
|
||||
return fmt.Errorf("room outbox task projection: %w", err)
|
||||
}
|
||||
_, err = services.roomTurnoverReward.HandleRoomEvent(eventCtx, envelope)
|
||||
return err
|
||||
if _, err := services.roomTurnoverReward.HandleRoomEvent(eventCtx, envelope); err != nil {
|
||||
return fmt.Errorf("room outbox turnover reward projection: %w", err)
|
||||
}
|
||||
return nil
|
||||
}); err != nil {
|
||||
_ = consumer.Shutdown()
|
||||
return nil, err
|
||||
@ -342,7 +345,12 @@ func newTaskUserOutboxConsumer(cfg config.Config, services *serviceBundle) (*roc
|
||||
}
|
||||
|
||||
func roomOutboxConsumerConfig(cfg config.RocketMQConfig) rocketmqx.ConsumerConfig {
|
||||
return rocketMQConsumerConfig(cfg, cfg.RoomOutbox.ConsumerGroup, cfg.RoomOutbox.ConsumerMaxReconsumeTimes)
|
||||
consumerConfig := rocketMQConsumerConfig(cfg, cfg.RoomOutbox.ConsumerGroup, cfg.RoomOutbox.ConsumerMaxReconsumeTimes)
|
||||
// 一条房间事实会串行投影到多组 MySQL 幂等表和聚合表。SDK 默认开 20 个消费协程，
|
||||
// 积压恢复时同一用户的多条送礼事实会并发争抢任务进度和活动聚合锁，把可恢复积压放大成死锁重试风暴。
|
||||
// 单实例固定单协程；生产的同 consumer group 仍由多实例分担 queue，不改变消费位点和重试语义。
|
||||
consumerConfig.ConsumeGoroutines = 1
|
||||
return consumerConfig
|
||||
}
|
||||
|
||||
func userOutboxConsumerConfig(cfg config.RocketMQConfig) rocketmqx.ConsumerConfig {
|
||||
|
||||
20
services/activity-service/internal/app/mq_test.go
Normal file
20
services/activity-service/internal/app/mq_test.go
Normal file
@ -0,0 +1,20 @@
|
||||
package app
|
||||
|
||||
import (
|
||||
"testing"
|
||||
|
||||
"hyapp/services/activity-service/internal/config"
|
||||
)
|
||||
|
||||
func TestRoomOutboxConsumerConfigSerializesCompositeProjection(t *testing.T) {
|
||||
cfg := config.Default().RocketMQ
|
||||
cfg.RoomOutbox.ConsumerGroup = "activity-room-outbox-test"
|
||||
|
||||
consumerConfig := roomOutboxConsumerConfig(cfg)
|
||||
if consumerConfig.ConsumeGoroutines != 1 {
|
||||
t.Fatalf("room outbox consume goroutines = %d, want 1", consumerConfig.ConsumeGoroutines)
|
||||
}
|
||||
if consumerConfig.GroupName != cfg.RoomOutbox.ConsumerGroup {
|
||||
t.Fatalf("room outbox group = %q, want %q", consumerConfig.GroupName, cfg.RoomOutbox.ConsumerGroup)
|
||||
}
|
||||
}
|
||||
@ -88,6 +88,94 @@ func TestGrowthLevelEventRewardFlow(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
func TestGrowthLevelEventConcurrentSameUserTrack(t *testing.T) {
|
||||
svc, _ := newGrowthService(t)
|
||||
ctx := appcode.WithContext(context.Background(), "lalu")
|
||||
seedGrowthRules(t, ctx, svc, domain.TrackWealth)
|
||||
|
||||
// 先创建账户，确保并发波次命中生产故障里的“已有热点账户”路径，而不是只覆盖首次插入。
|
||||
if _, err := svc.ConsumeLevelEvent(ctx, growthValueEvent("growth-hot-account-seed", 41001, 1)); err != nil {
|
||||
t.Fatalf("seed hot growth account: %v", err)
|
||||
}
|
||||
|
||||
const writers = 40
|
||||
start := make(chan struct{})
|
||||
errs := make(chan error, writers)
|
||||
for index := 0; index < writers; index++ {
|
||||
go func(index int) {
|
||||
<-start
|
||||
_, err := svc.ConsumeLevelEvent(ctx, growthValueEvent(fmt.Sprintf("growth-hot-account-%d", index), 41001, 1))
|
||||
errs <- err
|
||||
}(index)
|
||||
}
|
||||
close(start)
|
||||
for index := 0; index < writers; index++ {
|
||||
if err := <-errs; err != nil {
|
||||
t.Fatalf("concurrent growth event %d failed: %v", index, err)
|
||||
}
|
||||
}
|
||||
|
||||
// 同一幂等键并发只能有一个事务增加累计值，其余事务必须在首个提交后稳定返回 duplicate。
|
||||
const duplicateWriters = 10
|
||||
duplicateStart := make(chan struct{})
|
||||
type consumeResult struct {
|
||||
status string
|
||||
err error
|
||||
}
|
||||
duplicateResults := make(chan consumeResult, duplicateWriters)
|
||||
for index := 0; index < duplicateWriters; index++ {
|
||||
go func() {
|
||||
<-duplicateStart
|
||||
result, err := svc.ConsumeLevelEvent(ctx, growthValueEvent("growth-hot-account-duplicate", 41001, 1))
|
||||
duplicateResults <- consumeResult{status: result.Status, err: err}
|
||||
}()
|
||||
}
|
||||
close(duplicateStart)
|
||||
consumed := 0
|
||||
duplicates := 0
|
||||
for index := 0; index < duplicateWriters; index++ {
|
||||
result := <-duplicateResults
|
||||
if result.err != nil {
|
||||
t.Fatalf("duplicate concurrent growth event %d failed: %v", index, result.err)
|
||||
}
|
||||
switch result.status {
|
||||
case domain.EventStatusConsumed:
|
||||
consumed++
|
||||
case domain.EventStatusDuplicate:
|
||||
duplicates++
|
||||
default:
|
||||
t.Fatalf("unexpected duplicate concurrent status %q", result.status)
|
||||
}
|
||||
}
|
||||
if consumed != 1 || duplicates != duplicateWriters-1 {
|
||||
t.Fatalf("duplicate concurrency mismatch: consumed=%d duplicates=%d", consumed, duplicates)
|
||||
}
|
||||
|
||||
overview, err := svc.GetMyLevelOverview(ctx, 41001)
|
||||
if err != nil {
|
||||
t.Fatalf("get concurrent growth overview: %v", err)
|
||||
}
|
||||
wealth := findTrackOverview(overview.Tracks, domain.TrackWealth)
|
||||
if want := int64(1 + writers + 1); wealth.TotalValue != want {
|
||||
t.Fatalf("concurrent total value=%d want=%d", wealth.TotalValue, want)
|
||||
}
|
||||
}
|
||||
|
||||
func growthValueEvent(eventID string, userID int64, delta int64) domain.ValueEvent {
|
||||
return domain.ValueEvent{
|
||||
EventID: eventID,
|
||||
SourceEventID: eventID,
|
||||
SourceService: "room-service",
|
||||
SourceEventType: "RoomGiftSent",
|
||||
UserID: userID,
|
||||
Track: domain.TrackWealth,
|
||||
MetricType: domain.MetricGiftSpendCoin,
|
||||
ValueDelta: delta,
|
||||
OccurredAtMS: fixedGrowthNow().UnixMilli(),
|
||||
DimensionsJSON: `{"room_id":"room-concurrency"}`,
|
||||
}
|
||||
}
|
||||
|
||||
func TestGrowthLevelRewardGrantsLevelResources(t *testing.T) {
|
||||
svc, wallet := newGrowthService(t)
|
||||
ctx := appcode.WithContext(context.Background(), "lalu")
|
||||
|
||||
@ -225,9 +225,12 @@ func (s *Service) HandleRoomEvent(ctx context.Context, envelope *roomeventsv1.Ev
|
||||
return 0, err
|
||||
}
|
||||
eventCtx := appcode.WithContext(ctx, envelope.GetAppCode())
|
||||
// 一次送礼同时满足“消耗金币”和“赠送次数”两个指标；用不同 event_id 后缀隔离幂等键，避免两个指标互相覆盖。
|
||||
events := []taskdomain.Event{
|
||||
{
|
||||
// 零价礼物是已提交的合法房间事实：它不能制造 Value=0 的金币任务事件，但仍应累计
|
||||
// 送礼次数。因此转换层只过滤恰好为 0 的 spend；负值仍交给 ConsumeTaskEvent 拒绝，
|
||||
// 避免这个兼容修复顺带吞掉上游坏数据。
|
||||
events := make([]taskdomain.Event, 0, 4)
|
||||
if coinSpent != 0 {
|
||||
events = append(events, taskdomain.Event{
|
||||
EventID: envelope.GetEventId() + ":task:gift_spend",
|
||||
EventType: envelope.GetEventType(),
|
||||
SourceService: "room-service",
|
||||
@ -236,22 +239,23 @@ func (s *Service) HandleRoomEvent(ctx context.Context, envelope *roomeventsv1.Ev
|
||||
Value: coinSpent,
|
||||
OccurredAtMS: envelope.GetOccurredAtMs(),
|
||||
DimensionsJSON: dimensions,
|
||||
},
|
||||
{
|
||||
EventID: envelope.GetEventId() + ":task:gift_count",
|
||||
EventType: envelope.GetEventType(),
|
||||
SourceService: "room-service",
|
||||
UserID: gift.GetSenderUserId(),
|
||||
MetricType: taskdomain.MetricGiftSendCount,
|
||||
Value: int64(gift.GetGiftCount()),
|
||||
OccurredAtMS: envelope.GetOccurredAtMs(),
|
||||
DimensionsJSON: dimensions,
|
||||
},
|
||||
})
|
||||
}
|
||||
// 次数指标来自已校验为正数的 gift_count，与礼物是否免费无关；独立 event_id 保证重放幂等。
|
||||
events = append(events, taskdomain.Event{
|
||||
EventID: envelope.GetEventId() + ":task:gift_count",
|
||||
EventType: envelope.GetEventType(),
|
||||
SourceService: "room-service",
|
||||
UserID: gift.GetSenderUserId(),
|
||||
MetricType: taskdomain.MetricGiftSendCount,
|
||||
Value: int64(gift.GetGiftCount()),
|
||||
OccurredAtMS: envelope.GetOccurredAtMs(),
|
||||
DimensionsJSON: dimensions,
|
||||
})
|
||||
// 幸运礼物是普通送礼的子集；普通礼物指标照常累计，额外再写幸运礼物专属指标，方便后台用不同任务口径配置。
|
||||
if isLuckyGiftTaskEvent(&gift) {
|
||||
events = append(events,
|
||||
taskdomain.Event{
|
||||
if coinSpent != 0 {
|
||||
events = append(events, taskdomain.Event{
|
||||
EventID: envelope.GetEventId() + ":task:lucky_gift_spend",
|
||||
EventType: envelope.GetEventType(),
|
||||
SourceService: "room-service",
|
||||
@ -260,18 +264,18 @@ func (s *Service) HandleRoomEvent(ctx context.Context, envelope *roomeventsv1.Ev
|
||||
Value: coinSpent,
|
||||
OccurredAtMS: envelope.GetOccurredAtMs(),
|
||||
DimensionsJSON: dimensions,
|
||||
},
|
||||
taskdomain.Event{
|
||||
EventID: envelope.GetEventId() + ":task:lucky_gift_count",
|
||||
EventType: envelope.GetEventType(),
|
||||
SourceService: "room-service",
|
||||
UserID: gift.GetSenderUserId(),
|
||||
MetricType: taskdomain.MetricLuckyGiftSendCount,
|
||||
Value: int64(gift.GetGiftCount()),
|
||||
OccurredAtMS: envelope.GetOccurredAtMs(),
|
||||
DimensionsJSON: dimensions,
|
||||
},
|
||||
)
|
||||
})
|
||||
}
|
||||
events = append(events, taskdomain.Event{
|
||||
EventID: envelope.GetEventId() + ":task:lucky_gift_count",
|
||||
EventType: envelope.GetEventType(),
|
||||
SourceService: "room-service",
|
||||
UserID: gift.GetSenderUserId(),
|
||||
MetricType: taskdomain.MetricLuckyGiftSendCount,
|
||||
Value: int64(gift.GetGiftCount()),
|
||||
OccurredAtMS: envelope.GetOccurredAtMs(),
|
||||
DimensionsJSON: dimensions,
|
||||
})
|
||||
}
|
||||
consumed := int32(0)
|
||||
for _, event := range events {
|
||||
|
||||
@ -6,13 +6,156 @@ import (
|
||||
"time"
|
||||
|
||||
"google.golang.org/grpc"
|
||||
"google.golang.org/protobuf/proto"
|
||||
roomeventsv1 "hyapp.local/api/proto/events/room/v1"
|
||||
walletv1 "hyapp.local/api/proto/wallet/v1"
|
||||
"hyapp/pkg/appcode"
|
||||
"hyapp/pkg/xerr"
|
||||
taskdomain "hyapp/services/activity-service/internal/domain/task"
|
||||
taskservice "hyapp/services/activity-service/internal/service/task"
|
||||
"hyapp/services/activity-service/internal/testutil/mysqltest"
|
||||
)
|
||||
|
||||
// TestHandleRoomEventKeepsCountForProductionZeroValueGift 回归生产 gift_id=459 的零价礼物。
|
||||
// 这类事实必须确认并累计送礼次数，不能生成会被领域层拒绝的零增量 spend 事件。
|
||||
func TestHandleRoomEventKeepsCountForProductionZeroValueGift(t *testing.T) {
|
||||
repository := newRecordingTaskRepository()
|
||||
svc := taskservice.New(repository, &fakeWalletClient{})
|
||||
envelope := roomGiftEnvelope(t, "evt-production-free-gift-459", &roomeventsv1.RoomGiftSent{
|
||||
SenderUserId: 10001,
|
||||
TargetUserId: 20002,
|
||||
GiftId: "459",
|
||||
GiftCount: 1,
|
||||
GiftValue: 0,
|
||||
CoinSpent: 0,
|
||||
GiftTypeCode: "normal",
|
||||
})
|
||||
|
||||
consumed, err := svc.HandleRoomEvent(context.Background(), envelope)
|
||||
if err != nil {
|
||||
t.Fatalf("HandleRoomEvent zero-value gift failed: %v", err)
|
||||
}
|
||||
if consumed != 1 {
|
||||
t.Fatalf("zero-value gift consumed=%d, want one count projection", consumed)
|
||||
}
|
||||
assertRecordedTaskEvents(t, repository.events, []expectedTaskEvent{{
|
||||
EventID: "evt-production-free-gift-459:task:gift_count",
|
||||
MetricType: taskdomain.MetricGiftSendCount,
|
||||
Value: 1,
|
||||
}})
|
||||
|
||||
// RocketMQ 重投会用相同 envelope event_id 再次进入；稳定后缀让仓储幂等确认，不重复累加。
|
||||
if _, err := svc.HandleRoomEvent(context.Background(), envelope); err != nil {
|
||||
t.Fatalf("HandleRoomEvent duplicate zero-value gift failed: %v", err)
|
||||
}
|
||||
if len(repository.events) != 1 || repository.calls != 2 {
|
||||
t.Fatalf("duplicate projection must keep one unique event: calls=%d events=%+v", repository.calls, repository.events)
|
||||
}
|
||||
}
|
||||
|
||||
func TestHandleRoomEventUsesPositiveSpendCompatibilityFields(t *testing.T) {
|
||||
tests := []struct {
|
||||
name string
|
||||
eventID string
|
||||
giftValue int64
|
||||
coinSpent int64
|
||||
wantSpend int64
|
||||
}{
|
||||
{name: "legacy gift_value fallback", eventID: "evt-legacy-gift-value", giftValue: 40, coinSpent: 0, wantSpend: 40},
|
||||
{name: "coin_spent preferred", eventID: "evt-coin-spent", giftValue: 0, coinSpent: 60, wantSpend: 60},
|
||||
}
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
repository := newRecordingTaskRepository()
|
||||
svc := taskservice.New(repository, &fakeWalletClient{})
|
||||
envelope := roomGiftEnvelope(t, tt.eventID, &roomeventsv1.RoomGiftSent{
|
||||
SenderUserId: 10001,
|
||||
TargetUserId: 20002,
|
||||
GiftId: "gift-compat",
|
||||
GiftCount: 3,
|
||||
GiftValue: tt.giftValue,
|
||||
CoinSpent: tt.coinSpent,
|
||||
GiftTypeCode: "normal",
|
||||
})
|
||||
|
||||
if _, err := svc.HandleRoomEvent(context.Background(), envelope); err != nil {
|
||||
t.Fatalf("HandleRoomEvent failed: %v", err)
|
||||
}
|
||||
assertRecordedTaskEvents(t, repository.events, []expectedTaskEvent{
|
||||
{EventID: envelope.GetEventId() + ":task:gift_spend", MetricType: taskdomain.MetricGiftSpendCoin, Value: tt.wantSpend},
|
||||
{EventID: envelope.GetEventId() + ":task:gift_count", MetricType: taskdomain.MetricGiftSendCount, Value: 3},
|
||||
})
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
// TestHandleRoomEventKeepsLuckyCountsForZeroValueGift 锁定幸运礼物的两层计数口径：
|
||||
// 免费礼物不累计金币，但仍同时属于普通送礼次数和幸运礼物次数。
|
||||
func TestHandleRoomEventKeepsLuckyCountsForZeroValueGift(t *testing.T) {
|
||||
repository := newRecordingTaskRepository()
|
||||
svc := taskservice.New(repository, &fakeWalletClient{})
|
||||
envelope := roomGiftEnvelope(t, "evt-free-lucky-gift", &roomeventsv1.RoomGiftSent{
|
||||
SenderUserId: 10001,
|
||||
TargetUserId: 20002,
|
||||
GiftId: "free-lucky",
|
||||
GiftCount: 5,
|
||||
GiftValue: 0,
|
||||
CoinSpent: 0,
|
||||
PoolId: "lucky_pool_free",
|
||||
GiftTypeCode: "normal",
|
||||
})
|
||||
|
||||
if _, err := svc.HandleRoomEvent(context.Background(), envelope); err != nil {
|
||||
t.Fatalf("HandleRoomEvent zero-value lucky gift failed: %v", err)
|
||||
}
|
||||
assertRecordedTaskEvents(t, repository.events, []expectedTaskEvent{
|
||||
{EventID: "evt-free-lucky-gift:task:gift_count", MetricType: taskdomain.MetricGiftSendCount, Value: 5},
|
||||
{EventID: "evt-free-lucky-gift:task:lucky_gift_count", MetricType: taskdomain.MetricLuckyGiftSendCount, Value: 5},
|
||||
})
|
||||
}
|
||||
|
||||
// TestConsumeTaskEventStillRejectsZeroValue 保留通用领域边界：零增量应在事实转换层被过滤，
|
||||
// 不能进入仓储的幂等表和进度表，否则会混淆“已消费”与“有有效进度”。
|
||||
func TestConsumeTaskEventStillRejectsZeroValue(t *testing.T) {
|
||||
repository := newRecordingTaskRepository()
|
||||
svc := taskservice.New(repository, &fakeWalletClient{})
|
||||
_, err := svc.ConsumeTaskEvent(context.Background(), taskdomain.Event{
|
||||
EventID: "evt-zero-domain-value",
|
||||
EventType: "RoomGiftSent",
|
||||
SourceService: "room-service",
|
||||
UserID: 10001,
|
||||
MetricType: taskdomain.MetricGiftSpendCoin,
|
||||
Value: 0,
|
||||
})
|
||||
if !xerr.IsCode(err, xerr.InvalidArgument) {
|
||||
t.Fatalf("zero task value error=%v, want INVALID_ARGUMENT", err)
|
||||
}
|
||||
if repository.calls != 0 || len(repository.events) != 0 {
|
||||
t.Fatalf("invalid zero-value event must not reach repository: calls=%d events=%+v", repository.calls, repository.events)
|
||||
}
|
||||
}
|
||||
|
||||
// TestHandleRoomEventStillRejectsNegativeSpend 确认兼容分支只确认合法零值，不把负值上游事实降级成计数事件。
|
||||
func TestHandleRoomEventStillRejectsNegativeSpend(t *testing.T) {
|
||||
repository := newRecordingTaskRepository()
|
||||
svc := taskservice.New(repository, &fakeWalletClient{})
|
||||
envelope := roomGiftEnvelope(t, "evt-negative-gift-value", &roomeventsv1.RoomGiftSent{
|
||||
SenderUserId: 10001,
|
||||
TargetUserId: 20002,
|
||||
GiftId: "broken-gift",
|
||||
GiftCount: 1,
|
||||
GiftValue: -1,
|
||||
CoinSpent: 0,
|
||||
})
|
||||
|
||||
if _, err := svc.HandleRoomEvent(context.Background(), envelope); !xerr.IsCode(err, xerr.InvalidArgument) {
|
||||
t.Fatalf("negative spend error=%v, want INVALID_ARGUMENT", err)
|
||||
}
|
||||
if repository.calls != 0 || len(repository.events) != 0 {
|
||||
t.Fatalf("negative spend must fail before repository: calls=%d events=%+v", repository.calls, repository.events)
|
||||
}
|
||||
}
|
||||
|
||||
// TestTaskEventQueryAndClaimFlow 覆盖任务定义、事件进度、App 查询、领奖和领奖幂等的主闭环。
|
||||
func TestTaskEventQueryAndClaimFlow(t *testing.T) {
|
||||
repository := mysqltest.NewRepository(t)
|
||||
@ -365,6 +508,64 @@ func (f *fakeTaskPolicyRepository) UpsertTaskDefinition(_ context.Context, comma
|
||||
}, true, nil
|
||||
}
|
||||
|
||||
type expectedTaskEvent struct {
|
||||
EventID string
|
||||
MetricType string
|
||||
Value int64
|
||||
}
|
||||
|
||||
// recordingTaskRepository 只实现 HandleRoomEvent 依赖的写入边界，并用 event_id 模拟 MySQL
|
||||
// task_event_consumption 的幂等行为；测试因此可以同时断言投影内容和 MQ 重投结果。
|
||||
type recordingTaskRepository struct {
|
||||
taskservice.Repository
|
||||
|
||||
calls int
|
||||
events []taskdomain.Event
|
||||
seen map[string]struct{}
|
||||
}
|
||||
|
||||
func newRecordingTaskRepository() *recordingTaskRepository {
|
||||
return &recordingTaskRepository{seen: make(map[string]struct{})}
|
||||
}
|
||||
|
||||
func (r *recordingTaskRepository) ConsumeTaskEvent(_ context.Context, event taskdomain.Event, _ string, _ int64) (taskdomain.EventResult, error) {
|
||||
r.calls++
|
||||
if _, exists := r.seen[event.EventID]; exists {
|
||||
return taskdomain.EventResult{EventID: event.EventID, Status: taskdomain.EventStatusConsumed}, nil
|
||||
}
|
||||
r.seen[event.EventID] = struct{}{}
|
||||
r.events = append(r.events, event)
|
||||
return taskdomain.EventResult{EventID: event.EventID, Status: taskdomain.EventStatusConsumed, MatchedTaskCount: 1}, nil
|
||||
}
|
||||
|
||||
func roomGiftEnvelope(t *testing.T, eventID string, gift *roomeventsv1.RoomGiftSent) *roomeventsv1.EventEnvelope {
|
||||
t.Helper()
|
||||
body, err := proto.Marshal(gift)
|
||||
if err != nil {
|
||||
t.Fatalf("marshal RoomGiftSent failed: %v", err)
|
||||
}
|
||||
return &roomeventsv1.EventEnvelope{
|
||||
AppCode: "lalu",
|
||||
EventId: eventID,
|
||||
RoomId: "lalu-regression-room",
|
||||
EventType: "RoomGiftSent",
|
||||
OccurredAtMs: time.Date(2026, 7, 9, 0, 0, 0, 0, time.UTC).UnixMilli(),
|
||||
Body: body,
|
||||
}
|
||||
}
|
||||
|
||||
func assertRecordedTaskEvents(t *testing.T, got []taskdomain.Event, want []expectedTaskEvent) {
|
||||
t.Helper()
|
||||
if len(got) != len(want) {
|
||||
t.Fatalf("projected event count=%d, want=%d: %+v", len(got), len(want), got)
|
||||
}
|
||||
for i := range want {
|
||||
if got[i].EventID != want[i].EventID || got[i].MetricType != want[i].MetricType || got[i].Value != want[i].Value {
|
||||
t.Fatalf("projected event[%d]=%+v, want=%+v", i, got[i], want[i])
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
type fakeWalletClient struct {
|
||||
calls int
|
||||
last *walletv1.CreditTaskRewardRequest
|
||||
|
||||
@ -6,16 +6,26 @@ import (
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"hash/fnv"
|
||||
"strconv"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
mysqldriver "github.com/go-sql-driver/mysql"
|
||||
"hyapp/pkg/appcode"
|
||||
"hyapp/pkg/idgen"
|
||||
"hyapp/pkg/xerr"
|
||||
domain "hyapp/services/activity-service/internal/domain/growth"
|
||||
)
|
||||
|
||||
const (
|
||||
// MySQL 会主动回滚死锁受害事务；重试必须从 BeginTx 开始重建完整事务，不能在旧 tx 上续跑。
|
||||
growthTransactionMaxAttempts = 5
|
||||
// 四次等待分别以 10/20/30/40ms 为基线，再加入 0-10ms 的稳定抖动，既限制请求预算也避免热点事件同步重撞。
|
||||
growthTransactionRetryStep = 10 * time.Millisecond
|
||||
growthTransactionRetryJitterWindow = 10 * time.Millisecond
|
||||
)
|
||||
|
||||
type queryer interface {
|
||||
QueryContext(ctx context.Context, query string, args ...any) (*sql.Rows, error)
|
||||
QueryRowContext(ctx context.Context, query string, args ...any) *sql.Row
|
||||
@ -187,6 +197,35 @@ func (r *Repository) ListLevelRewards(ctx context.Context, query domain.RewardQu
|
||||
|
||||
// ConsumeLevelEvent 在同一事务内完成事件幂等、累计值、升级历史、奖励任务和展示投影。
|
||||
func (r *Repository) ConsumeLevelEvent(ctx context.Context, event domain.ValueEvent, nowMS int64) (domain.EventResult, error) {
|
||||
var lastErr error
|
||||
for attempt := 0; attempt < growthTransactionMaxAttempts; attempt++ {
|
||||
result, err := r.consumeLevelEventOnce(ctx, event, nowMS)
|
||||
if err == nil {
|
||||
return result, nil
|
||||
}
|
||||
if !isRetryableGrowthTransactionError(err) {
|
||||
return domain.EventResult{}, err
|
||||
}
|
||||
lastErr = err
|
||||
if attempt == growthTransactionMaxAttempts-1 {
|
||||
break
|
||||
}
|
||||
// MQ 会并发回放同一热门用户的礼物事件；本地短退避先吸收瞬时锁冲突，避免每次 1213 都升级为 broker 秒级重试。
|
||||
timer := time.NewTimer(growthTransactionRetryDelay(event.EventID, attempt))
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
if !timer.Stop() {
|
||||
<-timer.C
|
||||
}
|
||||
return domain.EventResult{}, ctx.Err()
|
||||
case <-timer.C:
|
||||
}
|
||||
}
|
||||
return domain.EventResult{}, lastErr
|
||||
}
|
||||
|
||||
// consumeLevelEventOnce 只执行一次完整事务；调用方只会在 MySQL 已回滚的 1213/1205 上重新 BeginTx。
|
||||
func (r *Repository) consumeLevelEventOnce(ctx context.Context, event domain.ValueEvent, nowMS int64) (domain.EventResult, error) {
|
||||
if r == nil || r.db == nil {
|
||||
return domain.EventResult{}, xerr.New(xerr.Unavailable, "mysql repository is not configured")
|
||||
}
|
||||
@ -619,11 +658,15 @@ func (r *Repository) insertLevelValueEvent(ctx context.Context, tx *sql.Tx, even
|
||||
}
|
||||
|
||||
func (r *Repository) ensureLevelAccountForUpdate(ctx context.Context, tx *sql.Tx, userID int64, track string, nowMS int64) (domain.Account, error) {
|
||||
// INSERT IGNORE 命中已有主键时会先持有共享记录锁，随后 FOR UPDATE 再升级排他锁；并发礼物落到同一
|
||||
// (app_code,user_id,track) 时，多个事务会各持 S 锁并互等 X 锁。空更新 upsert 从第一条语句就取得 X 锁，
|
||||
// 让热点账户串行进入后续读改写，同时不改变账户累计值或更新时间。
|
||||
_, err := tx.ExecContext(ctx, `
|
||||
INSERT IGNORE INTO user_growth_level_accounts (
|
||||
INSERT INTO user_growth_level_accounts (
|
||||
app_code, user_id, track, total_value, current_level, current_tier_id,
|
||||
level_updated_at_ms, tier_updated_at_ms, version, updated_at_ms
|
||||
) VALUES (?, ?, ?, 0, 0, 0, 0, 0, 0, ?)`,
|
||||
) VALUES (?, ?, ?, 0, 0, 0, 0, 0, 0, ?)
|
||||
ON DUPLICATE KEY UPDATE user_id = VALUES(user_id)`,
|
||||
appcode.FromContext(ctx), userID, track, nowMS,
|
||||
)
|
||||
if err != nil {
|
||||
@ -640,6 +683,27 @@ func (r *Repository) ensureLevelAccountForUpdate(ctx context.Context, tx *sql.Tx
|
||||
return scanLevelAccount(row)
|
||||
}
|
||||
|
||||
func isRetryableGrowthTransactionError(err error) bool {
|
||||
var mysqlErr *mysqldriver.MySQLError
|
||||
if !errors.As(err, &mysqlErr) {
|
||||
return false
|
||||
}
|
||||
// 1213 是死锁受害事务，1205 是锁等待超时；两者都保证当前语句/事务没有成功提交，允许整事务重建。
|
||||
return mysqlErr.Number == 1213 || mysqlErr.Number == 1205
|
||||
}
|
||||
|
||||
func growthTransactionRetryDelay(eventID string, attempt int) time.Duration {
|
||||
if attempt < 0 {
|
||||
attempt = 0
|
||||
}
|
||||
hash := fnv.New32a()
|
||||
_, _ = hash.Write([]byte(eventID))
|
||||
_, _ = hash.Write([]byte{byte(attempt)})
|
||||
jitterSlots := uint32(growthTransactionRetryJitterWindow/time.Millisecond) + 1
|
||||
jitter := time.Duration(hash.Sum32()%jitterSlots) * time.Millisecond
|
||||
return time.Duration(attempt+1)*growthTransactionRetryStep + jitter
|
||||
}
|
||||
|
||||
func (r *Repository) updateLevelAccount(ctx context.Context, tx *sql.Tx, userID int64, track string, totalValue int64, level int32, tierID int64, levelUpdatedAtMS int64, tierUpdatedAtMS int64, nowMS int64) error {
|
||||
_, err := tx.ExecContext(ctx, `
|
||||
UPDATE user_growth_level_accounts
|
||||
|
||||
@ -0,0 +1,50 @@
|
||||
package mysql
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"fmt"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
mysqldriver "github.com/go-sql-driver/mysql"
|
||||
)
|
||||
|
||||
func TestIsRetryableGrowthTransactionError(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
tests := []struct {
|
||||
name string
|
||||
err error
|
||||
want bool
|
||||
}{
|
||||
{name: "deadlock", err: &mysqldriver.MySQLError{Number: 1213, Message: "deadlock"}, want: true},
|
||||
{name: "wrapped deadlock", err: fmt.Errorf("consume growth event: %w", &mysqldriver.MySQLError{Number: 1213, Message: "deadlock"}), want: true},
|
||||
{name: "lock wait timeout", err: &mysqldriver.MySQLError{Number: 1205, Message: "lock wait timeout"}, want: true},
|
||||
{name: "duplicate key", err: &mysqldriver.MySQLError{Number: 1062, Message: "duplicate"}, want: false},
|
||||
{name: "text only", err: errors.New("Error 1213: Deadlock found"), want: false},
|
||||
{name: "context canceled", err: context.Canceled, want: false},
|
||||
{name: "nil", err: nil, want: false},
|
||||
}
|
||||
for _, test := range tests {
|
||||
t.Run(test.name, func(t *testing.T) {
|
||||
t.Parallel()
|
||||
if got := isRetryableGrowthTransactionError(test.err); got != test.want {
|
||||
t.Fatalf("isRetryableGrowthTransactionError()=%v want=%v err=%v", got, test.want, test.err)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestGrowthTransactionRetryDelayIsBounded(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
for attempt := 0; attempt < growthTransactionMaxAttempts-1; attempt++ {
|
||||
delay := growthTransactionRetryDelay("growth-event-1", attempt)
|
||||
minimum := time.Duration(attempt+1) * growthTransactionRetryStep
|
||||
maximum := minimum + growthTransactionRetryJitterWindow
|
||||
if delay < minimum || delay > maximum {
|
||||
t.Fatalf("attempt %d delay=%s outside [%s,%s]", attempt, delay, minimum, maximum)
|
||||
}
|
||||
}
|
||||
}
|
||||
Loading…
x
Reference in New Issue
Block a user