diff --git a/services/game-service/deploy/mysql/initdb/001_game_service.sql b/services/game-service/deploy/mysql/initdb/001_game_service.sql index 934660af..39976f52 100644 --- a/services/game-service/deploy/mysql/initdb/001_game_service.sql +++ b/services/game-service/deploy/mysql/initdb/001_game_service.sql @@ -169,6 +169,7 @@ CREATE TABLE IF NOT EXISTS game_level_event_outbox ( UNIQUE KEY uk_game_level_event_order(app_code, order_id), KEY idx_game_level_event_pending(app_code, status, next_retry_at_ms, created_at_ms), KEY idx_game_level_event_claim(app_code, status, locked_until_ms, next_retry_at_ms, created_at_ms, event_id), + KEY idx_game_level_event_claim_order(app_code, status, locked_until_ms, created_at_ms, event_id, next_retry_at_ms), KEY idx_game_level_event_retention(app_code, status, updated_at_ms, event_id), KEY idx_game_level_event_user(app_code, user_id, created_at_ms) ) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_unicode_ci COMMENT='游戏等级事件 outbox 表'; diff --git a/services/game-service/internal/storage/mysql/repository.go b/services/game-service/internal/storage/mysql/repository.go index cc8ce4bb..b62cddba 100644 --- a/services/game-service/internal/storage/mysql/repository.go +++ b/services/game-service/internal/storage/mysql/repository.go @@ -216,6 +216,7 @@ func (r *Repository) Migrate(ctx context.Context) error { UNIQUE KEY uk_game_level_event_order(app_code, order_id), KEY idx_game_level_event_pending(app_code, status, next_retry_at_ms, created_at_ms), KEY idx_game_level_event_claim(app_code, status, locked_until_ms, next_retry_at_ms, created_at_ms, event_id), + KEY idx_game_level_event_claim_order(app_code, status, locked_until_ms, created_at_ms, event_id, next_retry_at_ms), KEY idx_game_level_event_retention(app_code, status, updated_at_ms, event_id), KEY idx_game_level_event_user(app_code, user_id, created_at_ms) ) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_unicode_ci`, @@ -286,6 +287,12 @@ func (r *Repository) Migrate(ctx context.Context) error { if err := r.ensureIndexDefinition(ctx, "game_outbox", "idx_game_outbox_retention", []string{"app_code", "status", "updated_at_ms", "event_id"}, "ALTER TABLE game_outbox ADD INDEX idx_game_outbox_retention(app_code, status, updated_at_ms, event_id)"); err != nil { return err } + if err := r.ensureIndexDefinition(ctx, "game_level_event_outbox", "idx_game_level_event_claim", []string{"app_code", "status", "locked_until_ms", "next_retry_at_ms", "created_at_ms", "event_id"}, "ALTER TABLE game_level_event_outbox ADD INDEX idx_game_level_event_claim(app_code, status, locked_until_ms, next_retry_at_ms, created_at_ms, event_id)"); err != nil { + return err + } + if err := r.ensureIndexDefinition(ctx, "game_level_event_outbox", "idx_game_level_event_claim_order", []string{"app_code", "status", "locked_until_ms", "created_at_ms", "event_id", "next_retry_at_ms"}, "ALTER TABLE game_level_event_outbox ADD INDEX idx_game_level_event_claim_order(app_code, status, locked_until_ms, created_at_ms, event_id, next_retry_at_ms)"); err != nil { + return err + } if err := r.ensureIndexDefinition(ctx, "game_level_event_outbox", "idx_game_level_event_retention", []string{"app_code", "status", "updated_at_ms", "event_id"}, "ALTER TABLE game_level_event_outbox ADD INDEX idx_game_level_event_retention(app_code, status, updated_at_ms, event_id)"); err != nil { return err } @@ -867,6 +874,9 @@ func (r *Repository) ClaimPendingLevelEvents(ctx context.Context, workerID strin if r == nil || r.db == nil { return nil, xerr.New(xerr.Unavailable, "mysql repository is not configured") } + if batchSize <= 0 { + return nil, nil + } tx, err := r.db.BeginTx(ctx, nil) if err != nil { return nil, err @@ -874,34 +884,32 @@ func (r *Repository) ClaimPendingLevelEvents(ctx context.Context, workerID strin defer func() { _ = tx.Rollback() }() appCode := appcode.FromContext(ctx) - events, err := r.queryPendingLevelEvents(ctx, tx, ` - WHERE app_code = ? AND status IN ('pending', 'failed') AND next_retry_at_ms <= ? - AND locked_until_ms = 0 - ORDER BY created_at_ms ASC, event_id ASC - LIMIT ? FOR UPDATE SKIP LOCKED`, - appCode, nowMS, batchSize) + // 先用轻量主键查询完成行锁 claim,避免 payload_json 和完整业务列参与排序。 + keys, err := r.queryUnlockedLevelEventClaimKeys(ctx, tx, appCode, nowMS, batchSize) if err != nil { return nil, err } - if remaining := batchSize - len(events); remaining > 0 { - retryEvents, err := r.queryPendingLevelEvents(ctx, tx, ` - WHERE app_code = ? AND status IN ('pending', 'failed') AND next_retry_at_ms <= ? - AND locked_until_ms > 0 AND locked_until_ms <= ? - ORDER BY created_at_ms ASC, event_id ASC - LIMIT ? FOR UPDATE SKIP LOCKED`, - appCode, nowMS, nowMS, remaining) + // 普通未锁事件优先处理;不足一批时再接管锁过期事件,保持原有补偿顺序。 + if remaining := batchSize - len(keys); remaining > 0 { + retryKeys, err := r.queryExpiredLevelEventClaimKeys(ctx, tx, appCode, nowMS, remaining) if err != nil { return nil, err } - events = append(events, retryEvents...) + keys = append(keys, retryKeys...) + } + // 状态更新前按主键回读完整事件,让上层仍拿到 pending/failed 的原始状态和负载。 + events, err := r.fetchLevelEventsByClaimKeys(ctx, tx, appCode, keys) + if err != nil { + return nil, err } lockUntil := nowMS + lockTTL.Milliseconds() - for _, event := range events { + // 只有被本事务锁住且准备返回的事件会切到 running,额外扫描到的候选行随提交释放锁。 + for _, key := range keys { if _, err := tx.ExecContext(ctx, ` UPDATE game_level_event_outbox SET status = 'running', attempt_count = attempt_count + 1, locked_by = ?, locked_until_ms = ?, updated_at_ms = ? WHERE app_code = ? AND event_id = ?`, - workerID, lockUntil, nowMS, appcode.FromContext(ctx), event.EventID, + workerID, lockUntil, nowMS, appCode, key.EventID, ); err != nil { return nil, err } @@ -912,13 +920,144 @@ func (r *Repository) ClaimPendingLevelEvents(ctx context.Context, workerID strin return events, nil } -func (r *Repository) queryPendingLevelEvents(ctx context.Context, tx *sql.Tx, whereSQL string, args ...any) ([]gamedomain.LevelEventOutbox, error) { - rows, err := tx.QueryContext(ctx, levelEventOutboxSelectSQL()+whereSQL, args...) +type levelEventClaimKey struct { + EventID string + CreatedAtMS int64 +} + +func (r *Repository) queryUnlockedLevelEventClaimKeys(ctx context.Context, tx *sql.Tx, appCode string, nowMS int64, batchSize int) ([]levelEventClaimKey, error) { + // status 拆成两个等值查询,避免 MySQL 把 status IN 展开成多段 range 后再 filesort。 + return queryLevelEventClaimKeysByStatus(batchSize, func(status string) ([]levelEventClaimKey, error) { + rows, err := tx.QueryContext(ctx, ` + SELECT event_id, created_at_ms + FROM game_level_event_outbox FORCE INDEX (idx_game_level_event_claim_order) + WHERE app_code = ? AND status = ? AND locked_until_ms = 0 AND next_retry_at_ms <= ? + ORDER BY created_at_ms ASC, event_id ASC + LIMIT ? FOR UPDATE SKIP LOCKED`, + appCode, status, nowMS, batchSize, + ) + if err != nil { + return nil, err + } + defer rows.Close() + return scanLevelEventClaimKeys(rows) + }) +} + +func (r *Repository) queryExpiredLevelEventClaimKeys(ctx context.Context, tx *sql.Tx, appCode string, nowMS int64, batchSize int) ([]levelEventClaimKey, error) { + // 过期锁接管仍使用 retry/lock 前缀缩小范围,但只锁主键,避免大 JSON 字段进入排序和慢日志。 + return queryLevelEventClaimKeysByStatus(batchSize, func(status string) ([]levelEventClaimKey, error) { + rows, err := tx.QueryContext(ctx, ` + SELECT event_id, created_at_ms + FROM game_level_event_outbox FORCE INDEX (idx_game_level_event_claim) + WHERE app_code = ? AND status = ? AND next_retry_at_ms <= ? + AND locked_until_ms > 0 AND locked_until_ms <= ? + ORDER BY created_at_ms ASC, event_id ASC + LIMIT ? FOR UPDATE SKIP LOCKED`, + appCode, status, nowMS, nowMS, batchSize, + ) + if err != nil { + return nil, err + } + defer rows.Close() + return scanLevelEventClaimKeys(rows) + }) +} + +func queryLevelEventClaimKeysByStatus(batchSize int, query func(status string) ([]levelEventClaimKey, error)) ([]levelEventClaimKey, error) { + statuses := []string{"pending", "failed"} + groups := make([][]levelEventClaimKey, 0, len(statuses)) + for _, status := range statuses { + keys, err := query(status) + if err != nil { + return nil, err + } + groups = append(groups, keys) + } + return mergeLevelEventClaimKeys(batchSize, groups...), nil +} + +func scanLevelEventClaimKeys(rows *sql.Rows) ([]levelEventClaimKey, error) { + keys := make([]levelEventClaimKey, 0) + for rows.Next() { + var key levelEventClaimKey + if err := rows.Scan(&key.EventID, &key.CreatedAtMS); err != nil { + return nil, err + } + keys = append(keys, key) + } + return keys, rows.Err() +} + +func mergeLevelEventClaimKeys(limit int, groups ...[]levelEventClaimKey) []levelEventClaimKey { + if limit <= 0 { + return nil + } + offsets := make([]int, len(groups)) + merged := make([]levelEventClaimKey, 0, limit) + // 每个 status 查询内部已经按 created_at/event_id 排序,这里做小规模归并来维持原 SQL 的跨状态顺序。 + for len(merged) < limit { + bestGroup := -1 + var best levelEventClaimKey + for groupIndex, group := range groups { + offset := offsets[groupIndex] + if offset >= len(group) { + continue + } + candidate := group[offset] + if bestGroup == -1 || levelEventClaimKeyLess(candidate, best) { + bestGroup = groupIndex + best = candidate + } + } + if bestGroup == -1 { + break + } + merged = append(merged, best) + offsets[bestGroup]++ + } + return merged +} + +func levelEventClaimKeyLess(left levelEventClaimKey, right levelEventClaimKey) bool { + if left.CreatedAtMS != right.CreatedAtMS { + return left.CreatedAtMS < right.CreatedAtMS + } + return left.EventID < right.EventID +} + +func (r *Repository) fetchLevelEventsByClaimKeys(ctx context.Context, tx *sql.Tx, appCode string, keys []levelEventClaimKey) ([]gamedomain.LevelEventOutbox, error) { + if len(keys) == 0 { + return nil, nil + } + placeholders := strings.TrimRight(strings.Repeat("?,", len(keys)), ",") + args := make([]any, 0, len(keys)+1) + args = append(args, appCode) + for _, key := range keys { + args = append(args, key.EventID) + } + rows, err := tx.QueryContext(ctx, levelEventOutboxSelectSQL()+`WHERE app_code = ? AND event_id IN (`+placeholders+`)`, args...) if err != nil { return nil, err } defer rows.Close() - return scanLevelEventOutboxRows(rows) + items, err := scanLevelEventOutboxRows(rows) + if err != nil { + return nil, err + } + byID := make(map[string]gamedomain.LevelEventOutbox, len(items)) + for _, item := range items { + byID[item.EventID] = item + } + ordered := make([]gamedomain.LevelEventOutbox, 0, len(keys)) + for _, key := range keys { + item, ok := byID[key.EventID] + if !ok { + return nil, errors.New("claimed level event disappeared before fetch") + } + ordered = append(ordered, item) + } + return ordered, nil } func (r *Repository) MarkLevelEventDelivered(ctx context.Context, eventID string, nowMS int64) error { diff --git a/services/game-service/internal/storage/mysql/repository_test.go b/services/game-service/internal/storage/mysql/repository_test.go index c8923379..80f812b0 100644 --- a/services/game-service/internal/storage/mysql/repository_test.go +++ b/services/game-service/internal/storage/mysql/repository_test.go @@ -2,6 +2,7 @@ package mysql import ( "context" + "strings" "testing" "time" @@ -63,6 +64,108 @@ func TestMarkOrderSucceededCreatesClaimableLevelOutboxEvent(t *testing.T) { } } +func TestClaimPendingLevelEventsPreservesCreatedOrderAcrossStatuses(t *testing.T) { + caller := mysqlschema.CallerFile(t, 1) + schema := mysqlschema.New(t, mysqlschema.Config{ + InitDBPath: mysqlschema.InitDBPath(t, caller, "../../../deploy/mysql/initdb/001_game_service.sql"), + DatabasePrefix: "hy_game_test", + }) + + ctx := appcode.WithContext(context.Background(), "lalu") + repo, err := Open(ctx, schema.DSN) + if err != nil { + t.Fatalf("open repository failed: %v", err) + } + t.Cleanup(func() { _ = repo.Close() }) + + insertLevelEvent := func(eventID string, orderID string, status string, nextRetryAtMS int64, createdAtMS int64, payload string) { + t.Helper() + if _, err := repo.db.ExecContext(ctx, ` + INSERT INTO game_level_event_outbox ( + app_code, event_id, order_id, user_id, game_id, provider_order_id, + provider_round_id, coin_amount, wallet_transaction_id, payload_json, + status, attempt_count, next_retry_at_ms, locked_by, locked_until_ms, + failure_reason, occurred_at_ms, created_at_ms, updated_at_ms + ) VALUES (?, ?, ?, 42, 'game_1', ?, '', 100, 'wallet_tx_1', ?, + ?, 0, ?, '', 0, '', ?, ?, ?)`, + "lalu", eventID, orderID, "provider_"+orderID, payload, status, nextRetryAtMS, createdAtMS, createdAtMS, createdAtMS, + ); err != nil { + t.Fatalf("insert level event %s failed: %v", eventID, err) + } + } + insertLevelEvent("event_failed_old", "order_failed_old", "failed", 1000, 1000, `{"source":"failed"}`) + insertLevelEvent("event_failed_future", "order_failed_future", "failed", 999999, 1500, `{"source":"future"}`) + insertLevelEvent("event_pending_new", "order_pending_new", "pending", 0, 2000, `{"source":"pending"}`) + + events, err := repo.ClaimPendingLevelEvents(ctx, "worker_1", 3000, 30*time.Second, 2) + if err != nil { + t.Fatalf("claim level events failed: %v", err) + } + if len(events) != 2 { + t.Fatalf("expected two claimed events, got %+v", events) + } + if events[0].EventID != "event_failed_old" || events[0].Status != "failed" || !strings.Contains(events[0].PayloadJSON, `"source": "failed"`) { + t.Fatalf("first claimed event should be the older failed payload, got %+v", events[0]) + } + if events[1].EventID != "event_pending_new" || events[1].Status != "pending" || !strings.Contains(events[1].PayloadJSON, `"source": "pending"`) { + t.Fatalf("second claimed event should be the pending payload, got %+v", events[1]) + } + + var runningCount int + if err := repo.db.QueryRowContext(ctx, ` + SELECT COUNT(*) FROM game_level_event_outbox + WHERE app_code = 'lalu' AND status = 'running' AND locked_by = 'worker_1' AND locked_until_ms = 33000`, + ).Scan(&runningCount); err != nil { + t.Fatalf("query running events failed: %v", err) + } + if runningCount != 2 { + t.Fatalf("expected two running claimed events, got %d", runningCount) + } + var futureStatus string + if err := repo.db.QueryRowContext(ctx, ` + SELECT status FROM game_level_event_outbox WHERE app_code = 'lalu' AND event_id = 'event_failed_future'`, + ).Scan(&futureStatus); err != nil { + t.Fatalf("query future retry event failed: %v", err) + } + if futureStatus != "failed" { + t.Fatalf("future retry event should stay failed, got %s", futureStatus) + } +} + +func TestLevelEventClaimKeyQueryUsesOrderIndexWithoutFilesort(t *testing.T) { + caller := mysqlschema.CallerFile(t, 1) + schema := mysqlschema.New(t, mysqlschema.Config{ + InitDBPath: mysqlschema.InitDBPath(t, caller, "../../../deploy/mysql/initdb/001_game_service.sql"), + DatabasePrefix: "hy_game_test", + }) + + ctx := appcode.WithContext(context.Background(), "lalu") + repo, err := Open(ctx, schema.DSN) + if err != nil { + t.Fatalf("open repository failed: %v", err) + } + t.Cleanup(func() { _ = repo.Close() }) + + var plan string + if err := repo.db.QueryRowContext(ctx, ` + EXPLAIN FORMAT=JSON + SELECT event_id, created_at_ms + FROM game_level_event_outbox FORCE INDEX (idx_game_level_event_claim_order) + WHERE app_code = ? AND status = ? AND locked_until_ms = 0 AND next_retry_at_ms <= ? + ORDER BY created_at_ms ASC, event_id ASC + LIMIT 100`, + "lalu", "pending", int64(1780542672724), + ).Scan(&plan); err != nil { + t.Fatalf("explain level event claim key query failed: %v", err) + } + if !strings.Contains(plan, `"key": "idx_game_level_event_claim_order"`) { + t.Fatalf("claim key query did not use idx_game_level_event_claim_order:\n%s", plan) + } + if strings.Contains(strings.ToLower(plan), `"using_filesort": true`) { + t.Fatalf("claim key query should avoid filesort:\n%s", plan) + } +} + func TestUpsertCatalogCreatesDefaultVoiceRoomDisplayRule(t *testing.T) { caller := mysqlschema.CallerFile(t, 1) schema := mysqlschema.New(t, mysqlschema.Config{