fix: optimize game level outbox claiming
This commit is contained in:
zhx
parent
f33e965caf
commit
155b159c83
@ -169,6 +169,7 @@ CREATE TABLE IF NOT EXISTS game_level_event_outbox (
|
||||
UNIQUE KEY uk_game_level_event_order(app_code, order_id),
|
||||
KEY idx_game_level_event_pending(app_code, status, next_retry_at_ms, created_at_ms),
|
||||
KEY idx_game_level_event_claim(app_code, status, locked_until_ms, next_retry_at_ms, created_at_ms, event_id),
|
||||
KEY idx_game_level_event_claim_order(app_code, status, locked_until_ms, created_at_ms, event_id, next_retry_at_ms),
|
||||
KEY idx_game_level_event_retention(app_code, status, updated_at_ms, event_id),
|
||||
KEY idx_game_level_event_user(app_code, user_id, created_at_ms)
|
||||
) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_unicode_ci COMMENT='游戏等级事件 outbox 表';
|
||||
|
||||
@ -216,6 +216,7 @@ func (r *Repository) Migrate(ctx context.Context) error {
|
||||
UNIQUE KEY uk_game_level_event_order(app_code, order_id),
|
||||
KEY idx_game_level_event_pending(app_code, status, next_retry_at_ms, created_at_ms),
|
||||
KEY idx_game_level_event_claim(app_code, status, locked_until_ms, next_retry_at_ms, created_at_ms, event_id),
|
||||
KEY idx_game_level_event_claim_order(app_code, status, locked_until_ms, created_at_ms, event_id, next_retry_at_ms),
|
||||
KEY idx_game_level_event_retention(app_code, status, updated_at_ms, event_id),
|
||||
KEY idx_game_level_event_user(app_code, user_id, created_at_ms)
|
||||
) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_unicode_ci`,
|
||||
@ -286,6 +287,12 @@ func (r *Repository) Migrate(ctx context.Context) error {
|
||||
if err := r.ensureIndexDefinition(ctx, "game_outbox", "idx_game_outbox_retention", []string{"app_code", "status", "updated_at_ms", "event_id"}, "ALTER TABLE game_outbox ADD INDEX idx_game_outbox_retention(app_code, status, updated_at_ms, event_id)"); err != nil {
|
||||
return err
|
||||
}
|
||||
if err := r.ensureIndexDefinition(ctx, "game_level_event_outbox", "idx_game_level_event_claim", []string{"app_code", "status", "locked_until_ms", "next_retry_at_ms", "created_at_ms", "event_id"}, "ALTER TABLE game_level_event_outbox ADD INDEX idx_game_level_event_claim(app_code, status, locked_until_ms, next_retry_at_ms, created_at_ms, event_id)"); err != nil {
|
||||
return err
|
||||
}
|
||||
if err := r.ensureIndexDefinition(ctx, "game_level_event_outbox", "idx_game_level_event_claim_order", []string{"app_code", "status", "locked_until_ms", "created_at_ms", "event_id", "next_retry_at_ms"}, "ALTER TABLE game_level_event_outbox ADD INDEX idx_game_level_event_claim_order(app_code, status, locked_until_ms, created_at_ms, event_id, next_retry_at_ms)"); err != nil {
|
||||
return err
|
||||
}
|
||||
if err := r.ensureIndexDefinition(ctx, "game_level_event_outbox", "idx_game_level_event_retention", []string{"app_code", "status", "updated_at_ms", "event_id"}, "ALTER TABLE game_level_event_outbox ADD INDEX idx_game_level_event_retention(app_code, status, updated_at_ms, event_id)"); err != nil {
|
||||
return err
|
||||
}
|
||||
@ -867,6 +874,9 @@ func (r *Repository) ClaimPendingLevelEvents(ctx context.Context, workerID strin
|
||||
if r == nil || r.db == nil {
|
||||
return nil, xerr.New(xerr.Unavailable, "mysql repository is not configured")
|
||||
}
|
||||
if batchSize <= 0 {
|
||||
return nil, nil
|
||||
}
|
||||
tx, err := r.db.BeginTx(ctx, nil)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
@ -874,34 +884,32 @@ func (r *Repository) ClaimPendingLevelEvents(ctx context.Context, workerID strin
|
||||
defer func() { _ = tx.Rollback() }()
|
||||
|
||||
appCode := appcode.FromContext(ctx)
|
||||
events, err := r.queryPendingLevelEvents(ctx, tx, `
|
||||
WHERE app_code = ? AND status IN ('pending', 'failed') AND next_retry_at_ms <= ?
|
||||
AND locked_until_ms = 0
|
||||
ORDER BY created_at_ms ASC, event_id ASC
|
||||
LIMIT ? FOR UPDATE SKIP LOCKED`,
|
||||
appCode, nowMS, batchSize)
|
||||
// 先用轻量主键查询完成行锁 claim，避免 payload_json 和完整业务列参与排序。
|
||||
keys, err := r.queryUnlockedLevelEventClaimKeys(ctx, tx, appCode, nowMS, batchSize)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if remaining := batchSize - len(events); remaining > 0 {
|
||||
retryEvents, err := r.queryPendingLevelEvents(ctx, tx, `
|
||||
WHERE app_code = ? AND status IN ('pending', 'failed') AND next_retry_at_ms <= ?
|
||||
AND locked_until_ms > 0 AND locked_until_ms <= ?
|
||||
ORDER BY created_at_ms ASC, event_id ASC
|
||||
LIMIT ? FOR UPDATE SKIP LOCKED`,
|
||||
appCode, nowMS, nowMS, remaining)
|
||||
// 普通未锁事件优先处理；不足一批时再接管锁过期事件，保持原有补偿顺序。
|
||||
if remaining := batchSize - len(keys); remaining > 0 {
|
||||
retryKeys, err := r.queryExpiredLevelEventClaimKeys(ctx, tx, appCode, nowMS, remaining)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
events = append(events, retryEvents...)
|
||||
keys = append(keys, retryKeys...)
|
||||
}
|
||||
// 状态更新前按主键回读完整事件，让上层仍拿到 pending/failed 的原始状态和负载。
|
||||
events, err := r.fetchLevelEventsByClaimKeys(ctx, tx, appCode, keys)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
lockUntil := nowMS + lockTTL.Milliseconds()
|
||||
for _, event := range events {
|
||||
// 只有被本事务锁住且准备返回的事件会切到 running，额外扫描到的候选行随提交释放锁。
|
||||
for _, key := range keys {
|
||||
if _, err := tx.ExecContext(ctx, `
|
||||
UPDATE game_level_event_outbox
|
||||
SET status = 'running', attempt_count = attempt_count + 1, locked_by = ?, locked_until_ms = ?, updated_at_ms = ?
|
||||
WHERE app_code = ? AND event_id = ?`,
|
||||
workerID, lockUntil, nowMS, appcode.FromContext(ctx), event.EventID,
|
||||
workerID, lockUntil, nowMS, appCode, key.EventID,
|
||||
); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
@ -912,13 +920,144 @@ func (r *Repository) ClaimPendingLevelEvents(ctx context.Context, workerID strin
|
||||
return events, nil
|
||||
}
|
||||
|
||||
func (r *Repository) queryPendingLevelEvents(ctx context.Context, tx *sql.Tx, whereSQL string, args ...any) ([]gamedomain.LevelEventOutbox, error) {
|
||||
rows, err := tx.QueryContext(ctx, levelEventOutboxSelectSQL()+whereSQL, args...)
|
||||
type levelEventClaimKey struct {
|
||||
EventID string
|
||||
CreatedAtMS int64
|
||||
}
|
||||
|
||||
func (r *Repository) queryUnlockedLevelEventClaimKeys(ctx context.Context, tx *sql.Tx, appCode string, nowMS int64, batchSize int) ([]levelEventClaimKey, error) {
|
||||
// status 拆成两个等值查询，避免 MySQL 把 status IN 展开成多段 range 后再 filesort。
|
||||
return queryLevelEventClaimKeysByStatus(batchSize, func(status string) ([]levelEventClaimKey, error) {
|
||||
rows, err := tx.QueryContext(ctx, `
|
||||
SELECT event_id, created_at_ms
|
||||
FROM game_level_event_outbox FORCE INDEX (idx_game_level_event_claim_order)
|
||||
WHERE app_code = ? AND status = ? AND locked_until_ms = 0 AND next_retry_at_ms <= ?
|
||||
ORDER BY created_at_ms ASC, event_id ASC
|
||||
LIMIT ? FOR UPDATE SKIP LOCKED`,
|
||||
appCode, status, nowMS, batchSize,
|
||||
)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
defer rows.Close()
|
||||
return scanLevelEventClaimKeys(rows)
|
||||
})
|
||||
}
|
||||
|
||||
func (r *Repository) queryExpiredLevelEventClaimKeys(ctx context.Context, tx *sql.Tx, appCode string, nowMS int64, batchSize int) ([]levelEventClaimKey, error) {
|
||||
// 过期锁接管仍使用 retry/lock 前缀缩小范围，但只锁主键，避免大 JSON 字段进入排序和慢日志。
|
||||
return queryLevelEventClaimKeysByStatus(batchSize, func(status string) ([]levelEventClaimKey, error) {
|
||||
rows, err := tx.QueryContext(ctx, `
|
||||
SELECT event_id, created_at_ms
|
||||
FROM game_level_event_outbox FORCE INDEX (idx_game_level_event_claim)
|
||||
WHERE app_code = ? AND status = ? AND next_retry_at_ms <= ?
|
||||
AND locked_until_ms > 0 AND locked_until_ms <= ?
|
||||
ORDER BY created_at_ms ASC, event_id ASC
|
||||
LIMIT ? FOR UPDATE SKIP LOCKED`,
|
||||
appCode, status, nowMS, nowMS, batchSize,
|
||||
)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
defer rows.Close()
|
||||
return scanLevelEventClaimKeys(rows)
|
||||
})
|
||||
}
|
||||
|
||||
func queryLevelEventClaimKeysByStatus(batchSize int, query func(status string) ([]levelEventClaimKey, error)) ([]levelEventClaimKey, error) {
|
||||
statuses := []string{"pending", "failed"}
|
||||
groups := make([][]levelEventClaimKey, 0, len(statuses))
|
||||
for _, status := range statuses {
|
||||
keys, err := query(status)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
groups = append(groups, keys)
|
||||
}
|
||||
return mergeLevelEventClaimKeys(batchSize, groups...), nil
|
||||
}
|
||||
|
||||
func scanLevelEventClaimKeys(rows *sql.Rows) ([]levelEventClaimKey, error) {
|
||||
keys := make([]levelEventClaimKey, 0)
|
||||
for rows.Next() {
|
||||
var key levelEventClaimKey
|
||||
if err := rows.Scan(&key.EventID, &key.CreatedAtMS); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
keys = append(keys, key)
|
||||
}
|
||||
return keys, rows.Err()
|
||||
}
|
||||
|
||||
func mergeLevelEventClaimKeys(limit int, groups ...[]levelEventClaimKey) []levelEventClaimKey {
|
||||
if limit <= 0 {
|
||||
return nil
|
||||
}
|
||||
offsets := make([]int, len(groups))
|
||||
merged := make([]levelEventClaimKey, 0, limit)
|
||||
// 每个 status 查询内部已经按 created_at/event_id 排序，这里做小规模归并来维持原 SQL 的跨状态顺序。
|
||||
for len(merged) < limit {
|
||||
bestGroup := -1
|
||||
var best levelEventClaimKey
|
||||
for groupIndex, group := range groups {
|
||||
offset := offsets[groupIndex]
|
||||
if offset >= len(group) {
|
||||
continue
|
||||
}
|
||||
candidate := group[offset]
|
||||
if bestGroup == -1 || levelEventClaimKeyLess(candidate, best) {
|
||||
bestGroup = groupIndex
|
||||
best = candidate
|
||||
}
|
||||
}
|
||||
if bestGroup == -1 {
|
||||
break
|
||||
}
|
||||
merged = append(merged, best)
|
||||
offsets[bestGroup]++
|
||||
}
|
||||
return merged
|
||||
}
|
||||
|
||||
func levelEventClaimKeyLess(left levelEventClaimKey, right levelEventClaimKey) bool {
|
||||
if left.CreatedAtMS != right.CreatedAtMS {
|
||||
return left.CreatedAtMS < right.CreatedAtMS
|
||||
}
|
||||
return left.EventID < right.EventID
|
||||
}
|
||||
|
||||
func (r *Repository) fetchLevelEventsByClaimKeys(ctx context.Context, tx *sql.Tx, appCode string, keys []levelEventClaimKey) ([]gamedomain.LevelEventOutbox, error) {
|
||||
if len(keys) == 0 {
|
||||
return nil, nil
|
||||
}
|
||||
placeholders := strings.TrimRight(strings.Repeat("?,", len(keys)), ",")
|
||||
args := make([]any, 0, len(keys)+1)
|
||||
args = append(args, appCode)
|
||||
for _, key := range keys {
|
||||
args = append(args, key.EventID)
|
||||
}
|
||||
rows, err := tx.QueryContext(ctx, levelEventOutboxSelectSQL()+`WHERE app_code = ? AND event_id IN (`+placeholders+`)`, args...)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
defer rows.Close()
|
||||
return scanLevelEventOutboxRows(rows)
|
||||
items, err := scanLevelEventOutboxRows(rows)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
byID := make(map[string]gamedomain.LevelEventOutbox, len(items))
|
||||
for _, item := range items {
|
||||
byID[item.EventID] = item
|
||||
}
|
||||
ordered := make([]gamedomain.LevelEventOutbox, 0, len(keys))
|
||||
for _, key := range keys {
|
||||
item, ok := byID[key.EventID]
|
||||
if !ok {
|
||||
return nil, errors.New("claimed level event disappeared before fetch")
|
||||
}
|
||||
ordered = append(ordered, item)
|
||||
}
|
||||
return ordered, nil
|
||||
}
|
||||
|
||||
func (r *Repository) MarkLevelEventDelivered(ctx context.Context, eventID string, nowMS int64) error {
|
||||
|
||||
@ -2,6 +2,7 @@ package mysql
|
||||
|
||||
import (
|
||||
"context"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
@ -63,6 +64,108 @@ func TestMarkOrderSucceededCreatesClaimableLevelOutboxEvent(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
func TestClaimPendingLevelEventsPreservesCreatedOrderAcrossStatuses(t *testing.T) {
|
||||
caller := mysqlschema.CallerFile(t, 1)
|
||||
schema := mysqlschema.New(t, mysqlschema.Config{
|
||||
InitDBPath: mysqlschema.InitDBPath(t, caller, "../../../deploy/mysql/initdb/001_game_service.sql"),
|
||||
DatabasePrefix: "hy_game_test",
|
||||
})
|
||||
|
||||
ctx := appcode.WithContext(context.Background(), "lalu")
|
||||
repo, err := Open(ctx, schema.DSN)
|
||||
if err != nil {
|
||||
t.Fatalf("open repository failed: %v", err)
|
||||
}
|
||||
t.Cleanup(func() { _ = repo.Close() })
|
||||
|
||||
insertLevelEvent := func(eventID string, orderID string, status string, nextRetryAtMS int64, createdAtMS int64, payload string) {
|
||||
t.Helper()
|
||||
if _, err := repo.db.ExecContext(ctx, `
|
||||
INSERT INTO game_level_event_outbox (
|
||||
app_code, event_id, order_id, user_id, game_id, provider_order_id,
|
||||
provider_round_id, coin_amount, wallet_transaction_id, payload_json,
|
||||
status, attempt_count, next_retry_at_ms, locked_by, locked_until_ms,
|
||||
failure_reason, occurred_at_ms, created_at_ms, updated_at_ms
|
||||
) VALUES (?, ?, ?, 42, 'game_1', ?, '', 100, 'wallet_tx_1', ?,
|
||||
?, 0, ?, '', 0, '', ?, ?, ?)`,
|
||||
"lalu", eventID, orderID, "provider_"+orderID, payload, status, nextRetryAtMS, createdAtMS, createdAtMS, createdAtMS,
|
||||
); err != nil {
|
||||
t.Fatalf("insert level event %s failed: %v", eventID, err)
|
||||
}
|
||||
}
|
||||
insertLevelEvent("event_failed_old", "order_failed_old", "failed", 1000, 1000, `{"source":"failed"}`)
|
||||
insertLevelEvent("event_failed_future", "order_failed_future", "failed", 999999, 1500, `{"source":"future"}`)
|
||||
insertLevelEvent("event_pending_new", "order_pending_new", "pending", 0, 2000, `{"source":"pending"}`)
|
||||
|
||||
events, err := repo.ClaimPendingLevelEvents(ctx, "worker_1", 3000, 30*time.Second, 2)
|
||||
if err != nil {
|
||||
t.Fatalf("claim level events failed: %v", err)
|
||||
}
|
||||
if len(events) != 2 {
|
||||
t.Fatalf("expected two claimed events, got %+v", events)
|
||||
}
|
||||
if events[0].EventID != "event_failed_old" || events[0].Status != "failed" || !strings.Contains(events[0].PayloadJSON, `"source": "failed"`) {
|
||||
t.Fatalf("first claimed event should be the older failed payload, got %+v", events[0])
|
||||
}
|
||||
if events[1].EventID != "event_pending_new" || events[1].Status != "pending" || !strings.Contains(events[1].PayloadJSON, `"source": "pending"`) {
|
||||
t.Fatalf("second claimed event should be the pending payload, got %+v", events[1])
|
||||
}
|
||||
|
||||
var runningCount int
|
||||
if err := repo.db.QueryRowContext(ctx, `
|
||||
SELECT COUNT(*) FROM game_level_event_outbox
|
||||
WHERE app_code = 'lalu' AND status = 'running' AND locked_by = 'worker_1' AND locked_until_ms = 33000`,
|
||||
).Scan(&runningCount); err != nil {
|
||||
t.Fatalf("query running events failed: %v", err)
|
||||
}
|
||||
if runningCount != 2 {
|
||||
t.Fatalf("expected two running claimed events, got %d", runningCount)
|
||||
}
|
||||
var futureStatus string
|
||||
if err := repo.db.QueryRowContext(ctx, `
|
||||
SELECT status FROM game_level_event_outbox WHERE app_code = 'lalu' AND event_id = 'event_failed_future'`,
|
||||
).Scan(&futureStatus); err != nil {
|
||||
t.Fatalf("query future retry event failed: %v", err)
|
||||
}
|
||||
if futureStatus != "failed" {
|
||||
t.Fatalf("future retry event should stay failed, got %s", futureStatus)
|
||||
}
|
||||
}
|
||||
|
||||
func TestLevelEventClaimKeyQueryUsesOrderIndexWithoutFilesort(t *testing.T) {
|
||||
caller := mysqlschema.CallerFile(t, 1)
|
||||
schema := mysqlschema.New(t, mysqlschema.Config{
|
||||
InitDBPath: mysqlschema.InitDBPath(t, caller, "../../../deploy/mysql/initdb/001_game_service.sql"),
|
||||
DatabasePrefix: "hy_game_test",
|
||||
})
|
||||
|
||||
ctx := appcode.WithContext(context.Background(), "lalu")
|
||||
repo, err := Open(ctx, schema.DSN)
|
||||
if err != nil {
|
||||
t.Fatalf("open repository failed: %v", err)
|
||||
}
|
||||
t.Cleanup(func() { _ = repo.Close() })
|
||||
|
||||
var plan string
|
||||
if err := repo.db.QueryRowContext(ctx, `
|
||||
EXPLAIN FORMAT=JSON
|
||||
SELECT event_id, created_at_ms
|
||||
FROM game_level_event_outbox FORCE INDEX (idx_game_level_event_claim_order)
|
||||
WHERE app_code = ? AND status = ? AND locked_until_ms = 0 AND next_retry_at_ms <= ?
|
||||
ORDER BY created_at_ms ASC, event_id ASC
|
||||
LIMIT 100`,
|
||||
"lalu", "pending", int64(1780542672724),
|
||||
).Scan(&plan); err != nil {
|
||||
t.Fatalf("explain level event claim key query failed: %v", err)
|
||||
}
|
||||
if !strings.Contains(plan, `"key": "idx_game_level_event_claim_order"`) {
|
||||
t.Fatalf("claim key query did not use idx_game_level_event_claim_order:\n%s", plan)
|
||||
}
|
||||
if strings.Contains(strings.ToLower(plan), `"using_filesort": true`) {
|
||||
t.Fatalf("claim key query should avoid filesort:\n%s", plan)
|
||||
}
|
||||
}
|
||||
|
||||
func TestUpsertCatalogCreatesDefaultVoiceRoomDisplayRule(t *testing.T) {
|
||||
caller := mysqlschema.CallerFile(t, 1)
|
||||
schema := mysqlschema.New(t, mysqlschema.Config{
|
||||
|
||||
Loading…
x
Reference in New Issue
Block a user