hy/hyapp-server
2
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

兼容转盘核账线上索引

Browse Source
This commit is contained in:
zhx 2026-07-13 22:44:01 +08:00
parent f0c4df7aa3
commit fe468205e5
2 changed files with 257 additions and 19 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

174
services/activity-service/cmd/audit-wheel-rewards/main.go
View File

@ -79,6 +79,23 @@ type auditSummary struct {
AuditSHA256 string `json:"audit_sha256"`
}
// outboxAuditIndex 是 FORCE INDEX 唯一允许接受的封闭枚举。information_schema 返回的是数据库数据，
// 不能把其中的任意字符串直接拼接进 SQL；查询构造器会再次按用途校验枚举并只输出下列编译期常量。
type outboxAuditIndex uint8
const (
outboxAuditIndexUnknown outboxAuditIndex = iota
outboxAuditIndexEventRetry
outboxAuditIndexLuckyRetry
outboxAuditIndexEventLock
outboxAuditIndexLuckyLock
)
type outboxAuditIndexes struct {
Retry outboxAuditIndex
Lock outboxAuditIndex
}
type appCodeFlags []string
func (values *appCodeFlags) String() string {
@ -131,7 +148,11 @@ func main() {
if err := db.PingContext(ctx); err != nil {
log.Fatal(err)
}
rows, err := loadUnsettledRows(ctx, db, appCodes, *limit)
indexes, err := discoverOutboxAuditIndexes(ctx, db)
if err != nil {
log.Fatal(err)
}
rows, err := loadUnsettledRows(ctx, db, indexes, appCodes, *limit)
if err != nil {
log.Fatal(err)
}
@ -159,7 +180,7 @@ func main() {
fmt.Println(string(encoded))
}
func loadUnsettledRows(ctx context.Context, db *sql.DB, requestedAppCodes []string, limit int) ([]outboxRow, error) {
func loadUnsettledRows(ctx context.Context, db *sql.DB, indexes outboxAuditIndexes, requestedAppCodes []string, limit int) ([]outboxRow, error) {
if limit <= 0 || limit > 500 {
return nil, fmt.Errorf("limit must be between 1 and 500")
}
@ -167,6 +188,14 @@ func loadUnsettledRows(ctx context.Context, db *sql.DB, requestedAppCodes []stri
if err != nil {
return nil, err
}
retryQuery, err := unsettledRetryQuery(indexes.Retry)
if err != nil {
return nil, err
}
deliveringQuery, err := unsettledDeliveringQuery(indexes.Lock)
if err != nil {
return nil, err
}
// activity_outbox 的线上索引都以 app_code 开头。每个 App 拆成 retry/lock 两个等值分区查询，
// 既能强制使用对应索引，也避免 status OR 让优化器退化为全表扫描。每个分支只取 global remaining，
// 合并出该 App 最早的一页后再进入下一 App，因此返回量严格不超过 limit，数据库返回行数也被约束在约 2*limit 内。
@ -181,7 +210,7 @@ func loadUnsettledRows(ctx context.Context, db *sql.DB, requestedAppCodes []stri
args []any
}{
{
query: unsettledRetryQuery(),
query: retryQuery,
args: []any{
appCodeValue,
domain.EventTypeWheelRewardSettlement,
@ -191,7 +220,7 @@ func loadUnsettledRows(ctx context.Context, db *sql.DB, requestedAppCodes []stri
},
},
{
query: unsettledDeliveringQuery(),
query: deliveringQuery,
args: []any{
appCodeValue,
domain.EventTypeWheelRewardSettlement,
@ -233,22 +262,139 @@ func loadUnsettledRows(ctx context.Context, db *sql.DB, requestedAppCodes []stri
return result, nil
}
func unsettledRetryQuery() string {
return `
SELECT app_code, outbox_id, payload, status, retry_count, created_at_ms
FROM activity_outbox FORCE INDEX (idx_activity_outbox_event_retry)
WHERE app_code = ? AND event_type = ? AND status IN (?, ?)
ORDER BY created_at_ms, outbox_id
LIMIT ?`
func discoverOutboxAuditIndexes(ctx context.Context, db *sql.DB) (outboxAuditIndexes, error) {
rows, err := db.QueryContext(ctx, outboxAuditIndexDiscoveryQuery())
if err != nil {
return outboxAuditIndexes{}, fmt.Errorf("discover activity_outbox audit indexes: %w", err)
}
defer rows.Close()
type indexPrefix struct {
columns [3]string
seen [3]bool
invalid bool
}
prefixes := make(map[outboxAuditIndex]*indexPrefix, 4)
for rows.Next() {
var indexName, columnName string
var sequence int
if err := rows.Scan(&indexName, &sequence, &columnName); err != nil {
return outboxAuditIndexes{}, fmt.Errorf("scan activity_outbox audit index: %w", err)
}
index, ok := outboxAuditIndexByName(indexName)
if !ok || sequence < 1 || sequence > 3 {
continue
}
prefix := prefixes[index]
if prefix == nil {
prefix = &indexPrefix{}
prefixes[index] = prefix
}
position := sequence - 1
if prefix.seen[position] {
prefix.invalid = true
continue
}
prefix.seen[position] = true
prefix.columns[position] = strings.ToLower(strings.TrimSpace(columnName))
}
if err := rows.Err(); err != nil {
return outboxAuditIndexes{}, fmt.Errorf("iterate activity_outbox audit indexes: %w", err)
}
validPrefix := func(index outboxAuditIndex) bool {
prefix := prefixes[index]
if prefix == nil || prefix.invalid {
return false
}
expected := [3]string{"app_code", "event_type", "status"}
return prefix.seen == [3]bool{true, true, true} && prefix.columns == expected
}
pick := func(preferred, legacy outboxAuditIndex) outboxAuditIndex {
if validPrefix(preferred) {
return preferred
}
if validPrefix(legacy) {
return legacy
}
return outboxAuditIndexUnknown
}
indexes := outboxAuditIndexes{
Retry: pick(outboxAuditIndexEventRetry, outboxAuditIndexLuckyRetry),
Lock: pick(outboxAuditIndexEventLock, outboxAuditIndexLuckyLock),
}
// 缺失或列前缀不匹配时必须中止。核账命令不能静默退回无索引查询，否则线上只读审计也可能拖垮主库。
if indexes.Retry == outboxAuditIndexUnknown || indexes.Lock == outboxAuditIndexUnknown {
return outboxAuditIndexes{}, fmt.Errorf("activity_outbox requires whitelisted retry and lock indexes with prefix (app_code,event_type,status)")
}
return indexes, nil
}
func unsettledDeliveringQuery() string {
func outboxAuditIndexDiscoveryQuery() string {
return `
SELECT index_name, seq_in_index, column_name
FROM information_schema.statistics
WHERE table_schema = DATABASE()
AND table_name = 'activity_outbox'
AND index_name IN (
'idx_activity_outbox_event_retry',
'idx_activity_outbox_lucky_retry',
'idx_activity_outbox_event_lock',
'idx_activity_outbox_lucky_lock'
)
AND seq_in_index <= 3
ORDER BY index_name, seq_in_index`
}
func outboxAuditIndexByName(value string) (outboxAuditIndex, bool) {
switch strings.TrimSpace(value) {
case "idx_activity_outbox_event_retry":
return outboxAuditIndexEventRetry, true
case "idx_activity_outbox_lucky_retry":
return outboxAuditIndexLuckyRetry, true
case "idx_activity_outbox_event_lock":
return outboxAuditIndexEventLock, true
case "idx_activity_outbox_lucky_lock":
return outboxAuditIndexLuckyLock, true
default:
return outboxAuditIndexUnknown, false
}
}
func unsettledRetryQuery(index outboxAuditIndex) (string, error) {
var indexName string
switch index {
case outboxAuditIndexEventRetry:
indexName = "idx_activity_outbox_event_retry"
case outboxAuditIndexLuckyRetry:
indexName = "idx_activity_outbox_lucky_retry"
default:
return "", fmt.Errorf("retry FORCE INDEX is not whitelisted")
}
return fmt.Sprintf(`
SELECT app_code, outbox_id, payload, status, retry_count, created_at_ms
FROM activity_outbox FORCE INDEX (idx_activity_outbox_event_lock)
FROM activity_outbox FORCE INDEX (%s)
WHERE app_code = ? AND event_type = ? AND status IN (?, ?)
ORDER BY created_at_ms, outbox_id
LIMIT ?`, indexName), nil
}
func unsettledDeliveringQuery(index outboxAuditIndex) (string, error) {
var indexName string
switch index {
case outboxAuditIndexEventLock:
indexName = "idx_activity_outbox_event_lock"
case outboxAuditIndexLuckyLock:
indexName = "idx_activity_outbox_lucky_lock"
default:
return "", fmt.Errorf("lock FORCE INDEX is not whitelisted")
}
return fmt.Sprintf(`
SELECT app_code, outbox_id, payload, status, retry_count, created_at_ms
FROM activity_outbox FORCE INDEX (%s)
WHERE app_code = ? AND event_type = ? AND status = ?
ORDER BY created_at_ms, outbox_id
LIMIT ?`
LIMIT ?`, indexName), nil
}
func loadUnsettledBranch(ctx context.Context, db *sql.DB, query string, args ...any) ([]outboxRow, error) {

102
services/activity-service/cmd/audit-wheel-rewards/main_test.go
View File

@ -207,26 +207,35 @@ func TestLoadUnsettledRowsUsesAppScopedIndexesAndGlobalLimit(t *testing.T) {
defer db.Close()
const limit = 3
columns := []string{"app_code", "outbox_id", "payload", "status", "retry_count", "created_at_ms"}
indexes := outboxAuditIndexes{Retry: outboxAuditIndexLuckyRetry, Lock: outboxAuditIndexLuckyLock}
retryQuery, err := unsettledRetryQuery(indexes.Retry)
if err != nil {
t.Fatalf("build retry query: %v", err)
}
deliveringQuery, err := unsettledDeliveringQuery(indexes.Lock)
if err != nil {
t.Fatalf("build delivering query: %v", err)
}
// normalizeAuditAppCodes sorts and deduplicates, so SQL order is deterministic regardless of flag order.
mock.ExpectQuery(regexp.QuoteMeta(unsettledRetryQuery())).
mock.ExpectQuery(regexp.QuoteMeta(retryQuery)).
WithArgs("fami", domain.EventTypeWheelRewardSettlement, domain.OutboxStatusPending, domain.OutboxStatusRetryable, limit).
WillReturnRows(sqlmock.NewRows(columns).
AddRow("fami", "fami-pending", []byte(`{}`), domain.OutboxStatusPending, 0, int64(20)))
mock.ExpectQuery(regexp.QuoteMeta(unsettledDeliveringQuery())).
mock.ExpectQuery(regexp.QuoteMeta(deliveringQuery)).
WithArgs("fami", domain.EventTypeWheelRewardSettlement, domain.OutboxStatusDelivering, limit).
WillReturnRows(sqlmock.NewRows(columns).
AddRow("fami", "fami-delivering", []byte(`{}`), domain.OutboxStatusDelivering, 1, int64(10)))
mock.ExpectQuery(regexp.QuoteMeta(unsettledRetryQuery())).
mock.ExpectQuery(regexp.QuoteMeta(retryQuery)).
WithArgs("lalu", domain.EventTypeWheelRewardSettlement, domain.OutboxStatusPending, domain.OutboxStatusRetryable, 1).
WillReturnRows(sqlmock.NewRows(columns).
AddRow("lalu", "lalu-oldest", []byte(`{}`), domain.OutboxStatusRetryable, 2, int64(5)))
mock.ExpectQuery(regexp.QuoteMeta(unsettledDeliveringQuery())).
mock.ExpectQuery(regexp.QuoteMeta(deliveringQuery)).
WithArgs("lalu", domain.EventTypeWheelRewardSettlement, domain.OutboxStatusDelivering, 1).
WillReturnRows(sqlmock.NewRows(columns).
AddRow("lalu", "lalu-delivering", []byte(`{}`), domain.OutboxStatusDelivering, 1, int64(15)))
rows, err := loadUnsettledRows(context.Background(), db, []string{"lalu", "fami", "fami"}, limit)
rows, err := loadUnsettledRows(context.Background(), db, indexes, []string{"lalu", "fami", "fami"}, limit)
if err != nil {
t.Fatalf("loadUnsettledRows returned error: %v", err)
}
@ -244,6 +253,89 @@ func TestLoadUnsettledRowsUsesAppScopedIndexesAndGlobalLimit(t *testing.T) {
}
}
func TestDiscoverOutboxAuditIndexes(t *testing.T) {
indexColumns := []string{"index_name", "seq_in_index", "column_name"}
addPrefix := func(rows *sqlmock.Rows, indexName string, columns ...string) *sqlmock.Rows {
for position, column := range columns {
rows.AddRow(indexName, position+1, column)
}
return rows
}
tests := []struct {
name string
rows func() *sqlmock.Rows
want outboxAuditIndexes
wantError bool
}{
{
name: "prefers current event indexes",
rows: func() *sqlmock.Rows {
rows := sqlmock.NewRows(indexColumns)
addPrefix(rows, "idx_activity_outbox_lucky_retry", "app_code", "event_type", "status")
addPrefix(rows, "idx_activity_outbox_event_retry", "app_code", "event_type", "status")
addPrefix(rows, "idx_activity_outbox_lucky_lock", "app_code", "event_type", "status")
addPrefix(rows, "idx_activity_outbox_event_lock", "app_code", "event_type", "status")
return rows
},
want: outboxAuditIndexes{Retry: outboxAuditIndexEventRetry, Lock: outboxAuditIndexEventLock},
},
{
name: "falls back to deployed lucky indexes",
rows: func() *sqlmock.Rows {
rows := sqlmock.NewRows(indexColumns)
addPrefix(rows, "idx_activity_outbox_lucky_retry", "app_code", "event_type", "status")
addPrefix(rows, "idx_activity_outbox_lucky_lock", "app_code", "event_type", "status")
return rows
},
want: outboxAuditIndexes{Retry: outboxAuditIndexLuckyRetry, Lock: outboxAuditIndexLuckyLock},
},
{
name: "rejects wrong prefix without an index-free fallback",
rows: func() *sqlmock.Rows {
rows := sqlmock.NewRows(indexColumns)
addPrefix(rows, "idx_activity_outbox_event_retry", "event_type", "app_code", "status")
addPrefix(rows, "idx_activity_outbox_event_lock", "app_code", "event_type", "status")
return rows
},
wantError: true,
},
}
for _, test := range tests {
t.Run(test.name, func(t *testing.T) {
db, mock, err := sqlmock.New()
if err != nil {
t.Fatalf("create sqlmock: %v", err)
}
defer db.Close()
mock.ExpectQuery(regexp.QuoteMeta(outboxAuditIndexDiscoveryQuery())).WillReturnRows(test.rows())
got, err := discoverOutboxAuditIndexes(context.Background(), db)
if test.wantError {
if err == nil {
t.Fatalf("discoverOutboxAuditIndexes returned indexes=%+v, want error", got)
}
} else if err != nil || got != test.want {
t.Fatalf("discoverOutboxAuditIndexes=(%+v, %v), want=(%+v, nil)", got, err, test.want)
}
if err := mock.ExpectationsWereMet(); err != nil {
t.Fatalf("unmet SQL expectations: %v", err)
}
})
}
}
func TestUnsettledQueriesRejectNonWhitelistedIndexes(t *testing.T) {
if _, err := unsettledRetryQuery(outboxAuditIndexLuckyLock); err == nil {
t.Fatal("retry query accepted a lock index")
}
if _, err := unsettledDeliveringQuery(outboxAuditIndexLuckyRetry); err == nil {
t.Fatal("delivering query accepted a retry index")
}
if _, err := unsettledRetryQuery(outboxAuditIndex(255)); err == nil {
t.Fatal("retry query accepted an arbitrary identifier enum")
}
}
func TestNormalizeAuditAppCodesRequiresRepeatedExplicitScope(t *testing.T) {
if _, err := normalizeAuditAppCodes(nil); err == nil {
t.Fatal("missing --app-code must be rejected before opening or querying MySQL")