295 lines
15 KiB
Go
295 lines
15 KiB
Go
package main
|
||
|
||
import (
|
||
"context"
|
||
"database/sql"
|
||
"flag"
|
||
"fmt"
|
||
"log"
|
||
"os"
|
||
"strings"
|
||
"time"
|
||
|
||
mysqlstorage "hyapp/services/statistics-service/internal/storage/mysql"
|
||
|
||
_ "github.com/go-sql-driver/mysql"
|
||
)
|
||
|
||
type auditWindow struct {
|
||
startDay string
|
||
endDay string
|
||
startMS int64
|
||
endMS int64
|
||
}
|
||
|
||
type auditCheck struct {
|
||
key string
|
||
label string
|
||
source string
|
||
required bool
|
||
count int64
|
||
err error
|
||
}
|
||
|
||
func main() {
|
||
var (
|
||
appCode = flag.String("app", envDefault("APP_CODE", "lalu"), "app_code to audit")
|
||
statTZ = flag.String("timezone", envDefault("STAT_TZ", mysqlstorage.StatTZAsiaShanghai), "stat_tz to audit: UTC or Asia/Shanghai")
|
||
startDay = flag.String("start-day", envDefault("START_DAY", todayIn(mysqlstorage.StatTZAsiaShanghai)), "inclusive stat day in selected timezone, yyyy-mm-dd")
|
||
endDay = flag.String("end-day", envDefault("END_DAY", todayIn(mysqlstorage.StatTZAsiaShanghai)), "inclusive stat day in selected timezone, yyyy-mm-dd")
|
||
statisticsDSN = flag.String("statistics-dsn", firstNonEmptyEnv("STATISTICS_MYSQL_DSN", "MYSQL_DSN"), "statistics mysql dsn")
|
||
userDSN = flag.String("user-dsn", firstNonEmptyEnv("USER_MYSQL_DSN", "MYSQL_USER_DSN"), "user-service mysql dsn")
|
||
walletDSN = flag.String("wallet-dsn", firstNonEmptyEnv("WALLET_MYSQL_DSN", "MYSQL_WALLET_DSN"), "wallet-service mysql dsn")
|
||
roomDSN = flag.String("room-dsn", firstNonEmptyEnv("ROOM_MYSQL_DSN", "MYSQL_ROOM_DSN"), "room-service mysql dsn")
|
||
gameDSN = flag.String("game-dsn", firstNonEmptyEnv("GAME_MYSQL_DSN", "MYSQL_GAME_DSN"), "game-service mysql dsn")
|
||
)
|
||
flag.Parse()
|
||
|
||
statTZValue, loc, err := normalizeStatTZ(*statTZ)
|
||
if err != nil {
|
||
log.Fatal(err)
|
||
}
|
||
window, err := resolveWindow(*startDay, *endDay, loc)
|
||
if err != nil {
|
||
log.Fatal(err)
|
||
}
|
||
ctx := context.Background()
|
||
dbs := map[string]*sql.DB{}
|
||
for name, dsn := range map[string]string{
|
||
"statistics": *statisticsDSN,
|
||
"user": *userDSN,
|
||
"wallet": *walletDSN,
|
||
"room": *roomDSN,
|
||
"game": *gameDSN,
|
||
} {
|
||
if strings.TrimSpace(dsn) == "" {
|
||
log.Fatalf("%s dsn is required", name)
|
||
}
|
||
db, err := openDB(ctx, dsn)
|
||
if err != nil {
|
||
log.Fatalf("open %s db: %v", name, err)
|
||
}
|
||
defer db.Close()
|
||
dbs[name] = db
|
||
}
|
||
|
||
sourceChecks := auditSources(ctx, dbs, *appCode, window)
|
||
statsChecks := auditStatistics(ctx, dbs["statistics"], *appCode, statTZValue, window)
|
||
failed := printChecks("owner outbox", sourceChecks)
|
||
failed += printChecks("statistics aggregate", statsChecks)
|
||
if failed > 0 {
|
||
log.Fatalf("databi real flow audit failed: %d required checks have no data", failed)
|
||
}
|
||
fmt.Printf("databi real flow audit passed app=%s stat_tz=%s days=%s..%s\n", *appCode, statTZValue, window.startDay, window.endDay)
|
||
}
|
||
|
||
func auditSources(ctx context.Context, dbs map[string]*sql.DB, app string, window auditWindow) []auditCheck {
|
||
return []auditCheck{
|
||
countCheck(ctx, dbs["user"], "new_users", "新增用户", "user_outbox.UserRegistered", true, `
|
||
SELECT COUNT(*) FROM user_outbox
|
||
WHERE app_code = ? AND event_type = 'UserRegistered' AND created_at_ms >= ? AND created_at_ms < ?`, app, window.startMS, window.endMS),
|
||
countCheck(ctx, dbs["room"], "active_users", "活跃用户", "room_outbox.RoomUserJoined", true, `
|
||
SELECT COUNT(*) FROM room_outbox
|
||
WHERE app_code = ? AND event_type = 'RoomUserJoined' AND created_at_ms >= ? AND created_at_ms < ?`, app, window.startMS, window.endMS),
|
||
countCheck(ctx, dbs["user"], "mic_online_ms", "麦上时长", "user_outbox.UserMicDailyStatsUpdated", true, `
|
||
SELECT COUNT(*) FROM user_outbox
|
||
WHERE app_code = ? AND event_type = 'UserMicDailyStatsUpdated' AND created_at_ms >= ? AND created_at_ms < ?`, app, window.startMS, window.endMS),
|
||
countCheck(ctx, dbs["wallet"], "google_recharge_usd_minor", "谷歌充值", "wallet_outbox.WalletRechargeRecorded(recharge_type=google/google_play，可本地模拟)", true, `
|
||
SELECT COUNT(*) FROM wallet_outbox
|
||
WHERE app_code = ? AND event_type = 'WalletRechargeRecorded' AND created_at_ms >= ? AND created_at_ms < ?
|
||
AND LOWER(COALESCE(JSON_UNQUOTE(JSON_EXTRACT(payload, '$.recharge_type')), JSON_UNQUOTE(JSON_EXTRACT(payload, '$.channel')), '')) IN ('google','google_play')`, app, window.startMS, window.endMS),
|
||
countCheck(ctx, dbs["wallet"], "mifapay_recharge_usd_minor", "三方充值", "wallet_outbox.WalletRechargeRecorded(recharge_type=mifapay/v5pay，可本地模拟)", true, `
|
||
SELECT COUNT(*) FROM wallet_outbox
|
||
WHERE app_code = ? AND event_type = 'WalletRechargeRecorded' AND created_at_ms >= ? AND created_at_ms < ?
|
||
AND LOWER(COALESCE(JSON_UNQUOTE(JSON_EXTRACT(payload, '$.recharge_type')), JSON_UNQUOTE(JSON_EXTRACT(payload, '$.channel')), '')) IN ('mifapay','v5pay')`, app, window.startMS, window.endMS),
|
||
countCheck(ctx, dbs["wallet"], "coin_seller_stock_coin", "币商充值金币", "wallet_outbox.WalletCoinSellerStockPurchased", true, `
|
||
SELECT COUNT(*) FROM wallet_outbox
|
||
WHERE app_code = ? AND event_type = 'WalletCoinSellerStockPurchased' AND created_at_ms >= ? AND created_at_ms < ?`, app, window.startMS, window.endMS),
|
||
countCheck(ctx, dbs["wallet"], "coin_seller_transfer_coin", "币商出货金币", "wallet_outbox.WalletRechargeRecorded(recharge_type=coin_seller_transfer)", true, `
|
||
SELECT COUNT(*) FROM wallet_outbox
|
||
WHERE app_code = ? AND event_type = 'WalletRechargeRecorded' AND created_at_ms >= ? AND created_at_ms < ?
|
||
AND LOWER(COALESCE(JSON_UNQUOTE(JSON_EXTRACT(payload, '$.recharge_type')), '')) IN ('coin_seller_transfer','coin_seller')`, app, window.startMS, window.endMS),
|
||
countCheck(ctx, dbs["wallet"], "coin_total", "金币数量总和", "wallet_outbox.WalletBalanceChanged(asset=COIN)", true, `
|
||
SELECT COUNT(*) FROM wallet_outbox
|
||
WHERE app_code = ? AND event_type = 'WalletBalanceChanged' AND asset_type = 'COIN' AND created_at_ms >= ? AND created_at_ms < ?`, app, window.startMS, window.endMS),
|
||
countCheck(ctx, dbs["wallet"], "consumed_coin", "消耗金币", "wallet_outbox.WalletBalanceChanged(COIN消费biz_type)", true, `
|
||
SELECT COUNT(*) FROM wallet_outbox
|
||
WHERE app_code = ? AND event_type = 'WalletBalanceChanged' AND asset_type = 'COIN' AND available_delta + frozen_delta < 0
|
||
AND created_at_ms >= ? AND created_at_ms < ?
|
||
AND LOWER(COALESCE(JSON_UNQUOTE(JSON_EXTRACT(payload, '$.biz_type')), '')) IN ('gift_debit','direct_gift_debit','vip_purchase','resource_shop_purchase','red_packet_create','game_debit','wheel_draw_debit','cp_breakup_fee')`, app, window.startMS, window.endMS),
|
||
countCheck(ctx, dbs["wallet"], "platform_grant_coin", "平台发放金币", "wallet_outbox.Wallet*RewardCredited", true, `
|
||
SELECT COUNT(*) FROM wallet_outbox
|
||
WHERE app_code = ? AND event_type IN ('WalletTaskRewardCredited','WalletLuckyGiftRewardCredited','WalletWheelRewardCredited','WalletRoomTurnoverRewardCredited','WalletInviteActivityRewardCredited','WalletAgencyOpeningRewardCredited')
|
||
AND created_at_ms >= ? AND created_at_ms < ?`, app, window.startMS, window.endMS),
|
||
countCheck(ctx, dbs["wallet"], "manual_grant_coin", "人工发放金币", "wallet_outbox.WalletBalanceChanged(biz_type=manual_credit)", true, `
|
||
SELECT COUNT(*) FROM wallet_outbox
|
||
WHERE app_code = ? AND event_type = 'WalletBalanceChanged' AND asset_type = 'COIN' AND available_delta + frozen_delta > 0
|
||
AND created_at_ms >= ? AND created_at_ms < ?
|
||
AND LOWER(COALESCE(JSON_UNQUOTE(JSON_EXTRACT(payload, '$.biz_type')), '')) = 'manual_credit'`, app, window.startMS, window.endMS),
|
||
countCheck(ctx, dbs["wallet"], "salary_usd_minor", "工资总和", "wallet_outbox.WalletBalanceChanged(biz_type=host_salary_settlement)", true, `
|
||
SELECT COUNT(*) FROM wallet_outbox
|
||
WHERE app_code = ? AND event_type = 'WalletBalanceChanged' AND asset_type IN ('HOST_SALARY_USD','AGENCY_SALARY_USD')
|
||
AND available_delta + frozen_delta > 0 AND created_at_ms >= ? AND created_at_ms < ?
|
||
AND LOWER(COALESCE(JSON_UNQUOTE(JSON_EXTRACT(payload, '$.biz_type')), '')) = 'host_salary_settlement'`, app, window.startMS, window.endMS),
|
||
countCheck(ctx, dbs["wallet"], "salary_transfer_coin", "工资兑换金币", "wallet_outbox.WalletBalanceChanged(biz_type=salary_transfer_to_coin_seller)", true, `
|
||
SELECT COUNT(*) FROM wallet_outbox
|
||
WHERE app_code = ? AND event_type = 'WalletBalanceChanged' AND asset_type = 'COIN_SELLER_COIN'
|
||
AND available_delta + frozen_delta > 0 AND created_at_ms >= ? AND created_at_ms < ?
|
||
AND LOWER(COALESCE(JSON_UNQUOTE(JSON_EXTRACT(payload, '$.biz_type')), '')) = 'salary_transfer_to_coin_seller'`, app, window.startMS, window.endMS),
|
||
countCheck(ctx, dbs["room"], "gift_coin_spent", "礼物/幸运礼物流水", "room_outbox.RoomGiftSent", true, `
|
||
SELECT COUNT(*) FROM room_outbox
|
||
WHERE app_code = ? AND event_type = 'RoomGiftSent' AND created_at_ms >= ? AND created_at_ms < ?`, app, window.startMS, window.endMS),
|
||
countCheck(ctx, dbs["wallet"], "lucky_gift_payout", "幸运礼物返奖", "wallet_outbox.WalletLuckyGiftRewardCredited", true, `
|
||
SELECT COUNT(*) FROM wallet_outbox
|
||
WHERE app_code = ? AND event_type = 'WalletLuckyGiftRewardCredited' AND created_at_ms >= ? AND created_at_ms < ?`, app, window.startMS, window.endMS),
|
||
countCheck(ctx, dbs["game"], "game_turnover", "游戏流水/利润", "game_outbox.GameOrderSettled", true, `
|
||
SELECT COUNT(*) FROM game_outbox
|
||
WHERE app_code = ? AND event_type = 'GameOrderSettled' AND created_at_ms >= ? AND created_at_ms < ?`, app, window.startMS, window.endMS),
|
||
}
|
||
}
|
||
|
||
func auditStatistics(ctx context.Context, db *sql.DB, app string, statTZ string, window auditWindow) []auditCheck {
|
||
checks := []auditCheck{}
|
||
for _, spec := range []struct {
|
||
key string
|
||
label string
|
||
expr string
|
||
}{
|
||
{"new_users", "新增用户", "SUM(new_users)"},
|
||
{"active_users", "活跃用户", "SUM(active_users)"},
|
||
{"paid_users", "付费用户", "SUM(paid_users)"},
|
||
{"recharge_usd_minor", "总充值", "SUM(recharge_usd_minor)"},
|
||
{"coin_seller_recharge_usd_minor", "币商充值", "SUM(coin_seller_recharge_usd_minor)"},
|
||
{"coin_seller_stock_coin", "币商充值金币", "SUM(coin_seller_stock_coin)"},
|
||
{"coin_seller_transfer_coin", "币商出货金币", "SUM(coin_seller_transfer_coin)"},
|
||
{"google_recharge_usd_minor", "谷歌充值", "SUM(google_recharge_usd_minor)"},
|
||
{"mifapay_recharge_usd_minor", "三方充值", "SUM(mifapay_recharge_usd_minor)"},
|
||
{"coin_total", "金币数量总和", "SUM(coin_total)"},
|
||
{"consumed_coin", "消耗金币", "SUM(consumed_coin)"},
|
||
{"platform_grant_coin", "平台发放金币", "SUM(platform_grant_coin)"},
|
||
{"manual_grant_coin", "人工发放金币", "SUM(manual_grant_coin)"},
|
||
{"salary_usd_minor", "工资总和", "SUM(salary_usd_minor)"},
|
||
{"salary_transfer_coin", "工资兑换金币", "SUM(salary_transfer_coin)"},
|
||
{"mic_online_ms", "麦上总时长", "SUM(mic_online_ms)"},
|
||
{"mic_online_users", "平均麦上时间用户数", "SUM(mic_online_users)"},
|
||
{"gift_coin_spent", "礼物流水", "SUM(gift_coin_spent)"},
|
||
{"lucky_gift_turnover", "幸运礼物流水", "SUM(lucky_gift_turnover)"},
|
||
{"lucky_gift_payout", "幸运礼物返奖", "SUM(lucky_gift_payout)"},
|
||
{"game_turnover", "游戏流水", "SUM(game_turnover)"},
|
||
{"game_payout", "游戏返奖", "SUM(game_payout)"},
|
||
} {
|
||
checks = append(checks, countCheck(ctx, db, spec.key, spec.label, "stat_app_day_country."+spec.key, true,
|
||
"SELECT COALESCE("+spec.expr+",0) FROM stat_app_day_country WHERE app_code = ? AND stat_tz = ? AND stat_day BETWEEN ? AND ?",
|
||
app, statTZ, window.startDay, window.endDay))
|
||
}
|
||
checks = append(checks,
|
||
countCheck(ctx, db, "super_lucky_gift_turnover", "超级幸运礼物流水", "stat_lucky_gift_pool_day_country(pool_id LIKE %super_lucky%)", true, `
|
||
SELECT COALESCE(SUM(turnover_coin),0) FROM stat_lucky_gift_pool_day_country
|
||
WHERE app_code = ? AND stat_tz = ? AND stat_day BETWEEN ? AND ? AND LOWER(pool_id) LIKE '%super_lucky%'`,
|
||
app, statTZ, window.startDay, window.endDay),
|
||
countCheck(ctx, db, "super_lucky_gift_payout", "超级幸运礼物返奖", "stat_lucky_gift_pool_day_country(pool_id LIKE %super_lucky%)", true, `
|
||
SELECT COALESCE(SUM(payout_coin),0) FROM stat_lucky_gift_pool_day_country
|
||
WHERE app_code = ? AND stat_tz = ? AND stat_day BETWEEN ? AND ? AND LOWER(pool_id) LIKE '%super_lucky%'`,
|
||
app, statTZ, window.startDay, window.endDay),
|
||
)
|
||
return checks
|
||
}
|
||
|
||
func countCheck(ctx context.Context, db *sql.DB, key string, label string, source string, required bool, query string, args ...any) auditCheck {
|
||
check := auditCheck{key: key, label: label, source: source, required: required}
|
||
if db == nil {
|
||
check.err = fmt.Errorf("db is nil")
|
||
return check
|
||
}
|
||
err := db.QueryRowContext(ctx, query, args...).Scan(&check.count)
|
||
if err != nil {
|
||
check.err = err
|
||
}
|
||
return check
|
||
}
|
||
|
||
func printChecks(title string, checks []auditCheck) int {
|
||
fmt.Printf("\n%s\n", title)
|
||
failed := 0
|
||
for _, check := range checks {
|
||
status := "OK"
|
||
if check.err != nil || (check.required && check.count <= 0) {
|
||
status = "MISSING"
|
||
failed++
|
||
}
|
||
if check.err != nil {
|
||
fmt.Printf(" %-8s %-34s count=%d source=%s err=%v\n", status, check.label, check.count, check.source, check.err)
|
||
continue
|
||
}
|
||
fmt.Printf(" %-8s %-34s count=%d source=%s\n", status, check.label, check.count, check.source)
|
||
}
|
||
return failed
|
||
}
|
||
|
||
func normalizeStatTZ(value string) (string, *time.Location, error) {
|
||
switch strings.TrimSpace(value) {
|
||
case mysqlstorage.StatTZUTC:
|
||
return mysqlstorage.StatTZUTC, time.UTC, nil
|
||
case mysqlstorage.StatTZAsiaShanghai:
|
||
return mysqlstorage.StatTZAsiaShanghai, time.FixedZone(mysqlstorage.StatTZAsiaShanghai, 8*60*60), nil
|
||
default:
|
||
return "", nil, fmt.Errorf("unsupported stat_tz %q; only UTC and Asia/Shanghai are supported", value)
|
||
}
|
||
}
|
||
|
||
func resolveWindow(startDay string, endDay string, loc *time.Location) (auditWindow, error) {
|
||
start, err := time.ParseInLocation("2006-01-02", strings.TrimSpace(startDay), loc)
|
||
if err != nil {
|
||
return auditWindow{}, fmt.Errorf("parse start-day: %w", err)
|
||
}
|
||
end, err := time.ParseInLocation("2006-01-02", strings.TrimSpace(endDay), loc)
|
||
if err != nil {
|
||
return auditWindow{}, fmt.Errorf("parse end-day: %w", err)
|
||
}
|
||
if end.Before(start) {
|
||
return auditWindow{}, fmt.Errorf("end-day %s is before start-day %s", endDay, startDay)
|
||
}
|
||
return auditWindow{
|
||
startDay: start.Format("2006-01-02"),
|
||
endDay: end.Format("2006-01-02"),
|
||
startMS: start.UnixMilli(),
|
||
endMS: end.AddDate(0, 0, 1).UnixMilli(),
|
||
}, nil
|
||
}
|
||
|
||
func openDB(ctx context.Context, dsn string) (*sql.DB, error) {
|
||
db, err := sql.Open("mysql", strings.TrimSpace(dsn))
|
||
if err != nil {
|
||
return nil, err
|
||
}
|
||
if err := db.PingContext(ctx); err != nil {
|
||
_ = db.Close()
|
||
return nil, err
|
||
}
|
||
return db, nil
|
||
}
|
||
|
||
func todayIn(statTZ string) string {
|
||
_, loc, err := normalizeStatTZ(statTZ)
|
||
if err != nil {
|
||
loc = time.FixedZone(mysqlstorage.StatTZAsiaShanghai, 8*60*60)
|
||
}
|
||
return time.Now().In(loc).Format("2006-01-02")
|
||
}
|
||
|
||
func envDefault(key string, fallback string) string {
|
||
if value := strings.TrimSpace(os.Getenv(key)); value != "" {
|
||
return value
|
||
}
|
||
return fallback
|
||
}
|
||
|
||
func firstNonEmptyEnv(keys ...string) string {
|
||
for _, key := range keys {
|
||
if value := strings.TrimSpace(os.Getenv(key)); value != "" {
|
||
return value
|
||
}
|
||
}
|
||
return ""
|
||
}
|