1080 lines
40 KiB
Go
1080 lines
40 KiB
Go
package main
|
||
|
||
import (
|
||
"context"
|
||
"database/sql"
|
||
"flag"
|
||
"fmt"
|
||
"log"
|
||
"os"
|
||
"sort"
|
||
"strings"
|
||
"time"
|
||
|
||
_ "github.com/go-sql-driver/mysql"
|
||
"hyapp/pkg/appcode"
|
||
)
|
||
|
||
type countryRegion struct {
|
||
countryID int64
|
||
regionID int64
|
||
}
|
||
|
||
type appDayKey struct {
|
||
day string
|
||
countryID int64
|
||
regionID int64
|
||
}
|
||
|
||
type appDayAgg struct {
|
||
newUsers int64
|
||
activeUsers int64
|
||
paidUsers int64
|
||
rechargeUSDMinor int64
|
||
newUserRechargeUSDMinor int64
|
||
coinSellerRechargeUSDMinor int64
|
||
coinSellerStockCoin int64
|
||
mifaPayRechargeUSDMinor int64
|
||
googleRechargeUSDMinor int64
|
||
coinSellerTransferCoin int64
|
||
coinTotal int64
|
||
consumedCoin int64
|
||
platformGrantCoin int64
|
||
manualGrantCoin int64
|
||
salaryUSDMinor int64
|
||
salaryTransferCoin int64
|
||
micOnlineMS int64
|
||
micOnlineUsers int64
|
||
}
|
||
|
||
type userRegistrationRow struct {
|
||
userID int64
|
||
registeredDay string
|
||
dim countryRegion
|
||
registeredAtMS int64
|
||
}
|
||
|
||
type payerRow struct {
|
||
day string
|
||
userID int64
|
||
dim countryRegion
|
||
firstMS int64
|
||
}
|
||
|
||
type activeRow struct {
|
||
day string
|
||
userID int64
|
||
dim countryRegion
|
||
firstMS int64
|
||
}
|
||
|
||
type micRow struct {
|
||
day string
|
||
userID int64
|
||
dim countryRegion
|
||
micOnlineMS int64
|
||
}
|
||
|
||
type rebuildState struct {
|
||
appDays map[appDayKey]*appDayAgg
|
||
registrations []userRegistrationRow
|
||
registeredByUserDay map[string]struct{}
|
||
payersByKey map[string]payerRow
|
||
activeByKey map[string]activeRow
|
||
micByKey map[string]micRow
|
||
userIDs map[int64]struct{}
|
||
userDims map[int64]countryRegion
|
||
knownUsers map[int64]struct{}
|
||
}
|
||
|
||
func main() {
|
||
ctx := context.Background()
|
||
var (
|
||
appFlag = flag.String("app", envDefault("APP_CODE", "lalu"), "app_code to backfill")
|
||
tzFlag = flag.String("timezone", envDefault("BACKFILL_TIMEZONE", "UTC"), "day boundary timezone, e.g. UTC or Asia/Shanghai")
|
||
startFlag = flag.String("start-day", os.Getenv("START_DAY"), "inclusive yyyy-mm-dd in timezone; defaults to last 7 days")
|
||
endFlag = flag.String("end-day", os.Getenv("END_DAY"), "inclusive yyyy-mm-dd in timezone; defaults to today")
|
||
statsDSNFlag = flag.String("statistics-dsn", firstNonEmpty(os.Getenv("STATISTICS_MYSQL_DSN"), os.Getenv("MYSQL_DSN")), "statistics mysql dsn")
|
||
userDSNFlag = flag.String("user-dsn", os.Getenv("USER_MYSQL_DSN"), "user-service mysql dsn")
|
||
walletDSNFlag = flag.String("wallet-dsn", os.Getenv("WALLET_MYSQL_DSN"), "wallet-service mysql dsn")
|
||
applyFlag = flag.Bool("apply", envBool("APPLY"), "apply replacement; default dry-run")
|
||
)
|
||
flag.Parse()
|
||
|
||
if strings.TrimSpace(*statsDSNFlag) == "" || strings.TrimSpace(*userDSNFlag) == "" || strings.TrimSpace(*walletDSNFlag) == "" {
|
||
log.Fatal("statistics-dsn, user-dsn and wallet-dsn are required")
|
||
}
|
||
loc, err := time.LoadLocation(strings.TrimSpace(*tzFlag))
|
||
if err != nil {
|
||
log.Fatalf("load timezone: %v", err)
|
||
}
|
||
statTZ, err := normalizeBackfillStatTZ(loc.String())
|
||
if err != nil {
|
||
log.Fatal(err)
|
||
}
|
||
startDay, endDay, windowStartMS, windowEndMS, err := resolveWindow(*startFlag, *endFlag, loc)
|
||
if err != nil {
|
||
log.Fatal(err)
|
||
}
|
||
app := appcode.Normalize(*appFlag)
|
||
|
||
statsDB := mustOpen(ctx, *statsDSNFlag)
|
||
defer statsDB.Close()
|
||
userDB := mustOpen(ctx, *userDSNFlag)
|
||
defer userDB.Close()
|
||
walletDB := mustOpen(ctx, *walletDSNFlag)
|
||
defer walletDB.Close()
|
||
|
||
state := &rebuildState{
|
||
appDays: map[appDayKey]*appDayAgg{},
|
||
registeredByUserDay: map[string]struct{}{},
|
||
payersByKey: map[string]payerRow{},
|
||
activeByKey: map[string]activeRow{},
|
||
micByKey: map[string]micRow{},
|
||
userIDs: map[int64]struct{}{},
|
||
userDims: map[int64]countryRegion{},
|
||
knownUsers: map[int64]struct{}{},
|
||
}
|
||
// 第一阶段只收集窗口内会参与重算的 user_id，后续统一批量加载用户国家和区域，避免按充值/活跃记录逐条访问 user-service 数据库。
|
||
if err := collectRegistrationUsers(ctx, userDB, state, app, windowStartMS, windowEndMS); err != nil {
|
||
log.Fatalf("collect registrations: %v", err)
|
||
}
|
||
if err := collectActiveUserIDs(ctx, statsDB, state, app, statTZ, startDay, endDay); err != nil {
|
||
log.Fatalf("collect active ids: %v", err)
|
||
}
|
||
if err := collectRechargeUserIDs(ctx, walletDB, state, app, windowStartMS, windowEndMS); err != nil {
|
||
log.Fatalf("collect recharge ids: %v", err)
|
||
}
|
||
if err := collectReportMetricUserIDs(ctx, walletDB, userDB, state, app, startDay, endDay, windowEndMS); err != nil {
|
||
log.Fatalf("collect report metric ids: %v", err)
|
||
}
|
||
if err := loadUserDims(ctx, userDB, state, app); err != nil {
|
||
log.Fatalf("load user dims: %v", err)
|
||
}
|
||
if err := rebuildRegistrations(ctx, userDB, state, app, windowStartMS, windowEndMS, loc); err != nil {
|
||
log.Fatalf("rebuild registrations: %v", err)
|
||
}
|
||
if err := rebuildActive(ctx, statsDB, state, app, statTZ, startDay, endDay); err != nil {
|
||
log.Fatalf("rebuild active: %v", err)
|
||
}
|
||
if err := rebuildWalletRecharge(ctx, walletDB, state, app, windowStartMS, windowEndMS, loc); err != nil {
|
||
log.Fatalf("rebuild wallet recharge: %v", err)
|
||
}
|
||
if err := rebuildCoinSellerStock(ctx, walletDB, state, app, windowStartMS, windowEndMS, loc); err != nil {
|
||
log.Fatalf("rebuild coin seller stock: %v", err)
|
||
}
|
||
if err := rebuildWalletReportMetrics(ctx, walletDB, state, app, startDay, endDay, windowStartMS, windowEndMS, loc); err != nil {
|
||
log.Fatalf("rebuild wallet report metrics: %v", err)
|
||
}
|
||
if err := rebuildSalary(ctx, walletDB, state, app, windowStartMS, windowEndMS, loc); err != nil {
|
||
log.Fatalf("rebuild salary: %v", err)
|
||
}
|
||
if err := rebuildMicStats(ctx, userDB, state, app, startDay, endDay); err != nil {
|
||
log.Fatalf("rebuild mic stats: %v", err)
|
||
}
|
||
|
||
if err := stageAndMaybeApply(ctx, statsDB, state, app, statTZ, startDay, endDay, *applyFlag); err != nil {
|
||
log.Fatalf("stage/apply: %v", err)
|
||
}
|
||
mode := "dry-run"
|
||
if *applyFlag {
|
||
mode = "applied"
|
||
}
|
||
fmt.Printf("%s app=%s timezone=%s window=%s..%s app_day_rows=%d registrations=%d active=%d payers=%d\n",
|
||
mode, app, statTZ, startDay, endDay, len(state.appDays), len(state.registrations), len(state.activeByKey), len(state.payersByKey))
|
||
}
|
||
|
||
func normalizeBackfillStatTZ(value string) (string, error) {
|
||
switch strings.TrimSpace(value) {
|
||
case "UTC":
|
||
return "UTC", nil
|
||
case "Asia/Shanghai":
|
||
return "Asia/Shanghai", nil
|
||
default:
|
||
return "", fmt.Errorf("unsupported stat_tz %q; only UTC and Asia/Shanghai are queryable", value)
|
||
}
|
||
}
|
||
|
||
func envDefault(key string, fallback string) string {
|
||
if value := strings.TrimSpace(os.Getenv(key)); value != "" {
|
||
return value
|
||
}
|
||
return fallback
|
||
}
|
||
|
||
func envBool(key string) bool {
|
||
switch strings.ToLower(strings.TrimSpace(os.Getenv(key))) {
|
||
case "1", "true", "yes", "y":
|
||
return true
|
||
default:
|
||
return false
|
||
}
|
||
}
|
||
|
||
func firstNonEmpty(values ...string) string {
|
||
for _, value := range values {
|
||
if strings.TrimSpace(value) != "" {
|
||
return strings.TrimSpace(value)
|
||
}
|
||
}
|
||
return ""
|
||
}
|
||
|
||
func mustOpen(ctx context.Context, dsn string) *sql.DB {
|
||
db, err := sql.Open("mysql", strings.TrimSpace(dsn))
|
||
if err != nil {
|
||
log.Fatal(err)
|
||
}
|
||
if err := db.PingContext(ctx); err != nil {
|
||
log.Fatal(err)
|
||
}
|
||
return db
|
||
}
|
||
|
||
func resolveWindow(rawStart, rawEnd string, loc *time.Location) (string, string, int64, int64, error) {
|
||
now := time.Now().In(loc)
|
||
endDayTime := time.Date(now.Year(), now.Month(), now.Day(), 0, 0, 0, 0, loc)
|
||
startDayTime := endDayTime.AddDate(0, 0, -6)
|
||
var err error
|
||
if strings.TrimSpace(rawEnd) != "" {
|
||
endDayTime, err = parseDay(rawEnd, loc)
|
||
if err != nil {
|
||
return "", "", 0, 0, err
|
||
}
|
||
}
|
||
if strings.TrimSpace(rawStart) != "" {
|
||
startDayTime, err = parseDay(rawStart, loc)
|
||
if err != nil {
|
||
return "", "", 0, 0, err
|
||
}
|
||
}
|
||
if startDayTime.After(endDayTime) {
|
||
return "", "", 0, 0, fmt.Errorf("start-day must not be after end-day")
|
||
}
|
||
windowEnd := endDayTime.AddDate(0, 0, 1)
|
||
return formatDay(startDayTime), formatDay(endDayTime), startDayTime.UTC().UnixMilli(), windowEnd.UTC().UnixMilli(), nil
|
||
}
|
||
|
||
func parseDay(value string, loc *time.Location) (time.Time, error) {
|
||
parsed, err := time.ParseInLocation("2006-01-02", strings.TrimSpace(value), loc)
|
||
if err != nil {
|
||
return time.Time{}, err
|
||
}
|
||
return time.Date(parsed.Year(), parsed.Month(), parsed.Day(), 0, 0, 0, 0, loc), nil
|
||
}
|
||
|
||
func dayFromMS(ms int64, loc *time.Location) string {
|
||
return formatDay(time.UnixMilli(ms).In(loc))
|
||
}
|
||
|
||
func formatDay(value time.Time) string {
|
||
return value.Format("2006-01-02")
|
||
}
|
||
|
||
func collectRegistrationUsers(ctx context.Context, db *sql.DB, state *rebuildState, app string, startMS, endMS int64) error {
|
||
// 全站机器人和后台快捷账号使用 users 资料参与运营、联调或游戏展示，但不是 App 自然注册用户；补数入口必须和实时 UserRegistered 口径一致排除。
|
||
rows, err := db.QueryContext(ctx, `
|
||
SELECT u.user_id
|
||
FROM users u
|
||
WHERE u.app_code = ?
|
||
AND u.profile_completed = 1
|
||
AND COALESCE(NULLIF(u.profile_completed_at_ms, 0), u.created_at_ms) >= ?
|
||
AND COALESCE(NULLIF(u.profile_completed_at_ms, 0), u.created_at_ms) < ?
|
||
AND LOWER(COALESCE(u.source, '')) NOT IN ('game_robot', 'quick_account')`,
|
||
app, startMS, endMS)
|
||
if err != nil {
|
||
return err
|
||
}
|
||
defer rows.Close()
|
||
for rows.Next() {
|
||
var userID int64
|
||
if err := rows.Scan(&userID); err != nil {
|
||
return err
|
||
}
|
||
state.userIDs[userID] = struct{}{}
|
||
}
|
||
return rows.Err()
|
||
}
|
||
|
||
func collectActiveUserIDs(ctx context.Context, db *sql.DB, state *rebuildState, app, statTZ, startDay, endDay string) error {
|
||
rows, err := db.QueryContext(ctx, `
|
||
SELECT DISTINCT user_id
|
||
FROM stat_user_day_activity
|
||
WHERE app_code = ? AND stat_tz = ? AND stat_day BETWEEN ? AND ?`,
|
||
app, statTZ, startDay, endDay)
|
||
if err != nil {
|
||
return err
|
||
}
|
||
defer rows.Close()
|
||
for rows.Next() {
|
||
var userID int64
|
||
if err := rows.Scan(&userID); err != nil {
|
||
return err
|
||
}
|
||
state.userIDs[userID] = struct{}{}
|
||
}
|
||
return rows.Err()
|
||
}
|
||
|
||
func collectRechargeUserIDs(ctx context.Context, db *sql.DB, state *rebuildState, app string, startMS, endMS int64) error {
|
||
for _, query := range []string{
|
||
`SELECT DISTINCT user_id FROM wallet_recharge_records WHERE app_code = ? AND created_at_ms >= ? AND created_at_ms < ?`,
|
||
`SELECT DISTINCT seller_user_id FROM coin_seller_stock_records WHERE app_code = ? AND counts_as_seller_recharge = 1 AND created_at_ms >= ? AND created_at_ms < ?`,
|
||
} {
|
||
rows, err := db.QueryContext(ctx, query, app, startMS, endMS)
|
||
if err != nil {
|
||
return err
|
||
}
|
||
for rows.Next() {
|
||
var userID int64
|
||
if err := rows.Scan(&userID); err != nil {
|
||
rows.Close()
|
||
return err
|
||
}
|
||
state.userIDs[userID] = struct{}{}
|
||
}
|
||
if err := rows.Close(); err != nil {
|
||
return err
|
||
}
|
||
}
|
||
return nil
|
||
}
|
||
|
||
func collectReportMetricUserIDs(ctx context.Context, walletDB *sql.DB, userDB *sql.DB, state *rebuildState, app string, startDay string, endDay string, windowEndMS int64) error {
|
||
queries := []struct {
|
||
db *sql.DB
|
||
query string
|
||
args []any
|
||
}{
|
||
{
|
||
db: walletDB,
|
||
query: `
|
||
SELECT DISTINCT e.user_id
|
||
FROM wallet_entries e
|
||
JOIN wallet_transactions wt ON wt.transaction_id = e.transaction_id
|
||
WHERE wt.app_code = ? AND e.asset_type IN ('COIN', 'COIN_SELLER_COIN', 'HOST_SALARY_USD', 'AGENCY_SALARY_USD') AND e.created_at_ms < ?
|
||
UNION
|
||
SELECT DISTINCT e.counterparty_user_id
|
||
FROM wallet_entries e
|
||
JOIN wallet_transactions wt ON wt.transaction_id = e.transaction_id
|
||
WHERE wt.app_code = ? AND wt.biz_type = 'salary_transfer_to_coin_seller'
|
||
AND e.asset_type = 'COIN_SELLER_COIN' AND e.counterparty_user_id > 0 AND e.created_at_ms < ?`,
|
||
args: []any{app, windowEndMS, app, windowEndMS},
|
||
},
|
||
{
|
||
db: walletDB,
|
||
query: `
|
||
SELECT DISTINCT user_id
|
||
FROM wallet_accounts
|
||
WHERE app_code = ? AND asset_type = 'COIN' AND available_amount + frozen_amount <> 0`,
|
||
args: []any{app},
|
||
},
|
||
{
|
||
db: walletDB,
|
||
query: `
|
||
SELECT DISTINCT user_id
|
||
FROM host_salary_settlement_records
|
||
WHERE app_code = ? AND status = 'succeeded' AND created_at_ms < ?`,
|
||
args: []any{app, windowEndMS},
|
||
},
|
||
{
|
||
db: walletDB,
|
||
query: `
|
||
SELECT DISTINCT agency_owner_user_id
|
||
FROM host_salary_settlement_records
|
||
WHERE app_code = ? AND status = 'succeeded' AND agency_owner_user_id > 0 AND created_at_ms < ?`,
|
||
args: []any{app, windowEndMS},
|
||
},
|
||
{
|
||
db: userDB,
|
||
query: `
|
||
SELECT DISTINCT user_id
|
||
FROM user_mic_daily_stats
|
||
WHERE app_code = ? AND stat_date BETWEEN ? AND ?`,
|
||
args: []any{app, startDay, endDay},
|
||
},
|
||
}
|
||
for _, item := range queries {
|
||
rows, err := item.db.QueryContext(ctx, item.query, item.args...)
|
||
if err != nil {
|
||
return err
|
||
}
|
||
for rows.Next() {
|
||
var userID int64
|
||
if err := rows.Scan(&userID); err != nil {
|
||
rows.Close()
|
||
return err
|
||
}
|
||
if userID > 0 {
|
||
state.userIDs[userID] = struct{}{}
|
||
}
|
||
}
|
||
if err := rows.Close(); err != nil {
|
||
return err
|
||
}
|
||
}
|
||
return nil
|
||
}
|
||
|
||
func loadUserDims(ctx context.Context, db *sql.DB, state *rebuildState, app string) error {
|
||
userIDs := make([]int64, 0, len(state.userIDs))
|
||
for userID := range state.userIDs {
|
||
userIDs = append(userIDs, userID)
|
||
}
|
||
sort.Slice(userIDs, func(i, j int) bool { return userIDs[i] < userIDs[j] })
|
||
for start := 0; start < len(userIDs); start += 500 {
|
||
end := start + 500
|
||
if end > len(userIDs) {
|
||
end = len(userIDs)
|
||
}
|
||
placeholders := strings.TrimRight(strings.Repeat("?,", end-start), ",")
|
||
args := []any{app}
|
||
for _, userID := range userIDs[start:end] {
|
||
args = append(args, userID)
|
||
}
|
||
rows, err := db.QueryContext(ctx, `
|
||
SELECT u.user_id, COALESCE(c.country_id, 0), COALESCE(u.region_id, 0)
|
||
FROM users u
|
||
LEFT JOIN countries c ON c.app_code = u.app_code AND c.country_code = u.country
|
||
WHERE u.app_code = ? AND u.user_id IN (`+placeholders+`)`, args...)
|
||
if err != nil {
|
||
return err
|
||
}
|
||
for rows.Next() {
|
||
var userID int64
|
||
var dim countryRegion
|
||
if err := rows.Scan(&userID, &dim.countryID, &dim.regionID); err != nil {
|
||
rows.Close()
|
||
return err
|
||
}
|
||
state.userDims[userID] = dim
|
||
state.knownUsers[userID] = struct{}{}
|
||
}
|
||
if err := rows.Close(); err != nil {
|
||
return err
|
||
}
|
||
}
|
||
return nil
|
||
}
|
||
|
||
func rebuildRegistrations(ctx context.Context, db *sql.DB, state *rebuildState, app string, startMS, endMS int64, loc *time.Location) error {
|
||
// 注册国家以资料完成后的 users.country 映射为准；region_id 只作为独立维度写入，不能再兜底成国家。
|
||
// source=game_robot 是后台全站机器人池账号，source=quick_account 是后台快捷建号；两者都不是 App 自然注册，不能进入新增用户、注册 cohort 或留存口径。
|
||
rows, err := db.QueryContext(ctx, `
|
||
SELECT u.user_id, COALESCE(c.country_id, 0), COALESCE(u.region_id, 0),
|
||
COALESCE(NULLIF(u.profile_completed_at_ms, 0), u.created_at_ms) AS registered_at_ms
|
||
FROM users u
|
||
LEFT JOIN countries c ON c.app_code = u.app_code AND c.country_code = u.country
|
||
WHERE u.app_code = ?
|
||
AND u.profile_completed = 1
|
||
AND COALESCE(NULLIF(u.profile_completed_at_ms, 0), u.created_at_ms) >= ?
|
||
AND COALESCE(NULLIF(u.profile_completed_at_ms, 0), u.created_at_ms) < ?
|
||
AND LOWER(COALESCE(u.source, '')) NOT IN ('game_robot', 'quick_account')`,
|
||
app, startMS, endMS)
|
||
if err != nil {
|
||
return err
|
||
}
|
||
defer rows.Close()
|
||
for rows.Next() {
|
||
var row userRegistrationRow
|
||
if err := rows.Scan(&row.userID, &row.dim.countryID, &row.dim.regionID, &row.registeredAtMS); err != nil {
|
||
return err
|
||
}
|
||
row.registeredDay = dayFromMS(row.registeredAtMS, loc)
|
||
state.registrations = append(state.registrations, row)
|
||
state.registeredByUserDay[userDayKey(row.registeredDay, row.userID)] = struct{}{}
|
||
agg := state.ensureAgg(row.registeredDay, row.dim)
|
||
agg.newUsers++
|
||
// 产品口径要求完成注册即算当日活跃；补数时直接把注册事实写入活跃重建集合，
|
||
// 后续房间/游戏活跃若命中同一用户同一天，只会更新更早的 first_active_at_ms，不会重复计数。
|
||
state.addActive(row.registeredDay, row.userID, row.dim, row.registeredAtMS)
|
||
}
|
||
return rows.Err()
|
||
}
|
||
|
||
func rebuildActive(ctx context.Context, db *sql.DB, state *rebuildState, app, statTZ, startDay, endDay string) error {
|
||
// 房间/游戏活跃已经被实时消费进对应 stat_tz 的 stat_user_day_activity；回填只读取目标时区，避免用 UTC 活跃去污染中国自然日。
|
||
rows, err := db.QueryContext(ctx, `
|
||
SELECT DATE_FORMAT(stat_day, '%Y-%m-%d'), user_id, MIN(first_active_at_ms)
|
||
FROM stat_user_day_activity
|
||
WHERE app_code = ? AND stat_tz = ? AND stat_day BETWEEN ? AND ?
|
||
GROUP BY stat_day, user_id`,
|
||
app, statTZ, startDay, endDay)
|
||
if err != nil {
|
||
return err
|
||
}
|
||
defer rows.Close()
|
||
for rows.Next() {
|
||
var day string
|
||
var userID, firstMS int64
|
||
if err := rows.Scan(&day, &userID, &firstMS); err != nil {
|
||
return err
|
||
}
|
||
if _, ok := state.knownUsers[userID]; !ok {
|
||
// stat_user_day_activity 是历史活跃读模型，不是用户事实 owner；回填时必须丢弃 user-service 已不存在的孤儿活跃，避免脏事实继续汇总到 country_id=0。
|
||
continue
|
||
}
|
||
state.addActive(day, userID, state.userDims[userID], firstMS)
|
||
}
|
||
return rows.Err()
|
||
}
|
||
|
||
func rebuildWalletRecharge(ctx context.Context, db *sql.DB, state *rebuildState, app string, startMS, endMS int64, loc *time.Location) error {
|
||
// wallet_recharge_records 是到账事实表；Google 也会写入该表，payment_orders 只是 provider 审计，不能在回填里再扫一次导致双算。
|
||
// 币商向用户转账只代表金币充值用户，不产生用户 USDT 充值；Google Play 则按记录里的 usd_minor_amount 计入用户充值。
|
||
rows, err := db.QueryContext(ctx, `
|
||
SELECT rr.user_id, rr.recharge_sequence, rr.coin_amount, rr.usd_minor_amount, rr.created_at_ms, wt.biz_type
|
||
FROM wallet_recharge_records rr
|
||
JOIN wallet_transactions wt ON wt.app_code = rr.app_code AND wt.transaction_id = rr.transaction_id
|
||
WHERE rr.app_code = ? AND rr.created_at_ms >= ? AND rr.created_at_ms < ?`,
|
||
app, startMS, endMS)
|
||
if err != nil {
|
||
return err
|
||
}
|
||
defer rows.Close()
|
||
for rows.Next() {
|
||
var userID, sequence, coinAmount, usdMinor, createdAtMS int64
|
||
var bizType string
|
||
if err := rows.Scan(&userID, &sequence, &coinAmount, &usdMinor, &createdAtMS, &bizType); err != nil {
|
||
return err
|
||
}
|
||
day := dayFromMS(createdAtMS, loc)
|
||
dim := state.userDims[userID]
|
||
state.addPayer(day, userID, dim, createdAtMS)
|
||
agg := state.ensureAgg(day, dim)
|
||
switch strings.ToLower(strings.TrimSpace(bizType)) {
|
||
case "coin_seller_transfer":
|
||
agg.coinSellerTransferCoin += coinAmount
|
||
case "google_play_recharge":
|
||
agg.rechargeUSDMinor += usdMinor
|
||
agg.googleRechargeUSDMinor += usdMinor
|
||
case "mifapay", "v5pay":
|
||
agg.rechargeUSDMinor += usdMinor
|
||
agg.mifaPayRechargeUSDMinor += usdMinor
|
||
default:
|
||
if usdMinor > 0 {
|
||
agg.rechargeUSDMinor += usdMinor
|
||
}
|
||
}
|
||
if sequence == 1 && usdMinor > 0 {
|
||
agg.newUserRechargeUSDMinor += usdMinor
|
||
}
|
||
}
|
||
return rows.Err()
|
||
}
|
||
|
||
func rebuildCoinSellerStock(ctx context.Context, db *sql.DB, state *rebuildState, app string, startMS, endMS int64, loc *time.Location) error {
|
||
// 币商进货是平台收到的 USDT 充值，按 seller 的国家/区域进入总充值，但不写入用户金币转账金额。
|
||
rows, err := db.QueryContext(ctx, `
|
||
SELECT seller_user_id, paid_amount_micro, coin_amount, created_at_ms
|
||
FROM coin_seller_stock_records
|
||
WHERE app_code = ? AND counts_as_seller_recharge = 1 AND (paid_amount_micro <> 0 OR coin_amount <> 0) AND created_at_ms >= ? AND created_at_ms < ?`,
|
||
app, startMS, endMS)
|
||
if err != nil {
|
||
return err
|
||
}
|
||
defer rows.Close()
|
||
for rows.Next() {
|
||
var sellerUserID, amountMicro, coinAmount, createdAtMS int64
|
||
if err := rows.Scan(&sellerUserID, &amountMicro, &coinAmount, &createdAtMS); err != nil {
|
||
return err
|
||
}
|
||
usdMinor := amountMicro / 10_000
|
||
day := dayFromMS(createdAtMS, loc)
|
||
dim := state.userDims[sellerUserID]
|
||
agg := state.ensureAgg(day, dim)
|
||
agg.rechargeUSDMinor += usdMinor
|
||
agg.coinSellerRechargeUSDMinor += usdMinor
|
||
agg.coinSellerStockCoin += coinAmount
|
||
}
|
||
return rows.Err()
|
||
}
|
||
|
||
func rebuildWalletReportMetrics(ctx context.Context, db *sql.DB, state *rebuildState, app string, startDay string, endDay string, startMS int64, endMS int64, loc *time.Location) error {
|
||
if err := rebuildCoinTotals(ctx, db, state, app, startDay, endDay, endMS, loc); err != nil {
|
||
return err
|
||
}
|
||
rows, err := db.QueryContext(ctx, `
|
||
SELECT e.user_id, e.counterparty_user_id, e.asset_type, e.available_delta + e.frozen_delta AS amount_delta, wt.biz_type, e.created_at_ms
|
||
FROM wallet_entries e
|
||
JOIN wallet_transactions wt ON wt.transaction_id = e.transaction_id
|
||
WHERE wt.app_code = ? AND e.asset_type IN ('COIN', 'COIN_SELLER_COIN') AND e.created_at_ms >= ? AND e.created_at_ms < ?`,
|
||
app, startMS, endMS)
|
||
if err != nil {
|
||
return err
|
||
}
|
||
defer rows.Close()
|
||
for rows.Next() {
|
||
var userID, counterpartyUserID, amountDelta, createdAtMS int64
|
||
var assetType, bizType string
|
||
if err := rows.Scan(&userID, &counterpartyUserID, &assetType, &amountDelta, &bizType, &createdAtMS); err != nil {
|
||
return err
|
||
}
|
||
day := dayFromMS(createdAtMS, loc)
|
||
normalizedBiz := strings.ToLower(strings.TrimSpace(bizType))
|
||
normalizedAsset := strings.ToUpper(strings.TrimSpace(assetType))
|
||
if amountDelta > 0 && normalizedBiz == "salary_transfer_to_coin_seller" && normalizedAsset == "COIN_SELLER_COIN" {
|
||
dimensionUserID := userID
|
||
if counterpartyUserID > 0 {
|
||
dimensionUserID = counterpartyUserID
|
||
}
|
||
agg := state.ensureAgg(day, state.userDims[dimensionUserID])
|
||
agg.salaryTransferCoin += amountDelta
|
||
continue
|
||
}
|
||
if normalizedAsset != "COIN" {
|
||
continue
|
||
}
|
||
dim := state.userDims[userID]
|
||
agg := state.ensureAgg(day, dim)
|
||
switch {
|
||
case amountDelta < 0 && isConsumedCoinBizType(normalizedBiz):
|
||
agg.consumedCoin += -amountDelta
|
||
case amountDelta > 0 && normalizedBiz == "manual_credit":
|
||
agg.manualGrantCoin += amountDelta
|
||
case amountDelta > 0 && isPlatformGrantBizType(normalizedBiz):
|
||
agg.platformGrantCoin += amountDelta
|
||
}
|
||
}
|
||
return rows.Err()
|
||
}
|
||
|
||
type coinBalanceEntry struct {
|
||
userID int64
|
||
balance int64
|
||
createdAtMS int64
|
||
}
|
||
|
||
func rebuildCoinTotals(ctx context.Context, db *sql.DB, state *rebuildState, app string, startDay string, endDay string, windowEndMS int64, loc *time.Location) error {
|
||
dayEnds, err := dayEndPoints(startDay, endDay, loc)
|
||
if err != nil {
|
||
return err
|
||
}
|
||
rows, err := db.QueryContext(ctx, `
|
||
SELECT e.user_id, e.available_after + e.frozen_after AS balance_after, e.created_at_ms
|
||
FROM wallet_entries e
|
||
JOIN wallet_transactions wt ON wt.transaction_id = e.transaction_id
|
||
WHERE wt.app_code = ? AND e.asset_type = 'COIN' AND e.created_at_ms < ?
|
||
ORDER BY e.user_id ASC, e.created_at_ms ASC, e.entry_id ASC`,
|
||
app, windowEndMS)
|
||
if err != nil {
|
||
return err
|
||
}
|
||
defer rows.Close()
|
||
var currentUserID int64
|
||
entries := []coinBalanceEntry{}
|
||
flush := func() {
|
||
if currentUserID <= 0 || len(entries) == 0 {
|
||
return
|
||
}
|
||
dim := state.userDims[currentUserID]
|
||
index := 0
|
||
balance := int64(0)
|
||
hasBalance := false
|
||
for _, point := range dayEnds {
|
||
for index < len(entries) && entries[index].createdAtMS < point.endMS {
|
||
balance = entries[index].balance
|
||
hasBalance = true
|
||
index++
|
||
}
|
||
if hasBalance {
|
||
state.ensureAgg(point.day, dim).coinTotal += balance
|
||
}
|
||
}
|
||
}
|
||
for rows.Next() {
|
||
var entry coinBalanceEntry
|
||
if err := rows.Scan(&entry.userID, &entry.balance, &entry.createdAtMS); err != nil {
|
||
return err
|
||
}
|
||
if currentUserID != 0 && entry.userID != currentUserID {
|
||
flush()
|
||
entries = entries[:0]
|
||
}
|
||
currentUserID = entry.userID
|
||
entries = append(entries, entry)
|
||
}
|
||
if err := rows.Err(); err != nil {
|
||
return err
|
||
}
|
||
flush()
|
||
return nil
|
||
}
|
||
|
||
func rebuildSalary(ctx context.Context, db *sql.DB, state *rebuildState, app string, startMS int64, endMS int64, loc *time.Location) error {
|
||
rows, err := db.QueryContext(ctx, `
|
||
SELECT user_id, agency_owner_user_id, host_salary_usd_minor_delta, agency_salary_usd_minor_delta, created_at_ms
|
||
FROM host_salary_settlement_records
|
||
WHERE app_code = ? AND status = 'succeeded' AND created_at_ms >= ? AND created_at_ms < ?`,
|
||
app, startMS, endMS)
|
||
if err != nil {
|
||
return err
|
||
}
|
||
defer rows.Close()
|
||
for rows.Next() {
|
||
var hostUserID, agencyUserID, hostSalary, agencySalary, createdAtMS int64
|
||
if err := rows.Scan(&hostUserID, &agencyUserID, &hostSalary, &agencySalary, &createdAtMS); err != nil {
|
||
return err
|
||
}
|
||
day := dayFromMS(createdAtMS, loc)
|
||
if hostSalary > 0 {
|
||
state.ensureAgg(day, state.userDims[hostUserID]).salaryUSDMinor += hostSalary
|
||
}
|
||
if agencySalary > 0 && agencyUserID > 0 {
|
||
state.ensureAgg(day, state.userDims[agencyUserID]).salaryUSDMinor += agencySalary
|
||
}
|
||
}
|
||
return rows.Err()
|
||
}
|
||
|
||
func rebuildMicStats(ctx context.Context, db *sql.DB, state *rebuildState, app string, startDay string, endDay string) error {
|
||
rows, err := db.QueryContext(ctx, `
|
||
SELECT DATE_FORMAT(stat_date, '%Y-%m-%d'), user_id, mic_online_ms
|
||
FROM user_mic_daily_stats
|
||
WHERE app_code = ? AND stat_date BETWEEN ? AND ?`,
|
||
app, startDay, endDay)
|
||
if err != nil {
|
||
return err
|
||
}
|
||
defer rows.Close()
|
||
for rows.Next() {
|
||
var day string
|
||
var userID, micOnlineMS int64
|
||
if err := rows.Scan(&day, &userID, &micOnlineMS); err != nil {
|
||
return err
|
||
}
|
||
if micOnlineMS <= 0 {
|
||
continue
|
||
}
|
||
dim := state.userDims[userID]
|
||
state.addMic(day, userID, dim, micOnlineMS)
|
||
}
|
||
return rows.Err()
|
||
}
|
||
|
||
type dayEndPoint struct {
|
||
day string
|
||
endMS int64
|
||
}
|
||
|
||
func dayEndPoints(startDay string, endDay string, loc *time.Location) ([]dayEndPoint, error) {
|
||
start, err := parseDay(startDay, loc)
|
||
if err != nil {
|
||
return nil, err
|
||
}
|
||
end, err := parseDay(endDay, loc)
|
||
if err != nil {
|
||
return nil, err
|
||
}
|
||
points := []dayEndPoint{}
|
||
for cursor := start; !cursor.After(end); cursor = cursor.AddDate(0, 0, 1) {
|
||
points = append(points, dayEndPoint{
|
||
day: formatDay(cursor),
|
||
endMS: cursor.AddDate(0, 0, 1).UTC().UnixMilli(),
|
||
})
|
||
}
|
||
return points, nil
|
||
}
|
||
|
||
func isConsumedCoinBizType(value string) bool {
|
||
switch strings.ToLower(strings.TrimSpace(value)) {
|
||
case "gift_debit", "direct_gift_debit", "vip_purchase", "resource_shop_purchase",
|
||
"red_packet_create", "game_debit", "wheel_draw_debit", "cp_breakup_fee":
|
||
return true
|
||
default:
|
||
return false
|
||
}
|
||
}
|
||
|
||
func isPlatformGrantBizType(value string) bool {
|
||
switch strings.ToLower(strings.TrimSpace(value)) {
|
||
case "task_reward", "lucky_gift_reward", "wheel_reward", "room_turnover_reward",
|
||
"invite_activity_reward", "agency_opening_reward":
|
||
return true
|
||
default:
|
||
return false
|
||
}
|
||
}
|
||
|
||
func (s *rebuildState) ensureAgg(day string, dim countryRegion) *appDayAgg {
|
||
key := appDayKey{day: day, countryID: dim.countryID, regionID: dim.regionID}
|
||
if s.appDays[key] == nil {
|
||
s.appDays[key] = &appDayAgg{}
|
||
}
|
||
return s.appDays[key]
|
||
}
|
||
|
||
func (s *rebuildState) addPayer(day string, userID int64, dim countryRegion, firstMS int64) {
|
||
key := fmt.Sprintf("%s:%d", day, userID)
|
||
if existing, exists := s.payersByKey[key]; exists && existing.firstMS <= firstMS {
|
||
return
|
||
}
|
||
if _, exists := s.payersByKey[key]; !exists {
|
||
s.ensureAgg(day, dim).paidUsers++
|
||
}
|
||
s.payersByKey[key] = payerRow{day: day, userID: userID, dim: dim, firstMS: firstMS}
|
||
}
|
||
|
||
func (s *rebuildState) addActive(day string, userID int64, dim countryRegion, firstMS int64) {
|
||
key := userDayKey(day, userID)
|
||
if existing, exists := s.activeByKey[key]; exists {
|
||
// 注册、进房和游戏都能证明同一用户同一天活跃；主键按用户日收敛，聚合只加一次，
|
||
// 但 first_active_at_ms 要保留最早事实，便于后续排查“为什么算活跃”时能回到第一条证据。
|
||
if firstMS > 0 && (existing.firstMS == 0 || firstMS < existing.firstMS) {
|
||
existing.firstMS = firstMS
|
||
s.activeByKey[key] = existing
|
||
}
|
||
return
|
||
}
|
||
s.activeByKey[key] = activeRow{day: day, userID: userID, dim: dim, firstMS: firstMS}
|
||
s.ensureAgg(day, dim).activeUsers++
|
||
}
|
||
|
||
func (s *rebuildState) addMic(day string, userID int64, dim countryRegion, micOnlineMS int64) {
|
||
key := userDayKey(day, userID)
|
||
if existing, exists := s.micByKey[key]; exists {
|
||
agg := s.ensureAgg(day, existing.dim)
|
||
agg.micOnlineMS += micOnlineMS - existing.micOnlineMS
|
||
existing.micOnlineMS = micOnlineMS
|
||
s.micByKey[key] = existing
|
||
return
|
||
}
|
||
s.micByKey[key] = micRow{day: day, userID: userID, dim: dim, micOnlineMS: micOnlineMS}
|
||
agg := s.ensureAgg(day, dim)
|
||
agg.micOnlineMS += micOnlineMS
|
||
agg.micOnlineUsers++
|
||
}
|
||
|
||
func (s *rebuildState) userRegisteredOn(userID int64, day string) bool {
|
||
_, ok := s.registeredByUserDay[userDayKey(day, userID)]
|
||
return ok
|
||
}
|
||
|
||
func userDayKey(day string, userID int64) string {
|
||
return fmt.Sprintf("%s:%d", day, userID)
|
||
}
|
||
|
||
func stageAndMaybeApply(ctx context.Context, db *sql.DB, state *rebuildState, app, statTZ, startDay, endDay string, apply bool) error {
|
||
conn, err := db.Conn(ctx)
|
||
if err != nil {
|
||
return err
|
||
}
|
||
defer conn.Close()
|
||
tx, err := conn.BeginTx(ctx, nil)
|
||
if err != nil {
|
||
return err
|
||
}
|
||
defer tx.Rollback()
|
||
if err := createTempTables(ctx, tx); err != nil {
|
||
return err
|
||
}
|
||
if err := insertStageRows(ctx, tx, state, app, statTZ); err != nil {
|
||
return err
|
||
}
|
||
if err := validateStage(ctx, tx); err != nil {
|
||
return err
|
||
}
|
||
if !apply {
|
||
return tx.Rollback()
|
||
}
|
||
nowMS := time.Now().UTC().UnixMilli()
|
||
// 本 job 只覆盖能从当前源表确定的列；礼物和游戏等不在本次源表范围内的指标保留原值，且只替换目标 stat_tz，避免重跑中国日时覆盖 UTC 读模型。
|
||
if _, err := tx.ExecContext(ctx, `
|
||
UPDATE stat_app_day_country
|
||
SET new_users = 0, active_users = 0, paid_users = 0,
|
||
recharge_usd_minor = 0, new_user_recharge_usd_minor = 0,
|
||
coin_seller_recharge_usd_minor = 0,
|
||
coin_seller_stock_coin = 0,
|
||
mifapay_recharge_usd_minor = 0, google_recharge_usd_minor = 0, coin_seller_transfer_coin = 0,
|
||
coin_total = 0, consumed_coin = 0, platform_grant_coin = 0, manual_grant_coin = 0,
|
||
salary_usd_minor = 0, salary_transfer_coin = 0, mic_online_ms = 0, mic_online_users = 0,
|
||
updated_at_ms = ?
|
||
WHERE app_code = ? AND stat_tz = ? AND stat_day BETWEEN ? AND ?`,
|
||
nowMS, app, statTZ, startDay, endDay); err != nil {
|
||
return err
|
||
}
|
||
for _, statement := range []string{
|
||
`DELETE FROM stat_user_registration WHERE app_code = ? AND stat_tz = ? AND registered_day BETWEEN ? AND ?`,
|
||
`DELETE FROM stat_user_day_activity WHERE app_code = ? AND stat_tz = ? AND stat_day BETWEEN ? AND ?`,
|
||
`DELETE FROM stat_recharge_day_payers WHERE app_code = ? AND stat_tz = ? AND stat_day BETWEEN ? AND ?`,
|
||
`DELETE FROM stat_user_day_mic WHERE app_code = ? AND stat_tz = ? AND stat_day BETWEEN ? AND ?`,
|
||
} {
|
||
if _, err := tx.ExecContext(ctx, statement, app, statTZ, startDay, endDay); err != nil {
|
||
return err
|
||
}
|
||
}
|
||
for userID, dim := range state.userDims {
|
||
if _, err := tx.ExecContext(ctx, `
|
||
INSERT INTO stat_user_dimension (app_code, user_id, country_id, region_id, updated_at_ms)
|
||
VALUES (?, ?, ?, ?, ?)
|
||
ON DUPLICATE KEY UPDATE
|
||
country_id = VALUES(country_id),
|
||
region_id = VALUES(region_id),
|
||
updated_at_ms = VALUES(updated_at_ms)
|
||
`, app, userID, dim.countryID, dim.regionID, nowMS); err != nil {
|
||
return err
|
||
}
|
||
}
|
||
if _, err := tx.ExecContext(ctx, `
|
||
INSERT INTO stat_app_day_country (
|
||
app_code, stat_tz, stat_day, country_id, region_id, new_users, active_users, paid_users,
|
||
recharge_usd_minor, new_user_recharge_usd_minor, coin_seller_recharge_usd_minor,
|
||
coin_seller_stock_coin, mifapay_recharge_usd_minor, google_recharge_usd_minor, coin_seller_transfer_coin,
|
||
coin_total, consumed_coin, platform_grant_coin, manual_grant_coin,
|
||
salary_usd_minor, salary_transfer_coin, mic_online_ms, mic_online_users, updated_at_ms
|
||
)
|
||
SELECT app_code, stat_tz, stat_day, country_id, region_id, new_users, active_users, paid_users,
|
||
recharge_usd_minor, new_user_recharge_usd_minor, coin_seller_recharge_usd_minor,
|
||
coin_seller_stock_coin, mifapay_recharge_usd_minor, google_recharge_usd_minor, coin_seller_transfer_coin,
|
||
coin_total, consumed_coin, platform_grant_coin, manual_grant_coin,
|
||
salary_usd_minor, salary_transfer_coin, mic_online_ms, mic_online_users, ?
|
||
FROM tmp_stat_backfill_app_day
|
||
ON DUPLICATE KEY UPDATE
|
||
new_users = VALUES(new_users), active_users = VALUES(active_users), paid_users = VALUES(paid_users),
|
||
recharge_usd_minor = VALUES(recharge_usd_minor),
|
||
new_user_recharge_usd_minor = VALUES(new_user_recharge_usd_minor),
|
||
coin_seller_recharge_usd_minor = VALUES(coin_seller_recharge_usd_minor),
|
||
coin_seller_stock_coin = VALUES(coin_seller_stock_coin),
|
||
mifapay_recharge_usd_minor = VALUES(mifapay_recharge_usd_minor),
|
||
google_recharge_usd_minor = VALUES(google_recharge_usd_minor),
|
||
coin_seller_transfer_coin = VALUES(coin_seller_transfer_coin),
|
||
coin_total = VALUES(coin_total),
|
||
consumed_coin = VALUES(consumed_coin),
|
||
platform_grant_coin = VALUES(platform_grant_coin),
|
||
manual_grant_coin = VALUES(manual_grant_coin),
|
||
salary_usd_minor = VALUES(salary_usd_minor),
|
||
salary_transfer_coin = VALUES(salary_transfer_coin),
|
||
mic_online_ms = VALUES(mic_online_ms),
|
||
mic_online_users = VALUES(mic_online_users),
|
||
updated_at_ms = VALUES(updated_at_ms)`, nowMS); err != nil {
|
||
return err
|
||
}
|
||
if _, err := tx.ExecContext(ctx, `
|
||
INSERT INTO stat_user_registration (app_code, stat_tz, user_id, registered_day, country_id, region_id, registered_at_ms)
|
||
SELECT app_code, stat_tz, user_id, registered_day, country_id, region_id, registered_at_ms FROM tmp_stat_backfill_registration`); err != nil {
|
||
return err
|
||
}
|
||
if _, err := tx.ExecContext(ctx, `
|
||
INSERT INTO stat_user_day_activity (app_code, stat_tz, stat_day, country_id, region_id, user_id, first_active_at_ms)
|
||
SELECT app_code, stat_tz, stat_day, country_id, region_id, user_id, first_active_at_ms FROM tmp_stat_backfill_active`); err != nil {
|
||
return err
|
||
}
|
||
if _, err := tx.ExecContext(ctx, `
|
||
INSERT INTO stat_recharge_day_payers (app_code, stat_tz, stat_day, country_id, region_id, user_id, first_paid_at_ms)
|
||
SELECT app_code, stat_tz, stat_day, country_id, region_id, user_id, first_paid_at_ms FROM tmp_stat_backfill_payers`); err != nil {
|
||
return err
|
||
}
|
||
if _, err := tx.ExecContext(ctx, `
|
||
INSERT INTO stat_user_day_mic (app_code, stat_tz, stat_day, country_id, region_id, user_id, mic_online_ms, updated_at_ms)
|
||
SELECT app_code, stat_tz, stat_day, country_id, region_id, user_id, mic_online_ms, ? FROM tmp_stat_backfill_mic`, nowMS); err != nil {
|
||
return err
|
||
}
|
||
return tx.Commit()
|
||
}
|
||
|
||
func createTempTables(ctx context.Context, tx *sql.Tx) error {
|
||
statements := []string{
|
||
`CREATE TEMPORARY TABLE tmp_stat_backfill_app_day (
|
||
app_code VARCHAR(32) NOT NULL, stat_tz VARCHAR(64) NOT NULL, stat_day DATE NOT NULL, country_id BIGINT NOT NULL, region_id BIGINT NOT NULL,
|
||
new_users BIGINT NOT NULL, active_users BIGINT NOT NULL, paid_users BIGINT NOT NULL,
|
||
recharge_usd_minor BIGINT NOT NULL, new_user_recharge_usd_minor BIGINT NOT NULL,
|
||
coin_seller_recharge_usd_minor BIGINT NOT NULL, coin_seller_stock_coin BIGINT NOT NULL,
|
||
mifapay_recharge_usd_minor BIGINT NOT NULL, google_recharge_usd_minor BIGINT NOT NULL,
|
||
coin_seller_transfer_coin BIGINT NOT NULL,
|
||
coin_total BIGINT NOT NULL, consumed_coin BIGINT NOT NULL, platform_grant_coin BIGINT NOT NULL,
|
||
manual_grant_coin BIGINT NOT NULL, salary_usd_minor BIGINT NOT NULL, salary_transfer_coin BIGINT NOT NULL,
|
||
mic_online_ms BIGINT NOT NULL, mic_online_users BIGINT NOT NULL,
|
||
PRIMARY KEY(app_code, stat_tz, stat_day, country_id, region_id)
|
||
)`,
|
||
`CREATE TEMPORARY TABLE tmp_stat_backfill_registration (
|
||
app_code VARCHAR(32) NOT NULL, stat_tz VARCHAR(64) NOT NULL, user_id BIGINT NOT NULL, registered_day DATE NOT NULL,
|
||
country_id BIGINT NOT NULL, region_id BIGINT NOT NULL, registered_at_ms BIGINT NOT NULL,
|
||
PRIMARY KEY(app_code, stat_tz, user_id)
|
||
)`,
|
||
`CREATE TEMPORARY TABLE tmp_stat_backfill_active (
|
||
app_code VARCHAR(32) NOT NULL, stat_tz VARCHAR(64) NOT NULL, stat_day DATE NOT NULL, country_id BIGINT NOT NULL,
|
||
region_id BIGINT NOT NULL, user_id BIGINT NOT NULL, first_active_at_ms BIGINT NOT NULL,
|
||
PRIMARY KEY(app_code, stat_tz, stat_day, user_id)
|
||
)`,
|
||
`CREATE TEMPORARY TABLE tmp_stat_backfill_payers (
|
||
app_code VARCHAR(32) NOT NULL, stat_tz VARCHAR(64) NOT NULL, stat_day DATE NOT NULL, country_id BIGINT NOT NULL,
|
||
region_id BIGINT NOT NULL, user_id BIGINT NOT NULL, first_paid_at_ms BIGINT NOT NULL,
|
||
PRIMARY KEY(app_code, stat_tz, stat_day, user_id)
|
||
)`,
|
||
`CREATE TEMPORARY TABLE tmp_stat_backfill_mic (
|
||
app_code VARCHAR(32) NOT NULL, stat_tz VARCHAR(64) NOT NULL, stat_day DATE NOT NULL, country_id BIGINT NOT NULL,
|
||
region_id BIGINT NOT NULL, user_id BIGINT NOT NULL, mic_online_ms BIGINT NOT NULL,
|
||
PRIMARY KEY(app_code, stat_tz, stat_day, user_id)
|
||
)`,
|
||
}
|
||
for _, statement := range statements {
|
||
if _, err := tx.ExecContext(ctx, statement); err != nil {
|
||
return err
|
||
}
|
||
}
|
||
return nil
|
||
}
|
||
|
||
func insertStageRows(ctx context.Context, tx *sql.Tx, state *rebuildState, app string, statTZ string) error {
|
||
for key, agg := range state.appDays {
|
||
if _, err := tx.ExecContext(ctx, `
|
||
INSERT INTO tmp_stat_backfill_app_day (
|
||
app_code, stat_tz, stat_day, country_id, region_id, new_users, active_users, paid_users,
|
||
recharge_usd_minor, new_user_recharge_usd_minor, coin_seller_recharge_usd_minor,
|
||
coin_seller_stock_coin, mifapay_recharge_usd_minor, google_recharge_usd_minor, coin_seller_transfer_coin,
|
||
coin_total, consumed_coin, platform_grant_coin, manual_grant_coin, salary_usd_minor, salary_transfer_coin, mic_online_ms, mic_online_users
|
||
) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)`,
|
||
app, statTZ, key.day, key.countryID, key.regionID, agg.newUsers, agg.activeUsers, agg.paidUsers,
|
||
agg.rechargeUSDMinor, agg.newUserRechargeUSDMinor, agg.coinSellerRechargeUSDMinor,
|
||
agg.coinSellerStockCoin, agg.mifaPayRechargeUSDMinor, agg.googleRechargeUSDMinor, agg.coinSellerTransferCoin,
|
||
agg.coinTotal, agg.consumedCoin, agg.platformGrantCoin, agg.manualGrantCoin, agg.salaryUSDMinor, agg.salaryTransferCoin, agg.micOnlineMS, agg.micOnlineUsers); err != nil {
|
||
return err
|
||
}
|
||
}
|
||
for _, row := range state.registrations {
|
||
if _, err := tx.ExecContext(ctx, `
|
||
INSERT INTO tmp_stat_backfill_registration (app_code, stat_tz, user_id, registered_day, country_id, region_id, registered_at_ms)
|
||
VALUES (?, ?, ?, ?, ?, ?, ?)`, app, statTZ, row.userID, row.registeredDay, row.dim.countryID, row.dim.regionID, row.registeredAtMS); err != nil {
|
||
return err
|
||
}
|
||
}
|
||
for _, row := range state.activeByKey {
|
||
if _, err := tx.ExecContext(ctx, `
|
||
INSERT INTO tmp_stat_backfill_active (app_code, stat_tz, stat_day, country_id, region_id, user_id, first_active_at_ms)
|
||
VALUES (?, ?, ?, ?, ?, ?, ?)`, app, statTZ, row.day, row.dim.countryID, row.dim.regionID, row.userID, row.firstMS); err != nil {
|
||
return err
|
||
}
|
||
}
|
||
for _, row := range state.payersByKey {
|
||
if _, err := tx.ExecContext(ctx, `
|
||
INSERT INTO tmp_stat_backfill_payers (app_code, stat_tz, stat_day, country_id, region_id, user_id, first_paid_at_ms)
|
||
VALUES (?, ?, ?, ?, ?, ?, ?)`, app, statTZ, row.day, row.dim.countryID, row.dim.regionID, row.userID, row.firstMS); err != nil {
|
||
return err
|
||
}
|
||
}
|
||
for _, row := range state.micByKey {
|
||
if _, err := tx.ExecContext(ctx, `
|
||
INSERT INTO tmp_stat_backfill_mic (app_code, stat_tz, stat_day, country_id, region_id, user_id, mic_online_ms)
|
||
VALUES (?, ?, ?, ?, ?, ?, ?)`, app, statTZ, row.day, row.dim.countryID, row.dim.regionID, row.userID, row.micOnlineMS); err != nil {
|
||
return err
|
||
}
|
||
}
|
||
return nil
|
||
}
|
||
|
||
func validateStage(ctx context.Context, tx *sql.Tx) error {
|
||
var negativeCount int64
|
||
if err := tx.QueryRowContext(ctx, `
|
||
SELECT COUNT(*)
|
||
FROM tmp_stat_backfill_app_day
|
||
WHERE new_users < 0 OR active_users < 0 OR paid_users < 0 OR recharge_usd_minor < 0 OR coin_seller_transfer_coin < 0
|
||
OR coin_total < 0 OR consumed_coin < 0 OR platform_grant_coin < 0 OR manual_grant_coin < 0
|
||
OR salary_usd_minor < 0 OR salary_transfer_coin < 0 OR mic_online_ms < 0 OR mic_online_users < 0`).Scan(&negativeCount); err != nil {
|
||
return err
|
||
}
|
||
if negativeCount > 0 {
|
||
return fmt.Errorf("stage contains negative counters")
|
||
}
|
||
return nil
|
||
}
|