package storage import ( "context" "database/sql" "log/slog" "strconv" "strings" "time" "go.mongodb.org/mongo-driver/v2/bson" "go.mongodb.org/mongo-driver/v2/mongo" "go.mongodb.org/mongo-driver/v2/mongo/options" "dashboard-cdc-worker/internal/dashboard" ) const activeLogCollection = "user_daily_active_log" type activeLogRow struct { UserID int64 `bson:"userId"` ActiveDay string `bson:"activeDate"` SysOrigin string `bson:"sysOrigin"` RegisterCountryCode string `bson:"registerCountryCode"` } type activeLogEvent struct { UserID int64 SysOrigin string Country dashboard.Country } type activeDayStats struct { active map[string]map[int64]struct{} } type activeRetentionValues struct { countryName string activeUsers int64 d1User int64 d1Base int64 d7User int64 d7Base int64 d30User int64 d30Base int64 } type countryCount struct { name string count int64 } type retentionCohort struct { userID int64 sysOrigin string country dashboard.Country registerDate string } func (r *MySQLRepository) StartActiveReconciler(ctx context.Context, db *mongo.Database, logger *slog.Logger) { if !r.cfg.Active.Enabled || r.cfg.Active.Interval <= 0 { return } run := func() { if err := r.RebuildRecentActiveRetention(ctx, db); err != nil { if r.metrics != nil { r.metrics.RecordError(err) } logger.Error("dashboard active reconcile failed", "error", err) return } logger.Info("dashboard active reconcile applied", "lookbackDays", r.cfg.Active.LookbackDays) } go func() { run() ticker := time.NewTicker(r.cfg.Active.Interval) defer ticker.Stop() for { select { case <-ctx.Done(): return case <-ticker.C: run() } } }() } func (r *MySQLRepository) RebuildRecentActiveRetention(ctx context.Context, db *mongo.Database) error { location, err := time.LoadLocation(r.cfg.Dashboard.StorageTimezone) if err != nil { location = time.UTC } today := time.Now().In(location) end := time.Date(today.Year(), today.Month(), today.Day(), 0, 0, 0, 0, location).AddDate(0, 0, 1) start := end.AddDate(0, 0, -r.cfg.Active.LookbackDays) return r.RebuildActiveRetentionRange(ctx, db, start, end) } func (r *MySQLRepository) RebuildActiveRetentionRange(ctx context.Context, db *mongo.Database, start, end time.Time) error { events, err := r.loadActiveLogs(ctx, db, start, end) if err != nil { return err } return r.WithDashboardLock(ctx, r.cfg.CDC.BackfillLockWait, func() error { return r.WithTx(ctx, func(tx *sql.Tx) error { for day := start; day.Before(end); day = day.AddDate(0, 0, 1) { if err := r.replaceActiveRetentionDay(ctx, tx, db, day, events); err != nil { return err } } for _, statTimezone := range r.cfg.Dashboard.StatTimezones { if err := r.rebuildAllPeriodFromDays(ctx, tx, statTimezone); err != nil { return err } } return nil }) }) } func (r *MySQLRepository) loadActiveLogs(ctx context.Context, db *mongo.Database, start, end time.Time) (map[string]map[int64]activeLogEvent, error) { filter := bson.D{ {Key: "activeDate", Value: bson.D{{Key: "$gte", Value: start.Format("2006-01-02")}, {Key: "$lt", Value: end.Format("2006-01-02")}}}, {Key: "sysOrigin", Value: bson.D{{Key: "$in", Value: r.cfg.Dashboard.SysOrigins}}}, } findOptions := options.Find().SetProjection(bson.D{ {Key: "userId", Value: 1}, {Key: "activeDate", Value: 1}, {Key: "sysOrigin", Value: 1}, {Key: "registerCountryCode", Value: 1}, }) cursor, err := db.Collection(activeLogCollection).Find(ctx, filter, findOptions) if err != nil { return nil, err } defer cursor.Close(ctx) events := make(map[string]map[int64]activeLogEvent) for cursor.Next(ctx) { var row activeLogRow if err := cursor.Decode(&row); err != nil { return nil, err } if row.UserID == 0 || row.ActiveDay == "" || !r.allowedOrigin(row.SysOrigin) { continue } users := events[row.ActiveDay] if users == nil { users = make(map[int64]activeLogEvent) events[row.ActiveDay] = users } users[row.UserID] = activeLogEvent{ UserID: row.UserID, SysOrigin: strings.TrimSpace(row.SysOrigin), Country: r.activeLogCountry(row.RegisterCountryCode), } } return events, cursor.Err() } func (r *MySQLRepository) activeLogCountry(registerCountryCode string) dashboard.Country { code := strings.TrimSpace(registerCountryCode) if code == "" { code = r.cfg.Dashboard.UnknownCountryCode } return dashboard.Country{Code: code, Name: code} } func collectActiveUserIDs(events map[string]map[int64]activeLogEvent) []int64 { seen := make(map[int64]struct{}) for _, users := range events { for userID := range users { seen[userID] = struct{}{} } } out := make([]int64, 0, len(seen)) for userID := range seen { out = append(out, userID) } return out } func activeUserSet(events map[int64]activeLogEvent) map[int64]struct{} { out := make(map[int64]struct{}, len(events)) for userID := range events { out[userID] = struct{}{} } return out } func stringSetToSlice(values map[string]struct{}) []string { out := make([]string, 0, len(values)) for value := range values { out = append(out, value) } return out } func retentionTargetDate(registerDate string, retentionDays int) string { parsed, err := time.Parse("2006-01-02", strings.TrimSpace(registerDate)) if err != nil { return "" } return parsed.AddDate(0, 0, retentionDays).Format("2006-01-02") } func activeUserDateKey(userID int64, activeDate string) string { return strconv.FormatInt(userID, 10) + "|" + activeDate } func (r *MySQLRepository) loadUserCountriesByID(ctx context.Context, ids []int64) (map[int64]dashboard.UserCountry, error) { result := make(map[int64]dashboard.UserCountry) for start := 0; start < len(ids); start += bulkInsertChunkSize { end := min(start+bulkInsertChunkSize, len(ids)) args := make([]interface{}, 0, end-start) for _, id := range ids[start:end] { args = append(args, id) } rows, err := r.db.QueryContext(ctx, ` SELECT user_id, sys_origin, country_code, country_name, is_deleted, user_created_at, user_updated_at FROM dashboard_cdc_user_country WHERE user_id IN `+inPlaceholders(end-start), args...) if err != nil { return nil, err } for rows.Next() { var user dashboard.UserCountry var createdAt, updatedAt sql.NullTime if err := rows.Scan(&user.UserID, &user.SysOrigin, &user.Country.Code, &user.Country.Name, &user.IsDeleted, &createdAt, &updatedAt); err != nil { _ = rows.Close() return nil, err } if createdAt.Valid { user.CreatedAt = createdAt.Time } if updatedAt.Valid { user.UpdatedAt = updatedAt.Time } result[user.UserID] = user } if err := rows.Close(); err != nil { return nil, err } } return result, nil } func (r *MySQLRepository) replaceActiveRetentionDay(ctx context.Context, tx *sql.Tx, db *mongo.Database, day time.Time, events map[string]map[int64]activeLogEvent) error { dayKey := day.Format("2006-01-02") statsByOrigin := make(map[string]map[string]*activeDayStats) for userID, event := range events[dayKey] { sysOrigin := strings.TrimSpace(event.SysOrigin) if !r.allowedOrigin(sysOrigin) { continue } country := event.Country if country.Code == "" { country = dashboard.Country{Code: r.cfg.Dashboard.UnknownCountryCode, Name: r.cfg.Dashboard.UnknownCountryCode} } if country.Name == "" { country.Name = country.Code } stats := statsByOrigin[sysOrigin] if stats == nil { stats = make(map[string]*activeDayStats) statsByOrigin[sysOrigin] = stats } stat := stats[country.Code] if stat == nil { stat = &activeDayStats{active: make(map[string]map[int64]struct{})} stats[country.Code] = stat } active := stat.active[country.Name] if active == nil { active = make(map[int64]struct{}) stat.active[country.Name] = active } active[userID] = struct{}{} } retentionByTimezone, err := r.retentionStatsForDay(ctx, db, day) if err != nil { return err } for _, statTimezone := range r.cfg.Dashboard.StatTimezones { for _, sysOrigin := range r.cfg.Dashboard.SysOrigins { if _, err := tx.ExecContext(ctx, ` UPDATE country_dashboard_period_metric SET daily_active_user = 0, d1_retention_user = 0, d1_retention_base_user = 0, d7_retention_user = 0, d7_retention_base_user = 0, d30_retention_user = 0, d30_retention_base_user = 0, refreshed_at = NOW() WHERE period_type = 'DAY' AND period_key = ? AND stat_timezone = ? AND sys_origin = ?`, dayKey, statTimezone, sysOrigin); err != nil { return err } if err := r.upsertActiveRetentionRows(ctx, tx, day, statTimezone, sysOrigin, statsByOrigin[sysOrigin], retentionByTimezone[statTimezone][sysOrigin]); err != nil { return err } } } return nil } func (r *MySQLRepository) retentionStatsForDay(ctx context.Context, db *mongo.Database, day time.Time) (map[string]map[string]map[string]activeRetentionValues, error) { out := make(map[string]map[string]map[string]activeRetentionValues) for _, statTimezone := range r.cfg.Dashboard.StatTimezones { location, err := time.LoadLocation(statTimezone) if err != nil { location = time.UTC } statDay := day.In(location) dayStart := time.Date(statDay.Year(), statDay.Month(), statDay.Day(), 0, 0, 0, 0, location) today := time.Now().In(location).Format("2006-01-02") for _, days := range []int{1, 7, 30} { cohortStart := dayStart.AddDate(0, 0, -days) cohortEnd := cohortStart.AddDate(0, 0, 1) cohorts, err := r.loadRetentionCohorts(ctx, cohortStart, cohortEnd, statTimezone) if err != nil { return nil, err } if len(cohorts) == 0 { continue } userIDsByOrigin := make(map[string]map[int64]struct{}) activeDatesByOrigin := make(map[string]map[string]struct{}) targetByUser := make(map[int64]string, len(cohorts)) for _, cohort := range cohorts { targetDate := retentionTargetDate(cohort.registerDate, days) if targetDate == "" || targetDate > today { continue } targetByUser[cohort.userID] = targetDate if userIDsByOrigin[cohort.sysOrigin] == nil { userIDsByOrigin[cohort.sysOrigin] = make(map[int64]struct{}) } if activeDatesByOrigin[cohort.sysOrigin] == nil { activeDatesByOrigin[cohort.sysOrigin] = make(map[string]struct{}) } userIDsByOrigin[cohort.sysOrigin][cohort.userID] = struct{}{} activeDatesByOrigin[cohort.sysOrigin][targetDate] = struct{}{} } activeByOrigin := make(map[string]map[string]struct{}) for sysOrigin, ids := range userIDsByOrigin { activeDates, err := r.listActiveUserDateSet(ctx, db, int64SetToSlice(ids), stringSetToSlice(activeDatesByOrigin[sysOrigin]), sysOrigin) if err != nil { return nil, err } activeByOrigin[sysOrigin] = activeDates } for _, cohort := range cohorts { targetDate := targetByUser[cohort.userID] if targetDate == "" { continue } timezoneOut := out[statTimezone] if timezoneOut == nil { timezoneOut = make(map[string]map[string]activeRetentionValues) out[statTimezone] = timezoneOut } originOut := timezoneOut[cohort.sysOrigin] if originOut == nil { originOut = make(map[string]activeRetentionValues) timezoneOut[cohort.sysOrigin] = originOut } country := cohort.country.Code if country == "" { country = r.cfg.Dashboard.UnknownCountryCode } stats := originOut[country] if stats.countryName == "" { stats.countryName = cohort.country.Name if stats.countryName == "" { stats.countryName = country } } retained := false if activeByOrigin[cohort.sysOrigin] != nil { _, retained = activeByOrigin[cohort.sysOrigin][activeUserDateKey(cohort.userID, targetDate)] } switch days { case 1: stats.d1Base++ if retained { stats.d1User++ } case 7: stats.d7Base++ if retained { stats.d7User++ } case 30: stats.d30Base++ if retained { stats.d30User++ } } originOut[country] = stats } } } return out, nil } func (r *MySQLRepository) loadRetentionBase(ctx context.Context, cohortDay time.Time) (map[string]map[string]countryCount, error) { args := stringArgs(r.cfg.Dashboard.SysOrigins) args = append(args, cohortDay, cohortDay.AddDate(0, 0, 1)) rows, err := r.db.QueryContext(ctx, ` SELECT sys_origin, country_code, MAX(country_name), COUNT(1) FROM dashboard_cdc_user_country WHERE is_deleted = 0 AND sys_origin IN `+inStringPlaceholders(len(r.cfg.Dashboard.SysOrigins))+` AND user_created_at >= ? AND user_created_at < ? GROUP BY sys_origin, country_code`, args...) if err != nil { return nil, err } defer rows.Close() result := make(map[string]map[string]countryCount) for rows.Next() { var origin, country, name string var count int64 if err := rows.Scan(&origin, &country, &name, &count); err != nil { return nil, err } countries := result[origin] if countries == nil { countries = make(map[string]countryCount) result[origin] = countries } countries[country] = countryCount{name: name, count: count} } return result, rows.Err() } func (r *MySQLRepository) loadRetentionCohorts(ctx context.Context, start, end time.Time, statTimezone string) ([]retentionCohort, error) { args := []interface{}{timezoneOffset(statTimezone)} args = append(args, stringArgs(r.cfg.Dashboard.SysOrigins)...) args = append(args, start, end) rows, err := r.db.QueryContext(ctx, ` SELECT u.id, u.origin_sys, NULLIF(u.country_code, ''), NULLIF(u.country_name, ''), DATE_FORMAT(CONVERT_TZ(u.create_time, '+03:00', ?), '%Y-%m-%d') AS register_date FROM `+quoteIdent(r.cfg.MySQL.Schema)+`.user_base_info u WHERE (u.is_del = 0 OR u.is_del IS NULL) AND u.origin_sys IN `+inStringPlaceholders(len(r.cfg.Dashboard.SysOrigins))+` AND u.create_time >= ? AND u.create_time < ?`, args...) if err != nil { return nil, err } defer rows.Close() var result []retentionCohort for rows.Next() { var item retentionCohort var countryCode, countryName sql.NullString if err := rows.Scan(&item.userID, &item.sysOrigin, &countryCode, &countryName, &item.registerDate); err != nil { return nil, err } item.sysOrigin = strings.TrimSpace(item.sysOrigin) if !r.allowedOrigin(item.sysOrigin) || item.userID == 0 || item.registerDate == "" { continue } if countryCode.Valid && strings.TrimSpace(countryCode.String) != "" { item.country.Code = strings.TrimSpace(countryCode.String) } else { item.country.Code = r.cfg.Dashboard.UnknownCountryCode } if countryName.Valid && strings.TrimSpace(countryName.String) != "" { item.country.Name = strings.TrimSpace(countryName.String) } else { item.country.Name = item.country.Code } result = append(result, item) } return result, rows.Err() } func (r *MySQLRepository) listActiveUserDateSet(ctx context.Context, db *mongo.Database, userIDs []int64, activeDates []string, sysOrigin string) (map[string]struct{}, error) { result := make(map[string]struct{}) if len(userIDs) == 0 || len(activeDates) == 0 { return result, nil } for start := 0; start < len(userIDs); start += bulkInsertChunkSize { end := min(start+bulkInsertChunkSize, len(userIDs)) filter := bson.D{ {Key: "userId", Value: bson.D{{Key: "$in", Value: userIDs[start:end]}}}, {Key: "activeDate", Value: bson.D{{Key: "$in", Value: activeDates}}}, {Key: "sysOrigin", Value: sysOrigin}, } cursor, err := db.Collection(activeLogCollection).Find(ctx, filter, options.Find().SetProjection(bson.D{ {Key: "userId", Value: 1}, {Key: "activeDate", Value: 1}, })) if err != nil { return nil, err } for cursor.Next(ctx) { var row activeLogRow if err := cursor.Decode(&row); err != nil { _ = cursor.Close(ctx) return nil, err } if row.UserID != 0 && row.ActiveDay != "" { result[activeUserDateKey(row.UserID, row.ActiveDay)] = struct{}{} } } if err := cursor.Close(ctx); err != nil { return nil, err } } return result, nil } func (r *MySQLRepository) upsertActiveRetentionRows(ctx context.Context, tx *sql.Tx, day time.Time, statTimezone string, sysOrigin string, active map[string]*activeDayStats, retention map[string]activeRetentionValues) error { values := make(map[string]activeRetentionValues) for country, stat := range active { item := values[country] for name, users := range stat.active { item.countryName = name item.activeUsers += int64(len(users)) } values[country] = item } for country, item := range retention { existing := values[country] if existing.countryName == "" { existing.countryName = item.countryName } existing.d1User = item.d1User existing.d1Base = item.d1Base existing.d7User = item.d7User existing.d7Base = item.d7Base existing.d30User = item.d30User existing.d30Base = item.d30Base values[country] = existing } if len(values) == 0 { return nil } dayKey := day.Format("2006-01-02") args := make([]interface{}, 0, len(values)*16) for country, item := range values { name := item.countryName if name == "" { name = country } args = append(args, dashboard.PeriodDay, dayKey, statTimezone, dayKey, day, day.AddDate(0, 0, 1), sysOrigin, country, name, item.activeUsers, item.d1User, item.d1Base, item.d7User, item.d7Base, item.d30User, item.d30Base) } _, err := tx.ExecContext(ctx, ` INSERT INTO country_dashboard_period_metric ( period_type, period_key, stat_timezone, period_name, period_start_date, period_end_date, sys_origin, country_code, country_name, daily_active_user, d1_retention_user, d1_retention_base_user, d7_retention_user, d7_retention_base_user, d30_retention_user, d30_retention_base_user ) VALUES `+placeholders(len(values), 16)+` ON DUPLICATE KEY UPDATE period_name = VALUES(period_name), period_start_date = VALUES(period_start_date), period_end_date = VALUES(period_end_date), country_name = VALUES(country_name), daily_active_user = VALUES(daily_active_user), d1_retention_user = VALUES(d1_retention_user), d1_retention_base_user = VALUES(d1_retention_base_user), d7_retention_user = VALUES(d7_retention_user), d7_retention_base_user = VALUES(d7_retention_base_user), d30_retention_user = VALUES(d30_retention_user), d30_retention_base_user = VALUES(d30_retention_base_user), refreshed_at = NOW()`, args...) return err } func (r *MySQLRepository) rebuildAllPeriodFromDays(ctx context.Context, tx *sql.Tx, statTimezone string) error { for _, sysOrigin := range r.cfg.Dashboard.SysOrigins { if _, err := tx.ExecContext(ctx, ` DELETE FROM country_dashboard_period_user_metric WHERE period_type = 'ALL' AND period_key = 'ALL' AND stat_timezone = ? AND sys_origin = ?`, statTimezone, sysOrigin); err != nil { return err } if _, err := tx.ExecContext(ctx, ` INSERT IGNORE INTO country_dashboard_period_user_metric ( period_type, period_key, stat_timezone, period_name, period_start_date, period_end_date, sys_origin, country_code, country_name, user_id, metric_type ) SELECT DISTINCT 'ALL', 'ALL', ?, '全部', NULL, NULL, sys_origin, country_code, country_name, user_id, metric_type FROM country_dashboard_period_user_metric WHERE period_type = 'DAY' AND stat_timezone = ? AND sys_origin = ?`, statTimezone, statTimezone, sysOrigin); err != nil { return err } if _, err := tx.ExecContext(ctx, ` DELETE FROM country_dashboard_period_metric WHERE period_type = 'ALL' AND period_key = 'ALL' AND stat_timezone = ? AND sys_origin = ?`, statTimezone, sysOrigin); err != nil { return err } if _, err := tx.ExecContext(ctx, ` INSERT INTO country_dashboard_period_metric ( period_type, period_key, stat_timezone, period_name, period_start_date, period_end_date, sys_origin, country_code, country_name, country_new_user, daily_active_user, new_user_recharge, official_recharge, mifapay_recharge, google_recharge, dealer_recharge, user_recharge, new_dealer_user_recharge, salary_exchange, salary_transfer, gift_consume, lucky_gift_total_flow, lucky_gift_user, lucky_gift_payout, lucky_gift_anchor_share, game_total_flow, daily_recharge_user, game_user, game_payout, d1_retention_user, d1_retention_base_user, d7_retention_user, d7_retention_base_user, d30_retention_user, d30_retention_base_user, refreshed_at ) SELECT 'ALL', 'ALL', ?, '全部', NULL, NULL, d.sys_origin, d.country_code, MAX(d.country_name), IFNULL(MAX(u.country_new_user), 0), IFNULL(SUM(d.daily_active_user), 0), IFNULL(SUM(d.new_user_recharge), 0), IFNULL(SUM(d.official_recharge), 0), IFNULL(SUM(d.mifapay_recharge), 0), IFNULL(SUM(d.google_recharge), 0), IFNULL(SUM(d.dealer_recharge), 0), IFNULL(SUM(d.user_recharge), 0), IFNULL(SUM(d.new_dealer_user_recharge), 0), IFNULL(SUM(d.salary_exchange), 0), IFNULL(SUM(d.salary_transfer), 0), IFNULL(SUM(d.gift_consume), 0), IFNULL(SUM(d.lucky_gift_total_flow), 0), IFNULL(MAX(u.lucky_gift_user), 0), IFNULL(SUM(d.lucky_gift_payout), 0), IFNULL(SUM(d.lucky_gift_anchor_share), 0), IFNULL(SUM(d.game_total_flow), 0), IFNULL(MAX(u.daily_recharge_user), 0), IFNULL(MAX(u.game_user), 0), IFNULL(SUM(d.game_payout), 0), IFNULL(SUM(d.d1_retention_user), 0), IFNULL(SUM(d.d1_retention_base_user), 0), IFNULL(SUM(d.d7_retention_user), 0), IFNULL(SUM(d.d7_retention_base_user), 0), IFNULL(SUM(d.d30_retention_user), 0), IFNULL(SUM(d.d30_retention_base_user), 0), NOW() FROM country_dashboard_period_metric d LEFT JOIN ( SELECT country_code, SUM(metric_type = 'COUNTRY_NEW_USER') AS country_new_user, SUM(metric_type = 'LUCKY_GIFT_USER') AS lucky_gift_user, SUM(metric_type = 'RECHARGE_USER') AS daily_recharge_user, SUM(metric_type = 'GAME_USER') AS game_user FROM country_dashboard_period_user_metric WHERE period_type = 'ALL' AND period_key = 'ALL' AND stat_timezone = ? AND sys_origin = ? GROUP BY country_code ) u ON u.country_code = d.country_code WHERE d.period_type = 'DAY' AND d.stat_timezone = ? AND d.sys_origin = ? GROUP BY d.sys_origin, d.country_code`, statTimezone, statTimezone, sysOrigin, statTimezone, sysOrigin); err != nil { return err } } return nil } func inPlaceholders(count int) string { return "(" + placeholders(count, 1) + ")" } func inStringPlaceholders(count int) string { return "(" + placeholders(count, 1) + ")" } func stringArgs(values []string) []interface{} { args := make([]interface{}, 0, len(values)) for _, value := range values { args = append(args, value) } return args }