From e1844e26a033ed97435ef52bd6be1118e9941f48 Mon Sep 17 00:00:00 2001 From: zhx Date: Thu, 16 Jul 2026 13:35:03 +0800 Subject: [PATCH] =?UTF-8?q?=E7=BB=9F=E4=B8=80=E8=BF=90=E8=90=A5=E5=AE=BD?= =?UTF-8?q?=E8=A1=A8=20canonical=20Social=20DAU?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- docs/数据大屏统计口径.md | 12 +- docs/统计服务技术方案.md | 5 +- .../internal/storage/mysql/query.go | 193 +++++++++++++++--- .../internal/storage/mysql/query_test.go | 120 ++++++++++- .../mysql/social_requirements_query.go | 91 +++++++++ 5 files changed, 384 insertions(+), 37 deletions(-) diff --git a/docs/数据大屏统计口径.md b/docs/数据大屏统计口径.md index a9ff4317..a7118324 100644 --- a/docs/数据大屏统计口径.md +++ b/docs/数据大屏统计口径.md @@ -8,6 +8,7 @@ - 默认统计时区是 `UTC`；传 `stat_tz=Asia/Shanghai` 时按中国自然日查询统计读模型。 - 国家和区域只使用事件里的国家/区域字段或统计服务保存的用户维度，不用服务器所在地、IP 或本地时区推断。 - 查询只读 `statistics-service` 聚合表，不回查 wallet、room、game、user owner 明细表。 +- 运营宽表的 DAU 统一读取 `stat_social_user_day`，再用 `stat_social_identity_day` 把同一登录会话内的 `d:` 临时归并到 `u:`；维度筛选和国家/区域明细均在归并后计算。 - `coin_total` 为兼容前端保留 JSON 字段名，展示名改为“用户金币数”。 ## 字段口径 @@ -19,7 +20,7 @@ | `start_ms` / `end_ms` | 查询时间 | 本次查询的毫秒时间范围。 | | `country_id` / `region_id` | 国家/区域 | 聚合维度；未筛选国家时返回国家分解。 | | `new_users` | 新增用户 | `UserRegistered` 注册 cohort 去重用户数。 | -| `active_users` | 活跃用户 | `RoomUserJoined`、`GameOrderSettled` 和注册当天活跃写入 `stat_user_day_activity` 后按用户去重。 | +| `active_users` | 活跃用户 | `stat_social_user_day.active_user` 经设备日级 canonical 身份归并后的用户日去重数；多日汇总为活跃人天。 | | `paid_users` | 付费用户 | `WalletRechargeRecorded` 写入 `stat_recharge_day_payers` 后按用户去重。 | | `new_paid_users` | 新增付费用户 | 付费用户中注册 cohort 也落在查询周期内的用户数。 | | `recharge_usd_minor` | 总充值 | `WalletRechargeRecorded.usd_minor_amount` 累加，单位是美元最小单位。 | @@ -64,7 +65,7 @@ | `super_lucky_gift_payout` | super lucky 返奖 | `pool_id` 包含 `super_lucky` 的奖池返奖。 | | `super_lucky_gift_profit` | super lucky 利润 | `super_lucky_gift_turnover - super_lucky_gift_payout`。 | | `super_lucky_gift_profit_rate` | super lucky 利润率 | `super_lucky_gift_profit / super_lucky_gift_turnover`。 | -| `arpu_usd_minor` | ARPU | `recharge_usd_minor / active_users`。 | +| `arpu_usd_minor` | ARPU | `recharge_usd_minor / canonical active_users`。 | | `arppu_usd_minor` | ARPPU | `recharge_usd_minor / paid_users`。 | | `up_value_usd_minor` | UP 值 | 当前与 ARPPU 同口径。 | @@ -79,9 +80,9 @@ | 字段 | 口径 | | --- | --- | | `retention` | `registered_users` 为注册 cohort；`day1/day7/day30_users` 为注册后第 1/7/30 天有活跃的用户；rate 为对应用户数除以注册用户数。 | -| `daily_series` | 按日返回本页主要字段；`coin_total` 和 `salary_usd_minor` 是每天余额快照，其它流水字段按当天累加。 | -| `country_breakdown` | 按国家返回本页主要字段；`coin_total` 和 `salary_usd_minor` 是查询结束日该国家余额快照，其它字段按查询周期累加。 | -| `daily_country_breakdown` | 按日 + 国家返回主要字段，用于国家弹窗明细。 | +| `daily_series` | 按日返回本页主要字段；DAU 使用 canonical Social，`coin_total` 和 `salary_usd_minor` 是每天余额快照，其它流水字段按当天累加。 | +| `country_breakdown` | 按国家返回本页主要字段；DAU 维度在 canonical 归并后确定，`coin_total` 和 `salary_usd_minor` 是查询结束日该国家余额快照，其它字段按查询周期累加。 | +| `daily_country_breakdown` | 按日 + 国家返回主要字段，DAU 与顶部/按日使用同一 canonical Social 数据源。 | | `game_ranking` | 按 `platform_code/game_id` 聚合游戏流水、返奖、退款、玩家数、利润和利润率。 | | `lucky_gift_pools` | 按 `pool_id` 聚合 lucky 礼物流水、返奖、利润、返奖率和利润率。 | | `report_metric_sources` | 返回字段来源和口径说明，供前端展示或排查。 | @@ -92,3 +93,4 @@ - 平台发放的礼物价值同理，需要 `ResourceGranted/ResourceGroupGranted` 事件携带可统计的资源金币价值和来源；当前只可靠计入 COIN 入账和已存在的 `Wallet*RewardCredited` 金币奖励。 - 装扮价值不进入 `platform_grant_coin` 和 `output_coin`。 - 后台人工发放只进入 `manual_grant_coin`，不进入 `platform_grant_coin`。 +- `retention` 仍是注册 cohort 对 `stat_user_day_activity` 的回看口径，不属于运营宽表 DAU 分母；Social 数据需求页的用户留存则使用 canonical Social 用户日口径。 diff --git a/docs/统计服务技术方案.md b/docs/统计服务技术方案.md index 11f7139d..c90ca3e2 100644 --- a/docs/统计服务技术方案.md +++ b/docs/统计服务技术方案.md @@ -73,7 +73,8 @@ flowchart LR | 指标 | 口径 | | --- | --- | | 新增用户 | `UserRegistered` 按 UTC 日、`app_code`、国家维度聚合 | -| 活跃用户 | `RoomUserJoined` 和 `GameOrderSettled` 写入 `stat_user_day_activity`，按用户去重 | +| 运营宽表活跃用户 | `stat_social_user_day.active_user` 先经 `stat_social_identity_day` 完成设备→账号日级归并，再按 canonical 用户日去重 | +| 注册 cohort 留存活跃 | `RoomUserJoined` 和 `GameOrderSettled` 写入 `stat_user_day_activity`，按用户去重 | | 次日留存 | 注册 cohort 中，注册日 + 1 天存在活跃记录的用户数 / 注册用户数 | | 7 日留存 | 注册 cohort 中，注册日 + 7 天存在活跃记录的用户数 / 注册用户数 | | 30 日留存 | 注册 cohort 中，注册日 + 30 天存在活跃记录的用户数 / 注册用户数 | @@ -129,6 +130,8 @@ flowchart LR | `statistics_event_consumption` | 消费幂等表 | `(app_code, source, event_id)` | | `stat_app_day_country` | App/UTC 日/国家总览 | `(app_code, stat_day, country_id)` | | `stat_user_day_activity` | 用户日活跃去重 | `(app_code, stat_day, user_id)` | +| `stat_social_user_day` | Social 用户/匿名设备日级宽表 | `(app_code, stat_tz, stat_day, subject_key)` | +| `stat_social_identity_day` | 同日匿名设备的 canonical 登录账号解析 | `(app_code, stat_tz, stat_day, device_id)` | | `stat_user_registration` | 注册 cohort | `(app_code, user_id)` | | `stat_recharge_day_payers` | 日付费用户去重 | `(app_code, stat_day, user_id)` | | `stat_lucky_gift_day_payers` | 幸运礼物日付费用户去重 | `(app_code, stat_day, country_id, user_id)` | diff --git a/services/statistics-service/internal/storage/mysql/query.go b/services/statistics-service/internal/storage/mysql/query.go index 44cedec5..cad634f1 100644 --- a/services/statistics-service/internal/storage/mysql/query.go +++ b/services/statistics-service/internal/storage/mysql/query.go @@ -573,6 +573,20 @@ func (r *Repository) QueryOverview(ctx context.Context, query OverviewQuery) (Ov if err != nil { return Overview{}, err } + // Revenue and business counts remain on stat_app_day_country, but every DAU + // value and DAU-derived denominator must come from the same canonical Social + // user-day source. Load current and previous windows before deriving ratios + // so deltas cannot compare canonical current data with legacy previous data. + activeCube, err := r.queryCanonicalOverviewActiveCube(ctx, app, statTZ, startDay, endDay, query.CountryID, query.RegionID) + if err != nil { + return Overview{}, err + } + previousActiveCube, err := r.queryCanonicalOverviewActiveCube(ctx, app, statTZ, previousStartDay, previousEndDay, query.CountryID, query.RegionID) + if err != nil { + return Overview{}, err + } + overview.ActiveUsers = activeCube.Total + previous.ActiveUsers = previousActiveCube.Total overview.AppCode, overview.StatTZ, overview.StartMS, overview.EndMS, overview.CountryID = app, statTZ, query.StartMS, query.EndMS, query.CountryID previous.AppCode, previous.StatTZ, previous.CountryID = app, statTZ, query.CountryID // 新增付费用户来自日充值用户去重表和注册表的交集；只读聚合小表，避免 Databi 页面回扫钱包或订单明细。 @@ -635,12 +649,12 @@ func (r *Repository) QueryOverview(ctx context.Context, query OverviewQuery) (Ov } overview.GameRanking = ranking if query.CountryID <= 0 { - breakdown, err := r.queryCountryBreakdown(ctx, app, statTZ, startDay, endDay, query.RegionID, retentionCube.ByCountry) + breakdown, err := r.queryCountryBreakdown(ctx, app, statTZ, startDay, endDay, query.RegionID, retentionCube.ByCountry, activeCube) if err != nil { return Overview{}, err } overview.CountryBreakdown = breakdown - dailyCountryBreakdown, err := r.queryDailyCountryBreakdown(ctx, app, statTZ, startDay, endDay, query.RegionID, retentionCube.ByDayCountry) + dailyCountryBreakdown, err := r.queryDailyCountryBreakdown(ctx, app, statTZ, startDay, endDay, query.RegionID, retentionCube.ByDayCountry, activeCube) if err != nil { return Overview{}, err } @@ -652,15 +666,20 @@ func (r *Repository) QueryOverview(ctx context.Context, query OverviewQuery) (Ov seriesEndDay = statDayIn(query.SeriesEndMS-1, statTZ) } seriesRetentionByDay := retentionCube.ByDay + seriesActiveCube := activeCube if seriesStartDay != startDay || seriesEndDay != endDay { seriesRetentionCube, err := r.queryRetentionCube(ctx, app, statTZ, seriesStartDay, seriesEndDay, query.CountryID, query.RegionID) if err != nil { return Overview{}, err } seriesRetentionByDay = seriesRetentionCube.ByDay + seriesActiveCube, err = r.queryCanonicalOverviewActiveCube(ctx, app, statTZ, seriesStartDay, seriesEndDay, query.CountryID, query.RegionID) + if err != nil { + return Overview{}, err + } } // daily_series 服务卡片内 sparkline 和弹窗折线图；一条 GROUP BY stat_day 查询返回全部指标，避免前端逐日请求。 - dailySeries, err := r.queryDailySeries(ctx, app, statTZ, seriesStartDay, seriesEndDay, query.CountryID, query.RegionID, seriesRetentionByDay) + dailySeries, err := r.queryDailySeries(ctx, app, statTZ, seriesStartDay, seriesEndDay, query.CountryID, query.RegionID, seriesRetentionByDay, seriesActiveCube) if err != nil { return Overview{}, err } @@ -852,7 +871,7 @@ func (r *Repository) queryAppDayOverviewTotals(ctx context.Context, app, statTZ, args = append(args, regionID) } row := r.db.QueryRowContext(ctx, ` - SELECT COALESCE(SUM(new_users),0), COALESCE(SUM(active_users),0), COALESCE(SUM(paid_users),0), + SELECT COALESCE(SUM(new_users),0), COALESCE(SUM(paid_users),0), COALESCE(SUM(recharge_usd_minor),0), COALESCE(SUM(new_user_recharge_usd_minor),0), COALESCE(SUM(coin_seller_recharge_usd_minor),0), COALESCE(SUM(coin_seller_stock_coin),0), COALESCE(SUM(mifapay_recharge_usd_minor),0), COALESCE(SUM(google_recharge_usd_minor),0), COALESCE(SUM(coin_seller_transfer_coin),0), @@ -867,7 +886,7 @@ func (r *Repository) queryAppDayOverviewTotals(ctx context.Context, app, statTZ, FROM stat_app_day_country WHERE app_code = ? AND stat_tz = ? AND stat_day BETWEEN ? AND ?`+filter, args...) var overview Overview - if err := row.Scan(&overview.NewUsers, &overview.ActiveUsers, &overview.PaidUsers, &overview.RechargeUSDMinor, &overview.NewUserRechargeUSDMinor, &overview.CoinSellerRechargeUSDMinor, &overview.CoinSellerStockCoin, &overview.MifaPayRechargeUSDMinor, &overview.GoogleRechargeUSDMinor, &overview.CoinSellerTransferCoin, &overview.CoinTotal, &overview.ConsumedCoin, &overview.OutputCoin, &overview.PlatformGrantCoin, &overview.ManualGrantCoin, &overview.SalaryUSDMinor, &overview.SalaryTransferCoin, &overview.MicOnlineMS, &overview.MicOnlineUsers, &overview.GiftCoinSpent, &overview.LuckyGiftTurnover, &overview.LuckyGiftPayout, &overview.LuckyGiftPayers, &overview.GameTurnover, &overview.GamePayout, &overview.GameRefund, &overview.GamePlayers); err != nil { + if err := row.Scan(&overview.NewUsers, &overview.PaidUsers, &overview.RechargeUSDMinor, &overview.NewUserRechargeUSDMinor, &overview.CoinSellerRechargeUSDMinor, &overview.CoinSellerStockCoin, &overview.MifaPayRechargeUSDMinor, &overview.GoogleRechargeUSDMinor, &overview.CoinSellerTransferCoin, &overview.CoinTotal, &overview.ConsumedCoin, &overview.OutputCoin, &overview.PlatformGrantCoin, &overview.ManualGrantCoin, &overview.SalaryUSDMinor, &overview.SalaryTransferCoin, &overview.MicOnlineMS, &overview.MicOnlineUsers, &overview.GiftCoinSpent, &overview.LuckyGiftTurnover, &overview.LuckyGiftPayout, &overview.LuckyGiftPayers, &overview.GameTurnover, &overview.GamePayout, &overview.GameRefund, &overview.GamePlayers); err != nil { return Overview{}, err } return overview, nil @@ -925,7 +944,7 @@ func (r *Repository) queryRealRoomRobotGiftOverview(ctx context.Context, app, st return overview, nil } -func (r *Repository) queryDailySeries(ctx context.Context, app, statTZ, startDay, endDay string, countryID int64, regionID int64, retentionByDay map[string]retentionCount) ([]DailySeriesItem, error) { +func (r *Repository) queryDailySeries(ctx context.Context, app, statTZ, startDay, endDay string, countryID int64, regionID int64, retentionByDay map[string]retentionCount, activeCube canonicalOverviewActiveCube) ([]DailySeriesItem, error) { args := []any{app, normalizeStatTZ(statTZ), startDay, endDay} filter := "" if countryID > 0 { @@ -938,7 +957,7 @@ func (r *Repository) queryDailySeries(ctx context.Context, app, statTZ, startDay } rows, err := r.db.QueryContext(ctx, ` SELECT DATE_FORMAT(stat_day, '%Y-%m-%d'), - COALESCE(SUM(new_users),0), COALESCE(SUM(active_users),0), COALESCE(SUM(paid_users),0), + COALESCE(SUM(new_users),0), COALESCE(SUM(paid_users),0), COALESCE(SUM(recharge_usd_minor),0), COALESCE(SUM(new_user_recharge_usd_minor),0), COALESCE(SUM(coin_seller_recharge_usd_minor),0), COALESCE(SUM(coin_seller_stock_coin),0), COALESCE(SUM(mifapay_recharge_usd_minor),0), COALESCE(SUM(google_recharge_usd_minor),0), COALESCE(SUM(coin_seller_transfer_coin),0), @@ -960,7 +979,7 @@ func (r *Repository) queryDailySeries(ctx context.Context, app, statTZ, startDay for rows.Next() { var item DailySeriesItem var gamePayout, gameRefund int64 - if err := rows.Scan(&item.StatDay, &item.NewUsers, &item.ActiveUsers, &item.PaidUsers, &item.RechargeUSDMinor, &item.NewUserRechargeUSDMinor, &item.CoinSellerRechargeUSDMinor, &item.CoinSellerStockCoin, &item.MifaPayRechargeUSDMinor, &item.GoogleRechargeUSDMinor, &item.CoinSellerTransferCoin, &item.CoinTotal, &item.ConsumedCoin, &item.OutputCoin, &item.PlatformGrantCoin, &item.ManualGrantCoin, &item.SalaryUSDMinor, &item.SalaryTransferCoin, &item.MicOnlineMS, &item.MicOnlineUsers, &item.GiftCoinSpent, &item.LuckyGiftTurnover, &item.LuckyGiftPayout, &item.GameTurnover, &gamePayout, &gameRefund); err != nil { + if err := rows.Scan(&item.StatDay, &item.NewUsers, &item.PaidUsers, &item.RechargeUSDMinor, &item.NewUserRechargeUSDMinor, &item.CoinSellerRechargeUSDMinor, &item.CoinSellerStockCoin, &item.MifaPayRechargeUSDMinor, &item.GoogleRechargeUSDMinor, &item.CoinSellerTransferCoin, &item.CoinTotal, &item.ConsumedCoin, &item.OutputCoin, &item.PlatformGrantCoin, &item.ManualGrantCoin, &item.SalaryUSDMinor, &item.SalaryTransferCoin, &item.MicOnlineMS, &item.MicOnlineUsers, &item.GiftCoinSpent, &item.LuckyGiftTurnover, &item.LuckyGiftPayout, &item.GameTurnover, &gamePayout, &gameRefund); err != nil { return nil, err } item.Label = item.StatDay @@ -1021,7 +1040,7 @@ func (r *Repository) queryDailySeries(ctx context.Context, app, statTZ, startDay out = mergeActivityLuckyGiftDaily(out, activityDaily) } } - return out, nil + return overlayCanonicalDailyActive(out, activeCube), nil } func (r *Repository) queryDailyNewPaidUsers(ctx context.Context, app, statTZ, startDay, endDay string, countryID int64, regionID int64) (map[string]int64, error) { @@ -1255,6 +1274,28 @@ func applyDailySeriesDerivedMetrics(item *DailySeriesItem) { item.RechargeConversionRate = ratio(item.RechargeUsers, item.ActiveUsers) } +// overlayCanonicalDailyActive makes canonical Social the only DAU source. It +// also creates days that have client activity but no legacy business aggregate, +// then recalculates every metric whose denominator is DAU. +func overlayCanonicalDailyActive(items []DailySeriesItem, cube canonicalOverviewActiveCube) []DailySeriesItem { + indexByDay := make(map[string]int, len(items)) + for index := range items { + indexByDay[items[index].StatDay] = index + items[index].ActiveUsers = cube.ByDay[items[index].StatDay] + } + for day, activeUsers := range cube.ByDay { + if _, ok := indexByDay[day]; ok { + continue + } + items = append(items, DailySeriesItem{StatDay: day, Label: day, ActiveUsers: activeUsers}) + } + for index := range items { + applyDailySeriesDerivedMetrics(&items[index]) + } + sort.SliceStable(items, func(i, j int) bool { return items[i].StatDay < items[j].StatDay }) + return items +} + func applyOverviewDerivedMetrics(overview *Overview) { if overview == nil { return @@ -1345,7 +1386,7 @@ func (r *Repository) queryGameRanking(ctx context.Context, app, statTZ, startDay return out, rows.Err() } -func (r *Repository) queryCountryBreakdown(ctx context.Context, app, statTZ, startDay, endDay string, regionID int64, retentionByCountry map[int64]retentionCount) ([]CountryBreakdown, error) { +func (r *Repository) queryCountryBreakdown(ctx context.Context, app, statTZ, startDay, endDay string, regionID int64, retentionByCountry map[int64]retentionCount, activeCube canonicalOverviewActiveCube) ([]CountryBreakdown, error) { args := []any{endDay, endDay, app, normalizeStatTZ(statTZ), startDay, endDay} filter := "" if regionID > 0 { @@ -1354,7 +1395,7 @@ func (r *Repository) queryCountryBreakdown(ctx context.Context, app, statTZ, sta } rows, err := r.db.QueryContext(ctx, ` SELECT country_id, COALESCE(MAX(region_id),0), - COALESCE(SUM(new_users),0), COALESCE(SUM(active_users),0), COALESCE(SUM(paid_users),0), + COALESCE(SUM(new_users),0), COALESCE(SUM(paid_users),0), COALESCE(SUM(recharge_usd_minor),0), COALESCE(SUM(new_user_recharge_usd_minor),0), COALESCE(SUM(coin_seller_recharge_usd_minor),0), COALESCE(SUM(coin_seller_stock_coin),0), COALESCE(SUM(mifapay_recharge_usd_minor),0), COALESCE(SUM(google_recharge_usd_minor),0), COALESCE(SUM(coin_seller_transfer_coin),0), @@ -1369,8 +1410,7 @@ func (r *Repository) queryCountryBreakdown(ctx context.Context, app, statTZ, sta FROM stat_app_day_country WHERE app_code = ? AND stat_tz = ? AND stat_day BETWEEN ? AND ?`+filter+` GROUP BY country_id - ORDER BY COALESCE(SUM(recharge_usd_minor),0) DESC, COALESCE(SUM(active_users),0) DESC, country_id ASC - LIMIT 500`, args...) + ORDER BY COALESCE(SUM(recharge_usd_minor),0) DESC, country_id ASC`, args...) if err != nil { return nil, err } @@ -1378,7 +1418,7 @@ func (r *Repository) queryCountryBreakdown(ctx context.Context, app, statTZ, sta out := []CountryBreakdown{} for rows.Next() { var item CountryBreakdown - if err := rows.Scan(&item.CountryID, &item.RegionID, &item.NewUsers, &item.ActiveUsers, &item.PaidUsers, &item.RechargeUSDMinor, &item.NewUserRechargeUSDMinor, &item.CoinSellerRechargeUSDMinor, &item.CoinSellerStockCoin, &item.MifaPayRechargeUSDMinor, &item.GoogleRechargeUSDMinor, &item.CoinSellerTransferCoin, &item.CoinTotal, &item.ConsumedCoin, &item.OutputCoin, &item.PlatformGrantCoin, &item.ManualGrantCoin, &item.SalaryUSDMinor, &item.SalaryTransferCoin, &item.MicOnlineMS, &item.MicOnlineUsers, &item.GiftCoinSpent, &item.LuckyGiftTurnover, &item.LuckyGiftPayout, &item.LuckyGiftPayers, &item.GameTurnover, &item.GamePayout, &item.GameRefund, &item.GamePlayers); err != nil { + if err := rows.Scan(&item.CountryID, &item.RegionID, &item.NewUsers, &item.PaidUsers, &item.RechargeUSDMinor, &item.NewUserRechargeUSDMinor, &item.CoinSellerRechargeUSDMinor, &item.CoinSellerStockCoin, &item.MifaPayRechargeUSDMinor, &item.GoogleRechargeUSDMinor, &item.CoinSellerTransferCoin, &item.CoinTotal, &item.ConsumedCoin, &item.OutputCoin, &item.PlatformGrantCoin, &item.ManualGrantCoin, &item.SalaryUSDMinor, &item.SalaryTransferCoin, &item.MicOnlineMS, &item.MicOnlineUsers, &item.GiftCoinSpent, &item.LuckyGiftTurnover, &item.LuckyGiftPayout, &item.LuckyGiftPayers, &item.GameTurnover, &item.GamePayout, &item.GameRefund, &item.GamePlayers); err != nil { return nil, err } applyCountryBreakdownDerivedMetrics(&item) @@ -1421,10 +1461,10 @@ func (r *Repository) queryCountryBreakdown(ctx context.Context, app, statTZ, sta applyCountryBreakdownDerivedMetrics(&out[index]) } } - return out, nil + return overlayCanonicalCountryActive(out, activeCube), nil } -func (r *Repository) queryDailyCountryBreakdown(ctx context.Context, app, statTZ, startDay, endDay string, regionID int64, retentionByDayCountry map[string]retentionCount) ([]DailyCountryStat, error) { +func (r *Repository) queryDailyCountryBreakdown(ctx context.Context, app, statTZ, startDay, endDay string, regionID int64, retentionByDayCountry map[string]retentionCount, activeCube canonicalOverviewActiveCube) ([]DailyCountryStat, error) { args := []any{app, normalizeStatTZ(statTZ), startDay, endDay} filter := "" if regionID > 0 { @@ -1433,7 +1473,7 @@ func (r *Repository) queryDailyCountryBreakdown(ctx context.Context, app, statTZ } rows, err := r.db.QueryContext(ctx, ` SELECT DATE_FORMAT(stat_day, '%Y-%m-%d'), country_id, COALESCE(MAX(region_id),0), - COALESCE(SUM(new_users),0), COALESCE(SUM(active_users),0), COALESCE(SUM(paid_users),0), + COALESCE(SUM(new_users),0), COALESCE(SUM(paid_users),0), COALESCE(SUM(recharge_usd_minor),0), COALESCE(SUM(new_user_recharge_usd_minor),0), COALESCE(SUM(coin_seller_recharge_usd_minor),0), COALESCE(SUM(coin_seller_stock_coin),0), COALESCE(SUM(mifapay_recharge_usd_minor),0), COALESCE(SUM(google_recharge_usd_minor),0), COALESCE(SUM(coin_seller_transfer_coin),0), @@ -1447,8 +1487,7 @@ func (r *Repository) queryDailyCountryBreakdown(ctx context.Context, app, statTZ FROM stat_app_day_country WHERE app_code = ? AND stat_tz = ? AND stat_day BETWEEN ? AND ?`+filter+` GROUP BY stat_day, country_id - ORDER BY stat_day ASC, COALESCE(SUM(recharge_usd_minor),0) DESC, COALESCE(SUM(active_users),0) DESC, country_id ASC - LIMIT 5000`, args...) + ORDER BY stat_day ASC, COALESCE(SUM(recharge_usd_minor),0) DESC, country_id ASC`, args...) if err != nil { return nil, err } @@ -1456,7 +1495,7 @@ func (r *Repository) queryDailyCountryBreakdown(ctx context.Context, app, statTZ out := []DailyCountryStat{} for rows.Next() { var item DailyCountryStat - if err := rows.Scan(&item.StatDay, &item.CountryID, &item.RegionID, &item.NewUsers, &item.ActiveUsers, &item.PaidUsers, &item.RechargeUSDMinor, &item.NewUserRechargeUSDMinor, &item.CoinSellerRechargeUSDMinor, &item.CoinSellerStockCoin, &item.MifaPayRechargeUSDMinor, &item.GoogleRechargeUSDMinor, &item.CoinSellerTransferCoin, &item.CoinTotal, &item.ConsumedCoin, &item.OutputCoin, &item.PlatformGrantCoin, &item.ManualGrantCoin, &item.SalaryUSDMinor, &item.SalaryTransferCoin, &item.MicOnlineMS, &item.MicOnlineUsers, &item.GiftCoinSpent, &item.LuckyGiftTurnover, &item.LuckyGiftPayout, &item.LuckyGiftPayers, &item.GameTurnover, &item.GamePayout, &item.GameRefund, &item.GamePlayers); err != nil { + if err := rows.Scan(&item.StatDay, &item.CountryID, &item.RegionID, &item.NewUsers, &item.PaidUsers, &item.RechargeUSDMinor, &item.NewUserRechargeUSDMinor, &item.CoinSellerRechargeUSDMinor, &item.CoinSellerStockCoin, &item.MifaPayRechargeUSDMinor, &item.GoogleRechargeUSDMinor, &item.CoinSellerTransferCoin, &item.CoinTotal, &item.ConsumedCoin, &item.OutputCoin, &item.PlatformGrantCoin, &item.ManualGrantCoin, &item.SalaryUSDMinor, &item.SalaryTransferCoin, &item.MicOnlineMS, &item.MicOnlineUsers, &item.GiftCoinSpent, &item.LuckyGiftTurnover, &item.LuckyGiftPayout, &item.LuckyGiftPayers, &item.GameTurnover, &item.GamePayout, &item.GameRefund, &item.GamePlayers); err != nil { return nil, err } item.Label = item.StatDay @@ -1510,7 +1549,7 @@ func (r *Repository) queryDailyCountryBreakdown(ctx context.Context, app, statTZ applyCountryBreakdownDerivedMetrics(&out[index].CountryBreakdown) } } - return out, nil + return overlayCanonicalDailyCountryActive(out, activeCube), nil } func applyCountryBreakdownDerivedMetrics(item *CountryBreakdown) { @@ -1541,6 +1580,110 @@ func applyCountryBreakdownDerivedMetrics(item *CountryBreakdown) { item.SuperLuckyGiftProfitRate = ratio(item.SuperLuckyGiftProfit, item.SuperLuckyGiftTurnover) } +// overlayCanonicalCountryActive preserves monetary and business facts from the +// legacy aggregate while replacing its incomplete activity counter. A +// canonical-only country is added so country and region rollups still reconcile +// with the top-level canonical user-day total. +func overlayCanonicalCountryActive(items []CountryBreakdown, cube canonicalOverviewActiveCube) []CountryBreakdown { + indexByCountry := make(map[int64]int, len(items)) + for index := range items { + indexByCountry[items[index].CountryID] = index + dimension, ok := cube.ByCountry[items[index].CountryID] + items[index].ActiveUsers = 0 + if ok { + items[index].ActiveUsers = dimension.ActiveUsers + items[index].RegionID = dimension.RegionID + } + } + for countryID, dimension := range cube.ByCountry { + if _, ok := indexByCountry[countryID]; ok { + continue + } + items = append(items, CountryBreakdown{ + CountryID: countryID, + RegionID: dimension.RegionID, + ActiveUsers: dimension.ActiveUsers, + }) + } + for index := range items { + applyCountryBreakdownDerivedMetrics(&items[index]) + } + sort.SliceStable(items, func(i, j int) bool { + if items[i].RechargeUSDMinor != items[j].RechargeUSDMinor { + return items[i].RechargeUSDMinor > items[j].RechargeUSDMinor + } + if items[i].ActiveUsers != items[j].ActiveUsers { + return items[i].ActiveUsers > items[j].ActiveUsers + } + return items[i].CountryID < items[j].CountryID + }) + // Keep the existing response bound, but apply it only after canonical-only + // countries have participated in the same ordering as legacy fact rows. + if len(items) > 500 { + items = items[:500] + } + return items +} + +// overlayCanonicalDailyCountryActive is the day×country counterpart of the +// country overlay. It prevents stale legacy DAU from surviving on rows with no +// canonical activity and fills client-only detail rows before final ordering. +func overlayCanonicalDailyCountryActive(items []DailyCountryStat, cube canonicalOverviewActiveCube) []DailyCountryStat { + indexByDayCountry := make(map[string]int, len(items)) + for index := range items { + key := dailyCountryKey(items[index].StatDay, items[index].CountryID) + indexByDayCountry[key] = index + dimension, ok := cube.ByDayCountry[key] + items[index].ActiveUsers = 0 + if ok { + items[index].ActiveUsers = dimension.ActiveUsers + items[index].RegionID = dimension.RegionID + } + } + for key, dimension := range cube.ByDayCountry { + if _, ok := indexByDayCountry[key]; ok { + continue + } + separator := strings.LastIndex(key, ":") + if separator <= 0 { + continue + } + countryID, err := strconv.ParseInt(key[separator+1:], 10, 64) + if err != nil { + continue + } + day := key[:separator] + items = append(items, DailyCountryStat{ + StatDay: day, + Label: day, + CountryBreakdown: CountryBreakdown{ + CountryID: countryID, + RegionID: dimension.RegionID, + ActiveUsers: dimension.ActiveUsers, + }, + }) + } + for index := range items { + applyCountryBreakdownDerivedMetrics(&items[index].CountryBreakdown) + } + sort.SliceStable(items, func(i, j int) bool { + if items[i].StatDay != items[j].StatDay { + return items[i].StatDay < items[j].StatDay + } + if items[i].RechargeUSDMinor != items[j].RechargeUSDMinor { + return items[i].RechargeUSDMinor > items[j].RechargeUSDMinor + } + if items[i].ActiveUsers != items[j].ActiveUsers { + return items[i].ActiveUsers > items[j].ActiveUsers + } + return items[i].CountryID < items[j].CountryID + }) + if len(items) > 5000 { + items = items[:5000] + } + return items +} + func (r *Repository) queryLuckyGiftPools(ctx context.Context, app, statTZ, startDay, endDay string, countryID int64, regionID int64) ([]LuckyGiftPoolStat, error) { args := []any{app, normalizeStatTZ(statTZ), startDay, endDay} filter := "" @@ -1817,7 +1960,7 @@ func isSuperLuckyPool(poolID string) bool { func reportMetricSources() []ReportMetricSource { return []ReportMetricSource{ {Key: "new_users", Label: "注册", Status: "available", Source: "UserRegistered", Definition: "统计窗口内完成注册的用户数；registered_users 和 registrations 是同一口径别名。"}, - {Key: "active_users", Label: "活跃用户", Status: "available", Source: "RoomUserJoined / GameOrderSettled", Definition: "统计服务按用户维度去重后的日活，进房和游戏结算都会沉淀为本地活跃事实。"}, + {Key: "active_users", Label: "活跃用户", Status: "available", Source: "stat_social_user_day + stat_social_identity_day", Definition: "Social 用户日宽表先按同一登录会话把匿名设备归并到登录账号，再按 canonical 用户日去重；多日汇总为活跃人天。"}, {Key: "paid_users", Label: "付费用户", Status: "available", Source: "stat_recharge_day_payers", Definition: "统计窗口内发生充值的去重用户数；当前 Lalu 付费用户等同充值用户。"}, {Key: "recharge_users", Label: "充值用户", Status: "available", Source: "stat_recharge_day_payers", Definition: "统计窗口内发生充值的去重用户数；当前 Lalu 由 paid_users 派生。"}, {Key: "recharge_usd_minor", Label: "总充值", Status: "available", Source: "WalletRechargeRecorded", Definition: "所有充值渠道的美元最小单位总额，历史行缺总额时用可拆分渠道合计兜底。"}, @@ -1841,10 +1984,10 @@ func reportMetricSources() []ReportMetricSource { {Key: "salary_transfer_coin", Label: "工资兑换金币", Status: "available", Source: "WalletBalanceChanged biz_type=salary_transfer_to_coin_seller / wallet_transactions", Definition: "用户工资钱包转给币商时，币商 COIN_SELLER_COIN 入账的金币数量；国家/区域按发起转账用户归属。"}, {Key: "avg_mic_online_ms", Label: "平均麦上时间", Status: "available", Source: "user_mic_daily_stats", Definition: "麦上总时长 / 当日麦上用户数，只统计 mic_online_ms > 0 的用户。"}, {Key: "mic_online_ms", Label: "麦上总时长", Status: "available", Source: "user_mic_daily_stats.mic_online_ms", Definition: "用户每日麦上时长按国家/区域聚合后的毫秒总和。"}, - {Key: "arpu_usd_minor", Label: "ARPU", Status: "available", Source: "stat_app_day_country", Definition: "总充值 / 活跃用户，返回美元最小单位。"}, + {Key: "arpu_usd_minor", Label: "ARPU", Status: "available", Source: "stat_app_day_country + canonical Social DAU", Definition: "业务事实总充值 / canonical Social 活跃用户，返回美元最小单位。"}, {Key: "arppu_usd_minor", Label: "ARPPU", Status: "available", Source: "stat_app_day_country", Definition: "总充值 / 付费用户，返回美元最小单位。"}, - {Key: "paid_conversion_rate", Label: "付费转化率", Status: "available", Source: "stat_app_day_country", Definition: "付费用户 / 活跃用户；当前 Lalu 付费用户等同充值用户。"}, - {Key: "recharge_conversion_rate", Label: "充值转化率", Status: "available", Source: "stat_app_day_country", Definition: "充值用户 / 活跃用户；当前 Lalu 充值用户由 paid_users 派生。"}, + {Key: "paid_conversion_rate", Label: "付费转化率", Status: "available", Source: "stat_recharge_day_payers + canonical Social DAU", Definition: "付费用户 / canonical Social 活跃用户；当前 Lalu 付费用户等同充值用户。"}, + {Key: "recharge_conversion_rate", Label: "充值转化率", Status: "available", Source: "stat_recharge_day_payers + canonical Social DAU", Definition: "充值用户 / canonical Social 活跃用户；当前 Lalu 充值用户由 paid_users 派生。"}, } } diff --git a/services/statistics-service/internal/storage/mysql/query_test.go b/services/statistics-service/internal/storage/mysql/query_test.go index cc39401f..092a6001 100644 --- a/services/statistics-service/internal/storage/mysql/query_test.go +++ b/services/statistics-service/internal/storage/mysql/query_test.go @@ -51,6 +51,59 @@ func TestCanonicalizeSocialUserDaysMergesIdentityWithoutLosingDeviceCount(t *tes if row.AppFirstOpen != 1 || row.AppStayMS != 90_000 || row.RoomStayMS != 45_000 { t.Fatalf("canonical metric merge mismatch: %+v", row) } + cube := buildCanonicalOverviewActiveCube(got) + if cube.Total != 1 || cube.ByDay["2026-07-01"] != 1 || cube.ByCountry[86].ActiveUsers != 1 || cube.ByCountry[86].RegionID != 210 || cube.ByDayCountry[dailyCountryKey("2026-07-01", 86)].ActiveUsers != 1 { + t.Fatalf("canonical overview active cube mismatch: %+v", cube) + } +} + +func TestCanonicalOverviewActiveOverlaysReplaceLegacyDAUAndFillDimensions(t *testing.T) { + cube := canonicalOverviewActiveCube{ + Total: 11, + ByDay: map[string]int64{"2026-07-01": 8, "2026-07-02": 3}, + ByCountry: map[int64]canonicalOverviewActiveDimension{ + 86: {ActiveUsers: 8, RegionID: 210}, + 66: {ActiveUsers: 3, RegionID: 220}, + }, + ByDayCountry: map[string]canonicalOverviewActiveDimension{ + dailyCountryKey("2026-07-01", 86): {ActiveUsers: 8, RegionID: 210}, + dailyCountryKey("2026-07-02", 66): {ActiveUsers: 3, RegionID: 220}, + }, + } + + daily := overlayCanonicalDailyActive([]DailySeriesItem{{ + StatDay: "2026-07-01", Label: "2026-07-01", ActiveUsers: 999, PaidUsers: 2, RechargeUSDMinor: 100, + }}, cube) + if len(daily) != 2 || daily[0].ActiveUsers != 8 || daily[0].ARPUUSDMinor != 12 || daily[0].PayerRate != 0.25 || daily[1].StatDay != "2026-07-02" || daily[1].ActiveUsers != 3 { + t.Fatalf("daily canonical overlay mismatch: %+v", daily) + } + + countries := overlayCanonicalCountryActive([]CountryBreakdown{ + {CountryID: 86, RegionID: 999, ActiveUsers: 999, PaidUsers: 2, RechargeUSDMinor: 100}, + {CountryID: 99, RegionID: 230, ActiveUsers: 777, RechargeUSDMinor: 50}, + }, cube) + countryByID := map[int64]CountryBreakdown{} + for _, item := range countries { + countryByID[item.CountryID] = item + } + if countryByID[86].ActiveUsers != 8 || countryByID[86].RegionID != 210 || countryByID[86].ARPUUSDMinor != 12 || countryByID[86].PaidConversionRate != 0.25 { + t.Fatalf("existing country canonical overlay mismatch: %+v", countryByID[86]) + } + if countryByID[66].ActiveUsers != 3 || countryByID[66].RegionID != 220 { + t.Fatalf("canonical-only country row missing: %+v", countryByID[66]) + } + if countryByID[99].ActiveUsers != 0 || countryByID[99].RechargeUSDMinor != 50 { + t.Fatalf("legacy-only country must retain business facts but not legacy DAU: %+v", countryByID[99]) + } + + dailyCountries := overlayCanonicalDailyCountryActive([]DailyCountryStat{{ + StatDay: "2026-07-01", Label: "2026-07-01", CountryBreakdown: CountryBreakdown{ + CountryID: 86, RegionID: 999, ActiveUsers: 999, PaidUsers: 2, RechargeUSDMinor: 100, + }, + }}, cube) + if len(dailyCountries) != 2 || dailyCountries[0].ActiveUsers != 8 || dailyCountries[0].RegionID != 210 || dailyCountries[0].PayerRate != 0.25 || dailyCountries[1].StatDay != "2026-07-02" || dailyCountries[1].CountryID != 66 || dailyCountries[1].ActiveUsers != 3 { + t.Fatalf("daily country canonical overlay mismatch: %+v", dailyCountries) + } } func TestQueryOverviewUsesActivityLuckyGiftDayStats(t *testing.T) { @@ -269,6 +322,27 @@ func TestSocialIdentityCanonicalizesAnonymousAndLoggedSession(t *testing.T) { math.Abs(socialMetricFloat(t, retention.Total, "active_d1_active_rate")-1) > 0.0001 { t.Fatalf("canonical retention mismatch: %+v", retention.Total.Metrics) } + + // The operation-wide response must consume the exact same canonical user-day + // as Data Requirements. This fixture has both d:device-A and u:123, so any + // direct SUM(active_user) implementation would incorrectly return two. + overview, err := repository.QueryOverview(ctx, OverviewQuery{ + AppCode: "lalu", + StartMS: start.UnixMilli(), + EndMS: start.Add(24 * time.Hour).UnixMilli(), + }) + if err != nil { + t.Fatalf("query canonical overview: %v", err) + } + if overview.ActiveUsers != 1 || len(overview.DailySeries) != 1 || overview.DailySeries[0].ActiveUsers != 1 { + t.Fatalf("overview and daily DAU must share canonical identity: %+v", overview) + } + if len(overview.CountryBreakdown) != 1 || overview.CountryBreakdown[0].CountryID != 86 || overview.CountryBreakdown[0].RegionID != 210 || overview.CountryBreakdown[0].ActiveUsers != 1 { + t.Fatalf("country DAU must use the direct account dimensions: %+v", overview.CountryBreakdown) + } + if len(overview.DailyCountryBreakdown) != 1 || overview.DailyCountryBreakdown[0].CountryID != 86 || overview.DailyCountryBreakdown[0].RegionID != 210 || overview.DailyCountryBreakdown[0].ActiveUsers != 1 { + t.Fatalf("daily country DAU must use the direct account dimensions: %+v", overview.DailyCountryBreakdown) + } } func TestSocialIdentityMarksConcurrentAccountSwitchAmbiguous(t *testing.T) { @@ -1208,19 +1282,21 @@ func TestQueryOverviewReturnsPreviousPeriodDeltaRates(t *testing.T) { ('lalu', '2026-06-06', 210, 20, 5, 200, 100, 200, 600, 200, 400, 100, 1000, 400, 100, 1)`); err != nil { t.Fatalf("seed delta statistics: %v", err) } + seedCanonicalActiveUsers(t, ctx, repository, "lalu", StatTZUTC, "2026-06-05", 210, 0, 8, 10_000) + seedCanonicalActiveUsers(t, ctx, repository, "lalu", StatTZUTC, "2026-06-06", 210, 0, 12, 20_000) start := time.Date(2026, 6, 6, 0, 0, 0, 0, time.UTC).UnixMilli() overview, err := repository.QueryOverview(ctx, OverviewQuery{AppCode: "lalu", StartMS: start, EndMS: start + int64(24*time.Hour/time.Millisecond), CountryID: 210}) if err != nil { t.Fatalf("query overview: %v", err) } - if overview.RechargeDeltaRate != 1 || overview.NewUserRechargeDeltaRate != 1 || overview.ActiveUsersDeltaRate != 1 { + if overview.ActiveUsers != 12 || overview.RechargeDeltaRate != 1 || overview.NewUserRechargeDeltaRate != 1 || math.Abs(overview.ActiveUsersDeltaRate-0.5) > 0.000001 { t.Fatalf("top delta rates mismatch: %+v", overview) } - if overview.PaidUsersDeltaRate != 1.5 || overview.ARPUDeltaRate != 0 || overview.ARPPUDeltaRate != -0.2 { + if overview.ARPUUSDMinor != 16 || overview.PaidUsersDeltaRate != 1.5 || math.Abs(overview.ARPUDeltaRate-(1.0/3.0)) > 0.000001 || overview.ARPPUDeltaRate != -0.2 { t.Fatalf("user delta rates mismatch: %+v", overview) } - if overview.RechargeUsers != overview.PaidUsers || overview.PaidConversionRate != 0.25 || overview.RechargeConversionRate != 0.25 || overview.PayerRateDeltaPP != 5 { + if overview.RechargeUsers != overview.PaidUsers || math.Abs(overview.PaidConversionRate-(5.0/12.0)) > 0.000001 || math.Abs(overview.RechargeConversionRate-(5.0/12.0)) > 0.000001 || math.Abs(overview.PayerRateDeltaPP-(100.0/6.0)) > 0.000001 || math.Abs(overview.PaidConversionRateDeltaPP-(100.0/6.0)) > 0.000001 || math.Abs(overview.RechargeConversionRateDeltaPP-(100.0/6.0)) > 0.000001 { t.Fatalf("conversion aliases mismatch: %+v", overview) } if overview.GiftCoinSpentDeltaRate != 1 || overview.CoinSellerTransferDeltaRate != 1.5 || overview.LuckyGiftTurnoverDeltaRate != 3 || overview.LuckyGiftPayoutDeltaRate != 1 || overview.LuckyGiftProfitDeltaRate != 5 { @@ -1445,6 +1521,8 @@ func TestQueryOverviewReturnsCountryBreakdownAndIncludesCoinSellerRecharge(t *te ('lalu', '2026-06-06', 99, 220, 3, 1, 900, 0, 0, 900, 1)`); err != nil { t.Fatalf("seed country statistics: %v", err) } + seedCanonicalActiveUsers(t, ctx, repository, "lalu", StatTZUTC, "2026-06-06", 86, 210, 8, 30_000) + seedCanonicalActiveUsers(t, ctx, repository, "lalu", StatTZUTC, "2026-06-06", 66, 210, 4, 40_000) start := time.Date(2026, 6, 6, 0, 0, 0, 0, time.UTC).UnixMilli() overview, err := repository.QueryOverview(ctx, OverviewQuery{AppCode: "lalu", StartMS: start, EndMS: start + int64(24*time.Hour/time.Millisecond), RegionID: 210}) @@ -1454,7 +1532,7 @@ func TestQueryOverviewReturnsCountryBreakdownAndIncludesCoinSellerRecharge(t *te if overview.RechargeUSDMinor != 650 || overview.CoinSellerRechargeUSDMinor != 400 || overview.CoinSellerStockCoin != 880000 { t.Fatalf("overview recharge mismatch: %+v", overview) } - if overview.RechargeUsers != 3 || overview.PaidConversionRate != 0.2 || overview.RechargeConversionRate != 0.2 { + if overview.ActiveUsers != 12 || overview.ARPUUSDMinor != 54 || overview.RechargeUsers != 3 || overview.PaidConversionRate != 0.25 || overview.RechargeConversionRate != 0.25 { t.Fatalf("overview conversion fields mismatch: %+v", overview) } if len(overview.CountryBreakdown) != 2 { @@ -1463,7 +1541,7 @@ func TestQueryOverviewReturnsCountryBreakdownAndIncludesCoinSellerRecharge(t *te if overview.CountryBreakdown[0].CountryID != 86 || overview.CountryBreakdown[0].RechargeUSDMinor != 550 || overview.CountryBreakdown[0].CoinSellerStockCoin != 880000 || overview.CountryBreakdown[0].ARPPUUSDMinor != 275 { t.Fatalf("first country breakdown mismatch: %+v", overview.CountryBreakdown[0]) } - if overview.CountryBreakdown[0].RechargeUsers != 2 || overview.CountryBreakdown[0].PaidConversionRate != 0.2 || overview.CountryBreakdown[0].RechargeConversionRate != 0.2 { + if overview.CountryBreakdown[0].ActiveUsers != 8 || overview.CountryBreakdown[0].ARPUUSDMinor != 68 || overview.CountryBreakdown[0].RechargeUsers != 2 || overview.CountryBreakdown[0].PaidConversionRate != 0.25 || overview.CountryBreakdown[0].RechargeConversionRate != 0.25 { t.Fatalf("first country conversion fields mismatch: %+v", overview.CountryBreakdown[0]) } } @@ -1491,6 +1569,9 @@ func TestQueryOverviewReturnsDailyCountryBreakdownAndSuperLuckyGift(t *testing.T ('lalu', '2026-06-06', 86, 210, 12, 3, 150, 9000, 1200, 300, 1)`); err != nil { t.Fatalf("seed app country stats: %v", err) } + seedCanonicalActiveUsers(t, ctx, repository, "lalu", StatTZUTC, "2026-06-05", 86, 210, 8, 50_000) + seedCanonicalActiveUsers(t, ctx, repository, "lalu", StatTZUTC, "2026-06-05", 66, 210, 4, 60_000) + seedCanonicalActiveUsers(t, ctx, repository, "lalu", StatTZUTC, "2026-06-06", 86, 210, 9, 70_000) if _, err := repository.db.ExecContext(ctx, ` INSERT INTO stat_lucky_gift_pool_day_country ( app_code, stat_day, country_id, region_id, pool_id, turnover_coin, payout_coin, updated_at_ms @@ -1509,6 +1590,9 @@ func TestQueryOverviewReturnsDailyCountryBreakdownAndSuperLuckyGift(t *testing.T if overview.SuperLuckyGiftTurnover != 1600 || overview.SuperLuckyGiftPayout != 300 || overview.SuperLuckyGiftProfit != 1300 { t.Fatalf("super lucky overview mismatch: %+v", overview) } + if overview.ActiveUsers != 21 || len(overview.DailySeries) != 2 || overview.DailySeries[0].ActiveUsers != 12 || overview.DailySeries[1].ActiveUsers != 9 { + t.Fatalf("overview and daily DAU must come from canonical Social: %+v", overview.DailySeries) + } if len(overview.DailyCountryBreakdown) != 3 { t.Fatalf("daily country breakdown length mismatch: %+v", overview.DailyCountryBreakdown) } @@ -1516,7 +1600,7 @@ func TestQueryOverviewReturnsDailyCountryBreakdownAndSuperLuckyGift(t *testing.T if first.StatDay != "2026-06-05" || first.CountryID != 86 || first.CoinSellerStockCoin != 7000 || first.SuperLuckyGiftTurnover != 700 || first.SuperLuckyGiftProfitRate <= 0 { t.Fatalf("first daily country row mismatch: %+v", first) } - if first.RechargeUsers != 2 || first.PaidConversionRate != 0.2 || first.RechargeConversionRate != 0.2 { + if first.ActiveUsers != 8 || first.RechargeUsers != 2 || first.PaidConversionRate != 0.25 || first.RechargeConversionRate != 0.25 { t.Fatalf("first daily country conversion fields mismatch: %+v", first) } if len(overview.ReportMetricSources) == 0 { @@ -1652,6 +1736,30 @@ func requireNilRetention(t *testing.T, name string, users *int64, base *int64, r } } +// seedCanonicalActiveUsers intentionally writes a value that can differ from +// stat_app_day_country.active_users. Overview tests then prove DAU and every +// related denominator are sourced from the canonical Social user-day table. +func seedCanonicalActiveUsers(t *testing.T, ctx context.Context, repository *Repository, app, statTZ, day string, countryID, regionID int64, count int, firstUserID int64) { + t.Helper() + if count <= 0 { + return + } + placeholders := make([]string, 0, count) + args := make([]any, 0, count*7) + for offset := 0; offset < count; offset++ { + userID := firstUserID + int64(offset) + placeholders = append(placeholders, "(?, ?, ?, ?, ?, '', ?, ?, 1, 1)") + args = append(args, app, statTZ, day, "u:"+strconv.FormatInt(userID, 10), userID, countryID, regionID) + } + if _, err := repository.db.ExecContext(ctx, ` + INSERT INTO stat_social_user_day ( + app_code, stat_tz, stat_day, subject_key, user_id, device_id, + country_id, region_id, active_user, updated_at_ms + ) VALUES `+strings.Join(placeholders, ","), args...); err != nil { + t.Fatalf("seed canonical active users for %s/%s/%s: %v", app, statTZ, day, err) + } +} + // 回看口径下观察日整天零活跃时没有 stat_app_day_country 载体行； // 序列/按日国家下钻必须合成占位行,把真实的 0% 留存保住,而不是让该观察日消失。 func TestQueryOverviewSynthesizesRetentionRowsForZeroActivityDays(t *testing.T) { diff --git a/services/statistics-service/internal/storage/mysql/social_requirements_query.go b/services/statistics-service/internal/storage/mysql/social_requirements_query.go index 39e42221..4dbb5307 100644 --- a/services/statistics-service/internal/storage/mysql/social_requirements_query.go +++ b/services/statistics-service/internal/storage/mysql/social_requirements_query.go @@ -415,6 +415,97 @@ type socialIdentityDeviceDayKey struct { DeviceID string } +// canonicalOverviewActiveDimension is the compact DAU projection consumed by +// the operation-wide overview. ActiveUsers is a sum of canonical user-days; +// RegionID follows the existing country-breakdown contract where one country +// row carries the greatest observed region ID after the requested region filter. +type canonicalOverviewActiveDimension struct { + ActiveUsers int64 + RegionID int64 +} + +type canonicalOverviewActiveCube struct { + Total int64 + ByDay map[string]int64 + ByCountry map[int64]canonicalOverviewActiveDimension + ByDayCountry map[string]canonicalOverviewActiveDimension +} + +func newCanonicalOverviewActiveCube() canonicalOverviewActiveCube { + return canonicalOverviewActiveCube{ + ByDay: map[string]int64{}, + ByCountry: map[int64]canonicalOverviewActiveDimension{}, + ByDayCountry: map[string]canonicalOverviewActiveDimension{}, + } +} + +// queryCanonicalOverviewActiveCube deliberately reads only the identity and +// dimension columns needed by DAU. The PK prefixes bound both scans by +// app+timezone+day; canonicalization stays in memory because a direct SQL SUM +// would count the same session once as d: and again as u:. +func (r *Repository) queryCanonicalOverviewActiveCube(ctx context.Context, app, statTZ, startDay, endDay string, countryID, regionID int64) (canonicalOverviewActiveCube, error) { + rows, err := r.db.QueryContext(ctx, ` + SELECT DATE_FORMAT(stat_day, '%Y-%m-%d'), subject_key, user_id, device_id, + country_id, region_id, user_role, active_user + FROM stat_social_user_day + WHERE app_code = ? AND stat_tz = ? AND stat_day BETWEEN ? AND ? + AND (active_user > 0 OR user_id > 0) + ORDER BY stat_day, subject_key + `, appcode.Normalize(app), normalizeStatTZ(statTZ), startDay, endDay) + if err != nil { + return canonicalOverviewActiveCube{}, err + } + defer rows.Close() + + userDays := []socialRequirementUserDay{} + for rows.Next() { + var item socialRequirementUserDay + if err := rows.Scan(&item.StatDay, &item.Subject, &item.UserID, &item.DeviceID, + &item.CountryID, &item.RegionID, &item.UserRole, &item.Active); err != nil { + return canonicalOverviewActiveCube{}, err + } + item.DirectUser = item.UserID > 0 + userDays = append(userDays, item) + } + if err := rows.Err(); err != nil { + return canonicalOverviewActiveCube{}, err + } + + identityByDeviceDay, err := r.querySocialIdentityDays(ctx, app, statTZ, startDay, endDay) + if err != nil { + return canonicalOverviewActiveCube{}, err + } + canonicalRows := canonicalizeSocialUserDays(userDays, identityByDeviceDay, countryID, regionID, nil) + return buildCanonicalOverviewActiveCube(canonicalRows), nil +} + +func buildCanonicalOverviewActiveCube(canonicalRows []socialRequirementUserDay) canonicalOverviewActiveCube { + cube := newCanonicalOverviewActiveCube() + for _, row := range canonicalRows { + if row.Active <= 0 { + continue + } + cube.Total++ + cube.ByDay[row.StatDay]++ + + country := cube.ByCountry[row.CountryID] + country.ActiveUsers++ + if row.RegionID > country.RegionID { + country.RegionID = row.RegionID + } + cube.ByCountry[row.CountryID] = country + + dayCountryKey := dailyCountryKey(row.StatDay, row.CountryID) + dayCountry := cube.ByDayCountry[dayCountryKey] + dayCountry.ActiveUsers++ + if row.RegionID > dayCountry.RegionID { + dayCountry.RegionID = row.RegionID + } + cube.ByDayCountry[dayCountryKey] = dayCountry + } + return cube +} + func (r *Repository) querySocialIdentityDays(ctx context.Context, app, statTZ, startDay, endDay string) (map[socialIdentityDeviceDayKey]int64, error) { rows, err := r.db.QueryContext(ctx, ` SELECT DATE_FORMAT(stat_day, '%Y-%m-%d'), device_id, resolved_user_id