From 1409529f10efa0b7c2ccccf0a1a2d9b9762b5674 Mon Sep 17 00:00:00 2001 From: local Date: Sun, 28 Jun 2026 15:11:56 +0800 Subject: [PATCH] =?UTF-8?q?=E6=96=B0BI=E7=B3=BB=E7=BB=9F?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- Makefile | 11 +- docker-compose.yml | 8 +- scripts/apply-local-mysql-initdb.sh | 8 +- scripts/databi-real-flow-check.sh | 51 + scripts/prepare-local-mysql-initdb.sh | 35 + .../admin/migrations/063_money_permission.sql | 26 + .../cmd/backfill-last7/main.go | 404 ++++++- .../cmd/databi-real-flow-audit/main.go | 294 +++++ .../cmd/replay-stat-tz/main.go | 102 +- .../mysql/initdb/001_statistics_service.sql | 34 + .../statistics-service/internal/app/app.go | 164 ++- .../internal/app/local_flow_test.go | 103 +- .../internal/storage/mysql/query.go | 475 ++++++-- .../internal/storage/mysql/query_test.go | 137 ++- .../internal/storage/mysql/repository.go | 312 ++++- services/user-service/configs/config.yaml | 1 + .../internal/service/wallet/service_test.go | 3 + .../internal/storage/mysql/admin_ledger.go | 2 +- .../storage/mysql/coin_seller_ledger.go | 4 +- .../internal/storage/mysql/game_ledger.go | 2 +- .../mysql/gift_batch_debit_repository.go | 6 +- .../storage/mysql/gift_debit_repository.go | 6 +- .../storage/mysql/host_salary_settlement.go | 6 +- .../internal/storage/mysql/outbox.go | 7 +- .../mysql/red_packet_create_repository.go | 2 +- .../storage/mysql/resource_shop_repository.go | 2 +- .../internal/storage/mysql/reward_ledger.go | 4 +- .../storage/mysql/vip_purchase_repository.go | 2 +- tools/databi-real-flow/main.go | 1059 +++++++++++++++++ 29 files changed, 3112 insertions(+), 158 deletions(-) create mode 100755 scripts/databi-real-flow-check.sh create mode 100755 scripts/prepare-local-mysql-initdb.sh create mode 100644 server/admin/migrations/063_money_permission.sql create mode 100644 services/statistics-service/cmd/databi-real-flow-audit/main.go create mode 100644 tools/databi-real-flow/main.go diff --git a/Makefile b/Makefile index 8318d010..1261bf5a 100644 --- a/Makefile +++ b/Makefile @@ -7,7 +7,7 @@ ADMIN_CONFIG ?= configs/config.yaml DEV_RUN_SERVICES ?= all DEV_RUN_SERVICE_LIST := $(strip $(if $(filter mysql redis,$(SERVICE_ID)),,$(if $(SERVICE_ID),$(SERVICE_ID),$(DEV_RUN_SERVICES)))) -.PHONY: proto vet test build run admin admin-up admin-deps admin-bootstrap admin-test admin-build up db-init rebuild restart stop logs ps addr down up-service check-service resolve-compose-conflicts $(SERVICE_ALIASES) +.PHONY: proto vet test build run admin admin-up admin-deps admin-bootstrap admin-test admin-build up db-init databi-real-flow-check rebuild restart stop logs ps addr down up-service check-service resolve-compose-conflicts $(SERVICE_ALIASES) # `proto` 负责把独立 api module 内的 .proto 重新生成成 Go 代码。 proto: @@ -48,6 +48,7 @@ build: # 可用 `make run rs`、`make run SERVICE=room-service` 或 `make run DEV_RUN_SERVICES=user-service,gateway-service` 只跑子集。 run: ./scripts/resolve-compose-container-conflicts.sh + ./scripts/prepare-local-mysql-initdb.sh ./scripts/prepare-local-rocketmq-config.sh ROCKETMQ_BROKER_CONFIG=./tmp/rocketmq/broker.local.conf docker compose up -d mysql redis rocketmq-namesrv rocketmq-broker ./scripts/apply-local-mysql-initdb.sh @@ -69,6 +70,7 @@ admin-up: admin-deps admin # `admin-deps` 拉起后台本地运行所需的 MySQL、Redis 和 user-service。 admin-deps: ./scripts/resolve-compose-container-conflicts.sh + ./scripts/prepare-local-mysql-initdb.sh docker compose up -d mysql redis ./scripts/apply-local-mysql-initdb.sh ./scripts/apply-local-rocketmq-topics.sh @@ -92,6 +94,7 @@ admin-build: # 完整 up 结束后自动启动 admin 前台进程。 up: ./scripts/resolve-compose-container-conflicts.sh + ./scripts/prepare-local-mysql-initdb.sh @if [ -z "$(SERVICE_ID)" ]; then \ docker compose up -d mysql redis; \ ./scripts/apply-local-mysql-initdb.sh; \ @@ -114,9 +117,14 @@ db-init: ./scripts/resolve-compose-container-conflicts.sh mysql ./scripts/apply-local-mysql-initdb.sh +# `databi-real-flow-check` 用真实 owner outbox 重放并审计 Databi 口径；充值允许走本地模拟回调，其余必须先通过真实业务流程产生事实。 +databi-real-flow-check: + ./scripts/databi-real-flow-check.sh + # `rebuild` 重新构建并启动单个本地服务。 rebuild: check-service ./scripts/resolve-compose-container-conflicts.sh + ./scripts/prepare-local-mysql-initdb.sh @if [ "$(SERVICE_ID)" != "mysql" ] && [ "$(SERVICE_ID)" != "redis" ]; then \ ./scripts/apply-local-mysql-initdb.sh; \ ./scripts/apply-local-rocketmq-topics.sh; \ @@ -178,6 +186,7 @@ resolve-compose-conflicts: # `up-service` 是 `up SERVICE=...` 的内部实现。 up-service: check-service ./scripts/resolve-compose-container-conflicts.sh + ./scripts/prepare-local-mysql-initdb.sh @if [ "$(SERVICE_ID)" != "mysql" ] && [ "$(SERVICE_ID)" != "redis" ]; then \ ./scripts/apply-local-mysql-initdb.sh; \ ./scripts/apply-local-rocketmq-topics.sh; \ diff --git a/docker-compose.yml b/docker-compose.yml index ed779c93..a2de2367 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -310,16 +310,16 @@ services: - ./services/user-service/deploy/mysql/initdb/001_user_service.sql:/docker-entrypoint-initdb.d/002_user_service.sql:ro - ./services/wallet-service/deploy/mysql/initdb/001_wallet_service.sql:/docker-entrypoint-initdb.d/003_wallet_service.sql:ro - ./services/activity-service/deploy/mysql/initdb/001_activity_service.sql:/docker-entrypoint-initdb.d/004_activity_service.sql:ro - - ${DEPLOY_PLATFORM_ROOT:-../deploy-platform}/deploy/mysql/initdb/005_utf8mb4_chinese_support.sql:/docker-entrypoint-initdb.d/005_utf8mb4_chinese_support.sql:ro - - ${DEPLOY_PLATFORM_ROOT:-../deploy-platform}/deploy/mysql/initdb/006_admin_database.sql:/docker-entrypoint-initdb.d/006_admin_database.sql:ro - - ${DEPLOY_PLATFORM_ROOT:-../deploy-platform}/deploy/mysql/initdb/007_resource_group_wallet_asset_items.sql:/docker-entrypoint-initdb.d/007_resource_group_wallet_asset_items.sql:ro + - ./tmp/mysql/initdb/005_utf8mb4_chinese_support.sql:/docker-entrypoint-initdb.d/005_utf8mb4_chinese_support.sql:ro + - ./tmp/mysql/initdb/006_admin_database.sql:/docker-entrypoint-initdb.d/006_admin_database.sql:ro + - ./tmp/mysql/initdb/007_resource_group_wallet_asset_items.sql:/docker-entrypoint-initdb.d/007_resource_group_wallet_asset_items.sql:ro - ./services/cron-service/deploy/mysql/initdb/001_cron_service.sql:/docker-entrypoint-initdb.d/009_cron_service.sql:ro - ./services/game-service/deploy/mysql/initdb/001_game_service.sql:/docker-entrypoint-initdb.d/010_game_service.sql:ro - ./services/game-service/deploy/mysql/initdata/001_self_games.sql:/docker-entrypoint-initdb.d/010_game_service_data.sql:ro - ./services/robot-service/deploy/mysql/initdb/001_robot_service.sql:/docker-entrypoint-initdb.d/011_robot_service.sql:ro - ./services/notice-service/deploy/mysql/initdb/001_notice_service.sql:/docker-entrypoint-initdb.d/011_notice_service.sql:ro - ./services/statistics-service/deploy/mysql/initdb/001_statistics_service.sql:/docker-entrypoint-initdb.d/012_statistics_service.sql:ro - - ${DEPLOY_PLATFORM_ROOT:-../deploy-platform}/deploy/mysql/initdb/999_local_grants.sql:/docker-entrypoint-initdb.d/999_local_grants.sql:ro + - ./tmp/mysql/initdb/999_local_grants.sql:/docker-entrypoint-initdb.d/999_local_grants.sql:ro healthcheck: test: ["CMD-SHELL", "mysqladmin ping -h 127.0.0.1 -uroot -proot --silent"] interval: 5s diff --git a/scripts/apply-local-mysql-initdb.sh b/scripts/apply-local-mysql-initdb.sh index b594861c..d04b0c44 100755 --- a/scripts/apply-local-mysql-initdb.sh +++ b/scripts/apply-local-mysql-initdb.sh @@ -29,6 +29,7 @@ SQL_FILES=( # Keep MySQL as the only required dependency for this script. Business services # are started after schema/grants are known to match the current repo. +"${SCRIPT_DIR}/prepare-local-mysql-initdb.sh" "${SCRIPT_DIR}/resolve-compose-container-conflicts.sh" mysql docker compose up -d mysql >/dev/null @@ -49,8 +50,11 @@ fi for sql_file in "${SQL_FILES[@]}"; do if [[ ! -f "${sql_file}" ]]; then - printf 'missing mysql init file: %s\n' "${sql_file}" >&2 - exit 1 + # Platform/admin seed files may live in a sibling deploy repository that + # is absent on lightweight app-backend checkouts. Treat those inputs as + # optional so local service startup can still exercise the owned schemas. + printf 'skipping mysql init file because it is absent or not a regular file: %s\n' "${sql_file}" + continue fi printf 'applying mysql init file: %s\n' "${sql_file}" diff --git a/scripts/databi-real-flow-check.sh b/scripts/databi-real-flow-check.sh new file mode 100755 index 00000000..a545e426 --- /dev/null +++ b/scripts/databi-real-flow-check.sh @@ -0,0 +1,51 @@ +#!/usr/bin/env bash +set -euo pipefail + +APP_CODE="${APP_CODE:-lalu}" +STAT_TZ="${STAT_TZ:-Asia/Shanghai}" +START_DAY="${START_DAY:-$(TZ=Asia/Shanghai date +%F)}" +END_DAY="${END_DAY:-$START_DAY}" + +STATISTICS_MYSQL_DSN="${STATISTICS_MYSQL_DSN:-hyapp:hyapp@tcp(127.0.0.1:23306)/hyapp_statistics?parseTime=true&charset=utf8mb4&loc=UTC&multiStatements=true}" +USER_MYSQL_DSN="${USER_MYSQL_DSN:-hyapp:hyapp@tcp(127.0.0.1:23306)/hyapp_user?parseTime=true&charset=utf8mb4&loc=UTC&multiStatements=true}" +WALLET_MYSQL_DSN="${WALLET_MYSQL_DSN:-hyapp:hyapp@tcp(127.0.0.1:23306)/hyapp_wallet?parseTime=true&charset=utf8mb4&loc=UTC&multiStatements=true}" +ROOM_MYSQL_DSN="${ROOM_MYSQL_DSN:-hyapp:hyapp@tcp(127.0.0.1:23306)/hyapp_room?parseTime=true&charset=utf8mb4&loc=UTC&multiStatements=true}" +GAME_MYSQL_DSN="${GAME_MYSQL_DSN:-hyapp:hyapp@tcp(127.0.0.1:23306)/hyapp_game?parseTime=true&charset=utf8mb4&loc=UTC&multiStatements=true}" + +export APP_CODE STAT_TZ START_DAY END_DAY +export STATISTICS_MYSQL_DSN USER_MYSQL_DSN WALLET_MYSQL_DSN ROOM_MYSQL_DSN GAME_MYSQL_DSN + +echo "Databi real-flow check" +echo " app=${APP_CODE}" +echo " stat_tz=${STAT_TZ}" +echo " days=${START_DAY}..${END_DAY}" +echo +echo "Before running this check, exercise the real local business flows for the selected days." +echo "Recharge is the only metric that may use a local simulated recharge/payment callback." +echo + +go run ./services/statistics-service/cmd/replay-stat-tz \ + -app "${APP_CODE}" \ + -timezone "${STAT_TZ}" \ + -start-day "${START_DAY}" \ + -end-day "${END_DAY}" \ + -apply \ + -statistics-dsn "${STATISTICS_MYSQL_DSN}" \ + -user-dsn "${USER_MYSQL_DSN}" \ + -wallet-dsn "${WALLET_MYSQL_DSN}" \ + -room-dsn "${ROOM_MYSQL_DSN}" \ + -game-dsn "${GAME_MYSQL_DSN}" + +go run ./services/statistics-service/cmd/databi-real-flow-audit \ + -app "${APP_CODE}" \ + -timezone "${STAT_TZ}" \ + -start-day "${START_DAY}" \ + -end-day "${END_DAY}" \ + -statistics-dsn "${STATISTICS_MYSQL_DSN}" \ + -user-dsn "${USER_MYSQL_DSN}" \ + -wallet-dsn "${WALLET_MYSQL_DSN}" \ + -room-dsn "${ROOM_MYSQL_DSN}" \ + -game-dsn "${GAME_MYSQL_DSN}" + +echo +echo "Databi real-flow check passed." diff --git a/scripts/prepare-local-mysql-initdb.sh b/scripts/prepare-local-mysql-initdb.sh new file mode 100755 index 00000000..48e46b13 --- /dev/null +++ b/scripts/prepare-local-mysql-initdb.sh @@ -0,0 +1,35 @@ +#!/usr/bin/env bash +set -euo pipefail + +# Docker creates a missing bind-mounted file source as a directory. MySQL's +# entrypoint then treats that directory as a SQL input and exits before the +# local database is usable. Keep optional platform SQL behind generated files: +# present sources are copied and executed, absent sources become harmless no-op +# SQL files so lightweight backend checkouts can still start. +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +PROJECT_ROOT="$(cd "${SCRIPT_DIR}/.." && pwd)" +DEPLOY_PLATFORM_ROOT="${DEPLOY_PLATFORM_ROOT:-${PROJECT_ROOT}/../deploy-platform}" +LOCAL_INITDB_DIR="${PROJECT_ROOT}/tmp/mysql/initdb" + +OPTIONAL_SQL_FILES=( + "005_utf8mb4_chinese_support.sql" + "006_admin_database.sql" + "007_resource_group_wallet_asset_items.sql" + "999_local_grants.sql" +) + +mkdir -p "${LOCAL_INITDB_DIR}" + +for sql_name in "${OPTIONAL_SQL_FILES[@]}"; do + source_file="${DEPLOY_PLATFORM_ROOT}/deploy/mysql/initdb/${sql_name}" + target_file="${LOCAL_INITDB_DIR}/${sql_name}" + + if [[ -f "${source_file}" ]]; then + cp "${source_file}" "${target_file}" + printf 'prepared optional mysql init file: %s\n' "${source_file}" + continue + fi + + printf -- '-- optional mysql init file is absent locally: %s\n' "${source_file}" > "${target_file}" + printf 'prepared empty optional mysql init file: %s\n' "${source_file}" +done diff --git a/server/admin/migrations/063_money_permission.sql b/server/admin/migrations/063_money_permission.sql new file mode 100644 index 00000000..6bd91afb --- /dev/null +++ b/server/admin/migrations/063_money_permission.sql @@ -0,0 +1,26 @@ +SET NAMES utf8mb4 COLLATE utf8mb4_unicode_ci; + +-- 财务系统是独立子系统入口，不挂后台左侧菜单；这里只登记访问权限，供 /money 独立入口校验当前账号是否可进入。 +SET @now_ms = CAST(UNIX_TIMESTAMP(UTC_TIMESTAMP(3)) * 1000 AS UNSIGNED); + +INSERT INTO admin_permissions (name, code, kind, description, created_at_ms, updated_at_ms) VALUES + ('财务系统查看', 'money:view', 'menu', '允许访问独立财务子系统 /money', @now_ms, @now_ms) +ON DUPLICATE KEY UPDATE + name = VALUES(name), + kind = VALUES(kind), + description = VALUES(description), + updated_at_ms = @now_ms; + +INSERT IGNORE INTO admin_role_permissions (role_id, permission_id) +SELECT admin_role.id, admin_permission.id +FROM admin_roles admin_role +JOIN admin_permissions admin_permission +WHERE admin_role.code = 'platform-admin' + AND admin_permission.code = 'money:view'; + +INSERT IGNORE INTO admin_user_roles (user_id, role_id) +SELECT admin_user.id, admin_role.id +FROM admin_users admin_user +JOIN admin_roles admin_role +WHERE admin_user.username IN ('admin', 'hyappadmin') + AND admin_role.code = 'platform-admin'; diff --git a/services/statistics-service/cmd/backfill-last7/main.go b/services/statistics-service/cmd/backfill-last7/main.go index bf8b88aa..dbd91bb4 100644 --- a/services/statistics-service/cmd/backfill-last7/main.go +++ b/services/statistics-service/cmd/backfill-last7/main.go @@ -33,9 +33,18 @@ type appDayAgg struct { rechargeUSDMinor int64 newUserRechargeUSDMinor int64 coinSellerRechargeUSDMinor int64 + coinSellerStockCoin int64 mifaPayRechargeUSDMinor int64 googleRechargeUSDMinor int64 coinSellerTransferCoin int64 + coinTotal int64 + consumedCoin int64 + platformGrantCoin int64 + manualGrantCoin int64 + salaryUSDMinor int64 + salaryTransferCoin int64 + micOnlineMS int64 + micOnlineUsers int64 } type userRegistrationRow struct { @@ -59,12 +68,20 @@ type activeRow struct { firstMS int64 } +type micRow struct { + day string + userID int64 + dim countryRegion + micOnlineMS int64 +} + type rebuildState struct { appDays map[appDayKey]*appDayAgg registrations []userRegistrationRow registeredByUserDay map[string]struct{} payersByKey map[string]payerRow activeByKey map[string]activeRow + micByKey map[string]micRow userIDs map[int64]struct{} userDims map[int64]countryRegion knownUsers map[int64]struct{} @@ -113,6 +130,7 @@ func main() { registeredByUserDay: map[string]struct{}{}, payersByKey: map[string]payerRow{}, activeByKey: map[string]activeRow{}, + micByKey: map[string]micRow{}, userIDs: map[int64]struct{}{}, userDims: map[int64]countryRegion{}, knownUsers: map[int64]struct{}{}, @@ -127,6 +145,9 @@ func main() { if err := collectRechargeUserIDs(ctx, walletDB, state, app, windowStartMS, windowEndMS); err != nil { log.Fatalf("collect recharge ids: %v", err) } + if err := collectReportMetricUserIDs(ctx, walletDB, userDB, state, app, startDay, endDay, windowEndMS); err != nil { + log.Fatalf("collect report metric ids: %v", err) + } if err := loadUserDims(ctx, userDB, state, app); err != nil { log.Fatalf("load user dims: %v", err) } @@ -142,6 +163,15 @@ func main() { if err := rebuildCoinSellerStock(ctx, walletDB, state, app, windowStartMS, windowEndMS, loc); err != nil { log.Fatalf("rebuild coin seller stock: %v", err) } + if err := rebuildWalletReportMetrics(ctx, walletDB, state, app, startDay, endDay, windowStartMS, windowEndMS, loc); err != nil { + log.Fatalf("rebuild wallet report metrics: %v", err) + } + if err := rebuildSalary(ctx, walletDB, state, app, windowStartMS, windowEndMS, loc); err != nil { + log.Fatalf("rebuild salary: %v", err) + } + if err := rebuildMicStats(ctx, userDB, state, app, startDay, endDay); err != nil { + log.Fatalf("rebuild mic stats: %v", err) + } if err := stageAndMaybeApply(ctx, statsDB, state, app, statTZ, startDay, endDay, *applyFlag); err != nil { log.Fatalf("stage/apply: %v", err) @@ -310,6 +340,82 @@ func collectRechargeUserIDs(ctx context.Context, db *sql.DB, state *rebuildState return nil } +func collectReportMetricUserIDs(ctx context.Context, walletDB *sql.DB, userDB *sql.DB, state *rebuildState, app string, startDay string, endDay string, windowEndMS int64) error { + queries := []struct { + db *sql.DB + query string + args []any + }{ + { + db: walletDB, + query: ` + SELECT DISTINCT e.user_id + FROM wallet_entries e + JOIN wallet_transactions wt ON wt.transaction_id = e.transaction_id + WHERE wt.app_code = ? AND e.asset_type IN ('COIN', 'COIN_SELLER_COIN', 'HOST_SALARY_USD', 'AGENCY_SALARY_USD') AND e.created_at_ms < ? + UNION + SELECT DISTINCT e.counterparty_user_id + FROM wallet_entries e + JOIN wallet_transactions wt ON wt.transaction_id = e.transaction_id + WHERE wt.app_code = ? AND wt.biz_type = 'salary_transfer_to_coin_seller' + AND e.asset_type = 'COIN_SELLER_COIN' AND e.counterparty_user_id > 0 AND e.created_at_ms < ?`, + args: []any{app, windowEndMS, app, windowEndMS}, + }, + { + db: walletDB, + query: ` + SELECT DISTINCT user_id + FROM wallet_accounts + WHERE app_code = ? AND asset_type = 'COIN' AND available_amount + frozen_amount <> 0`, + args: []any{app}, + }, + { + db: walletDB, + query: ` + SELECT DISTINCT user_id + FROM host_salary_settlement_records + WHERE app_code = ? AND status = 'succeeded' AND created_at_ms < ?`, + args: []any{app, windowEndMS}, + }, + { + db: walletDB, + query: ` + SELECT DISTINCT agency_owner_user_id + FROM host_salary_settlement_records + WHERE app_code = ? AND status = 'succeeded' AND agency_owner_user_id > 0 AND created_at_ms < ?`, + args: []any{app, windowEndMS}, + }, + { + db: userDB, + query: ` + SELECT DISTINCT user_id + FROM user_mic_daily_stats + WHERE app_code = ? AND stat_date BETWEEN ? AND ?`, + args: []any{app, startDay, endDay}, + }, + } + for _, item := range queries { + rows, err := item.db.QueryContext(ctx, item.query, item.args...) + if err != nil { + return err + } + for rows.Next() { + var userID int64 + if err := rows.Scan(&userID); err != nil { + rows.Close() + return err + } + if userID > 0 { + state.userIDs[userID] = struct{}{} + } + } + if err := rows.Close(); err != nil { + return err + } + } + return nil +} + func loadUserDims(ctx context.Context, db *sql.DB, state *rebuildState, app string) error { userIDs := make([]int64, 0, len(state.userIDs)) for userID := range state.userIDs { @@ -460,17 +566,17 @@ func rebuildWalletRecharge(ctx context.Context, db *sql.DB, state *rebuildState, func rebuildCoinSellerStock(ctx context.Context, db *sql.DB, state *rebuildState, app string, startMS, endMS int64, loc *time.Location) error { // 币商进货是平台收到的 USDT 充值，按 seller 的国家/区域进入总充值，但不写入用户金币转账金额。 rows, err := db.QueryContext(ctx, ` - SELECT seller_user_id, paid_amount_micro, created_at_ms + SELECT seller_user_id, paid_amount_micro, coin_amount, created_at_ms FROM coin_seller_stock_records - WHERE app_code = ? AND counts_as_seller_recharge = 1 AND paid_amount_micro <> 0 AND created_at_ms >= ? AND created_at_ms < ?`, + WHERE app_code = ? AND counts_as_seller_recharge = 1 AND (paid_amount_micro <> 0 OR coin_amount <> 0) AND created_at_ms >= ? AND created_at_ms < ?`, app, startMS, endMS) if err != nil { return err } defer rows.Close() for rows.Next() { - var sellerUserID, amountMicro, createdAtMS int64 - if err := rows.Scan(&sellerUserID, &amountMicro, &createdAtMS); err != nil { + var sellerUserID, amountMicro, coinAmount, createdAtMS int64 + if err := rows.Scan(&sellerUserID, &amountMicro, &coinAmount, &createdAtMS); err != nil { return err } usdMinor := amountMicro / 10_000 @@ -479,10 +585,217 @@ func rebuildCoinSellerStock(ctx context.Context, db *sql.DB, state *rebuildState agg := state.ensureAgg(day, dim) agg.rechargeUSDMinor += usdMinor agg.coinSellerRechargeUSDMinor += usdMinor + agg.coinSellerStockCoin += coinAmount } return rows.Err() } +func rebuildWalletReportMetrics(ctx context.Context, db *sql.DB, state *rebuildState, app string, startDay string, endDay string, startMS int64, endMS int64, loc *time.Location) error { + if err := rebuildCoinTotals(ctx, db, state, app, startDay, endDay, endMS, loc); err != nil { + return err + } + rows, err := db.QueryContext(ctx, ` + SELECT e.user_id, e.counterparty_user_id, e.asset_type, e.available_delta + e.frozen_delta AS amount_delta, wt.biz_type, e.created_at_ms + FROM wallet_entries e + JOIN wallet_transactions wt ON wt.transaction_id = e.transaction_id + WHERE wt.app_code = ? AND e.asset_type IN ('COIN', 'COIN_SELLER_COIN') AND e.created_at_ms >= ? AND e.created_at_ms < ?`, + app, startMS, endMS) + if err != nil { + return err + } + defer rows.Close() + for rows.Next() { + var userID, counterpartyUserID, amountDelta, createdAtMS int64 + var assetType, bizType string + if err := rows.Scan(&userID, &counterpartyUserID, &assetType, &amountDelta, &bizType, &createdAtMS); err != nil { + return err + } + day := dayFromMS(createdAtMS, loc) + normalizedBiz := strings.ToLower(strings.TrimSpace(bizType)) + normalizedAsset := strings.ToUpper(strings.TrimSpace(assetType)) + if amountDelta > 0 && normalizedBiz == "salary_transfer_to_coin_seller" && normalizedAsset == "COIN_SELLER_COIN" { + dimensionUserID := userID + if counterpartyUserID > 0 { + dimensionUserID = counterpartyUserID + } + agg := state.ensureAgg(day, state.userDims[dimensionUserID]) + agg.salaryTransferCoin += amountDelta + continue + } + if normalizedAsset != "COIN" { + continue + } + dim := state.userDims[userID] + agg := state.ensureAgg(day, dim) + switch { + case amountDelta < 0 && isConsumedCoinBizType(normalizedBiz): + agg.consumedCoin += -amountDelta + case amountDelta > 0 && normalizedBiz == "manual_credit": + agg.manualGrantCoin += amountDelta + case amountDelta > 0 && isPlatformGrantBizType(normalizedBiz): + agg.platformGrantCoin += amountDelta + } + } + return rows.Err() +} + +type coinBalanceEntry struct { + userID int64 + balance int64 + createdAtMS int64 +} + +func rebuildCoinTotals(ctx context.Context, db *sql.DB, state *rebuildState, app string, startDay string, endDay string, windowEndMS int64, loc *time.Location) error { + dayEnds, err := dayEndPoints(startDay, endDay, loc) + if err != nil { + return err + } + rows, err := db.QueryContext(ctx, ` + SELECT e.user_id, e.available_after + e.frozen_after AS balance_after, e.created_at_ms + FROM wallet_entries e + JOIN wallet_transactions wt ON wt.transaction_id = e.transaction_id + WHERE wt.app_code = ? AND e.asset_type = 'COIN' AND e.created_at_ms < ? + ORDER BY e.user_id ASC, e.created_at_ms ASC, e.entry_id ASC`, + app, windowEndMS) + if err != nil { + return err + } + defer rows.Close() + var currentUserID int64 + entries := []coinBalanceEntry{} + flush := func() { + if currentUserID <= 0 || len(entries) == 0 { + return + } + dim := state.userDims[currentUserID] + index := 0 + balance := int64(0) + hasBalance := false + for _, point := range dayEnds { + for index < len(entries) && entries[index].createdAtMS < point.endMS { + balance = entries[index].balance + hasBalance = true + index++ + } + if hasBalance { + state.ensureAgg(point.day, dim).coinTotal += balance + } + } + } + for rows.Next() { + var entry coinBalanceEntry + if err := rows.Scan(&entry.userID, &entry.balance, &entry.createdAtMS); err != nil { + return err + } + if currentUserID != 0 && entry.userID != currentUserID { + flush() + entries = entries[:0] + } + currentUserID = entry.userID + entries = append(entries, entry) + } + if err := rows.Err(); err != nil { + return err + } + flush() + return nil +} + +func rebuildSalary(ctx context.Context, db *sql.DB, state *rebuildState, app string, startMS int64, endMS int64, loc *time.Location) error { + rows, err := db.QueryContext(ctx, ` + SELECT user_id, agency_owner_user_id, host_salary_usd_minor_delta, agency_salary_usd_minor_delta, created_at_ms + FROM host_salary_settlement_records + WHERE app_code = ? AND status = 'succeeded' AND created_at_ms >= ? AND created_at_ms < ?`, + app, startMS, endMS) + if err != nil { + return err + } + defer rows.Close() + for rows.Next() { + var hostUserID, agencyUserID, hostSalary, agencySalary, createdAtMS int64 + if err := rows.Scan(&hostUserID, &agencyUserID, &hostSalary, &agencySalary, &createdAtMS); err != nil { + return err + } + day := dayFromMS(createdAtMS, loc) + if hostSalary > 0 { + state.ensureAgg(day, state.userDims[hostUserID]).salaryUSDMinor += hostSalary + } + if agencySalary > 0 && agencyUserID > 0 { + state.ensureAgg(day, state.userDims[agencyUserID]).salaryUSDMinor += agencySalary + } + } + return rows.Err() +} + +func rebuildMicStats(ctx context.Context, db *sql.DB, state *rebuildState, app string, startDay string, endDay string) error { + rows, err := db.QueryContext(ctx, ` + SELECT DATE_FORMAT(stat_date, '%Y-%m-%d'), user_id, mic_online_ms + FROM user_mic_daily_stats + WHERE app_code = ? AND stat_date BETWEEN ? AND ?`, + app, startDay, endDay) + if err != nil { + return err + } + defer rows.Close() + for rows.Next() { + var day string + var userID, micOnlineMS int64 + if err := rows.Scan(&day, &userID, &micOnlineMS); err != nil { + return err + } + if micOnlineMS <= 0 { + continue + } + dim := state.userDims[userID] + state.addMic(day, userID, dim, micOnlineMS) + } + return rows.Err() +} + +type dayEndPoint struct { + day string + endMS int64 +} + +func dayEndPoints(startDay string, endDay string, loc *time.Location) ([]dayEndPoint, error) { + start, err := parseDay(startDay, loc) + if err != nil { + return nil, err + } + end, err := parseDay(endDay, loc) + if err != nil { + return nil, err + } + points := []dayEndPoint{} + for cursor := start; !cursor.After(end); cursor = cursor.AddDate(0, 0, 1) { + points = append(points, dayEndPoint{ + day: formatDay(cursor), + endMS: cursor.AddDate(0, 0, 1).UTC().UnixMilli(), + }) + } + return points, nil +} + +func isConsumedCoinBizType(value string) bool { + switch strings.ToLower(strings.TrimSpace(value)) { + case "gift_debit", "direct_gift_debit", "vip_purchase", "resource_shop_purchase", + "red_packet_create", "game_debit", "wheel_draw_debit", "cp_breakup_fee": + return true + default: + return false + } +} + +func isPlatformGrantBizType(value string) bool { + switch strings.ToLower(strings.TrimSpace(value)) { + case "task_reward", "lucky_gift_reward", "wheel_reward", "room_turnover_reward", + "invite_activity_reward", "agency_opening_reward": + return true + default: + return false + } +} + func (s *rebuildState) ensureAgg(day string, dim countryRegion) *appDayAgg { key := appDayKey{day: day, countryID: dim.countryID, regionID: dim.regionID} if s.appDays[key] == nil { @@ -517,6 +830,21 @@ func (s *rebuildState) addActive(day string, userID int64, dim countryRegion, fi s.ensureAgg(day, dim).activeUsers++ } +func (s *rebuildState) addMic(day string, userID int64, dim countryRegion, micOnlineMS int64) { + key := userDayKey(day, userID) + if existing, exists := s.micByKey[key]; exists { + agg := s.ensureAgg(day, existing.dim) + agg.micOnlineMS += micOnlineMS - existing.micOnlineMS + existing.micOnlineMS = micOnlineMS + s.micByKey[key] = existing + return + } + s.micByKey[key] = micRow{day: day, userID: userID, dim: dim, micOnlineMS: micOnlineMS} + agg := s.ensureAgg(day, dim) + agg.micOnlineMS += micOnlineMS + agg.micOnlineUsers++ +} + func (s *rebuildState) userRegisteredOn(userID int64, day string) bool { _, ok := s.registeredByUserDay[userDayKey(day, userID)] return ok @@ -556,7 +884,10 @@ func stageAndMaybeApply(ctx context.Context, db *sql.DB, state *rebuildState, ap SET new_users = 0, active_users = 0, paid_users = 0, recharge_usd_minor = 0, new_user_recharge_usd_minor = 0, coin_seller_recharge_usd_minor = 0, + coin_seller_stock_coin = 0, mifapay_recharge_usd_minor = 0, google_recharge_usd_minor = 0, coin_seller_transfer_coin = 0, + coin_total = 0, consumed_coin = 0, platform_grant_coin = 0, manual_grant_coin = 0, + salary_usd_minor = 0, salary_transfer_coin = 0, mic_online_ms = 0, mic_online_users = 0, updated_at_ms = ? WHERE app_code = ? AND stat_tz = ? AND stat_day BETWEEN ? AND ?`, nowMS, app, statTZ, startDay, endDay); err != nil { @@ -566,29 +897,55 @@ func stageAndMaybeApply(ctx context.Context, db *sql.DB, state *rebuildState, ap `DELETE FROM stat_user_registration WHERE app_code = ? AND stat_tz = ? AND registered_day BETWEEN ? AND ?`, `DELETE FROM stat_user_day_activity WHERE app_code = ? AND stat_tz = ? AND stat_day BETWEEN ? AND ?`, `DELETE FROM stat_recharge_day_payers WHERE app_code = ? AND stat_tz = ? AND stat_day BETWEEN ? AND ?`, + `DELETE FROM stat_user_day_mic WHERE app_code = ? AND stat_tz = ? AND stat_day BETWEEN ? AND ?`, } { if _, err := tx.ExecContext(ctx, statement, app, statTZ, startDay, endDay); err != nil { return err } } + for userID, dim := range state.userDims { + if _, err := tx.ExecContext(ctx, ` + INSERT INTO stat_user_dimension (app_code, user_id, country_id, region_id, updated_at_ms) + VALUES (?, ?, ?, ?, ?) + ON DUPLICATE KEY UPDATE + country_id = VALUES(country_id), + region_id = VALUES(region_id), + updated_at_ms = VALUES(updated_at_ms) + `, app, userID, dim.countryID, dim.regionID, nowMS); err != nil { + return err + } + } if _, err := tx.ExecContext(ctx, ` INSERT INTO stat_app_day_country ( app_code, stat_tz, stat_day, country_id, region_id, new_users, active_users, paid_users, recharge_usd_minor, new_user_recharge_usd_minor, coin_seller_recharge_usd_minor, - mifapay_recharge_usd_minor, google_recharge_usd_minor, coin_seller_transfer_coin, updated_at_ms + coin_seller_stock_coin, mifapay_recharge_usd_minor, google_recharge_usd_minor, coin_seller_transfer_coin, + coin_total, consumed_coin, platform_grant_coin, manual_grant_coin, + salary_usd_minor, salary_transfer_coin, mic_online_ms, mic_online_users, updated_at_ms ) SELECT app_code, stat_tz, stat_day, country_id, region_id, new_users, active_users, paid_users, recharge_usd_minor, new_user_recharge_usd_minor, coin_seller_recharge_usd_minor, - mifapay_recharge_usd_minor, google_recharge_usd_minor, coin_seller_transfer_coin, ? + coin_seller_stock_coin, mifapay_recharge_usd_minor, google_recharge_usd_minor, coin_seller_transfer_coin, + coin_total, consumed_coin, platform_grant_coin, manual_grant_coin, + salary_usd_minor, salary_transfer_coin, mic_online_ms, mic_online_users, ? FROM tmp_stat_backfill_app_day ON DUPLICATE KEY UPDATE new_users = VALUES(new_users), active_users = VALUES(active_users), paid_users = VALUES(paid_users), recharge_usd_minor = VALUES(recharge_usd_minor), new_user_recharge_usd_minor = VALUES(new_user_recharge_usd_minor), coin_seller_recharge_usd_minor = VALUES(coin_seller_recharge_usd_minor), + coin_seller_stock_coin = VALUES(coin_seller_stock_coin), mifapay_recharge_usd_minor = VALUES(mifapay_recharge_usd_minor), google_recharge_usd_minor = VALUES(google_recharge_usd_minor), coin_seller_transfer_coin = VALUES(coin_seller_transfer_coin), + coin_total = VALUES(coin_total), + consumed_coin = VALUES(consumed_coin), + platform_grant_coin = VALUES(platform_grant_coin), + manual_grant_coin = VALUES(manual_grant_coin), + salary_usd_minor = VALUES(salary_usd_minor), + salary_transfer_coin = VALUES(salary_transfer_coin), + mic_online_ms = VALUES(mic_online_ms), + mic_online_users = VALUES(mic_online_users), updated_at_ms = VALUES(updated_at_ms)`, nowMS); err != nil { return err } @@ -607,6 +964,11 @@ func stageAndMaybeApply(ctx context.Context, db *sql.DB, state *rebuildState, ap SELECT app_code, stat_tz, stat_day, country_id, region_id, user_id, first_paid_at_ms FROM tmp_stat_backfill_payers`); err != nil { return err } + if _, err := tx.ExecContext(ctx, ` + INSERT INTO stat_user_day_mic (app_code, stat_tz, stat_day, country_id, region_id, user_id, mic_online_ms, updated_at_ms) + SELECT app_code, stat_tz, stat_day, country_id, region_id, user_id, mic_online_ms, ? FROM tmp_stat_backfill_mic`, nowMS); err != nil { + return err + } return tx.Commit() } @@ -616,8 +978,12 @@ func createTempTables(ctx context.Context, tx *sql.Tx) error { app_code VARCHAR(32) NOT NULL, stat_tz VARCHAR(64) NOT NULL, stat_day DATE NOT NULL, country_id BIGINT NOT NULL, region_id BIGINT NOT NULL, new_users BIGINT NOT NULL, active_users BIGINT NOT NULL, paid_users BIGINT NOT NULL, recharge_usd_minor BIGINT NOT NULL, new_user_recharge_usd_minor BIGINT NOT NULL, - coin_seller_recharge_usd_minor BIGINT NOT NULL, mifapay_recharge_usd_minor BIGINT NOT NULL, google_recharge_usd_minor BIGINT NOT NULL, + coin_seller_recharge_usd_minor BIGINT NOT NULL, coin_seller_stock_coin BIGINT NOT NULL, + mifapay_recharge_usd_minor BIGINT NOT NULL, google_recharge_usd_minor BIGINT NOT NULL, coin_seller_transfer_coin BIGINT NOT NULL, + coin_total BIGINT NOT NULL, consumed_coin BIGINT NOT NULL, platform_grant_coin BIGINT NOT NULL, + manual_grant_coin BIGINT NOT NULL, salary_usd_minor BIGINT NOT NULL, salary_transfer_coin BIGINT NOT NULL, + mic_online_ms BIGINT NOT NULL, mic_online_users BIGINT NOT NULL, PRIMARY KEY(app_code, stat_tz, stat_day, country_id, region_id) )`, `CREATE TEMPORARY TABLE tmp_stat_backfill_registration ( @@ -635,6 +1001,11 @@ func createTempTables(ctx context.Context, tx *sql.Tx) error { region_id BIGINT NOT NULL, user_id BIGINT NOT NULL, first_paid_at_ms BIGINT NOT NULL, PRIMARY KEY(app_code, stat_tz, stat_day, user_id) )`, + `CREATE TEMPORARY TABLE tmp_stat_backfill_mic ( + app_code VARCHAR(32) NOT NULL, stat_tz VARCHAR(64) NOT NULL, stat_day DATE NOT NULL, country_id BIGINT NOT NULL, + region_id BIGINT NOT NULL, user_id BIGINT NOT NULL, mic_online_ms BIGINT NOT NULL, + PRIMARY KEY(app_code, stat_tz, stat_day, user_id) + )`, } for _, statement := range statements { if _, err := tx.ExecContext(ctx, statement); err != nil { @@ -650,11 +1021,13 @@ func insertStageRows(ctx context.Context, tx *sql.Tx, state *rebuildState, app s INSERT INTO tmp_stat_backfill_app_day ( app_code, stat_tz, stat_day, country_id, region_id, new_users, active_users, paid_users, recharge_usd_minor, new_user_recharge_usd_minor, coin_seller_recharge_usd_minor, - mifapay_recharge_usd_minor, google_recharge_usd_minor, coin_seller_transfer_coin - ) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)`, + coin_seller_stock_coin, mifapay_recharge_usd_minor, google_recharge_usd_minor, coin_seller_transfer_coin, + coin_total, consumed_coin, platform_grant_coin, manual_grant_coin, salary_usd_minor, salary_transfer_coin, mic_online_ms, mic_online_users + ) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)`, app, statTZ, key.day, key.countryID, key.regionID, agg.newUsers, agg.activeUsers, agg.paidUsers, agg.rechargeUSDMinor, agg.newUserRechargeUSDMinor, agg.coinSellerRechargeUSDMinor, - agg.mifaPayRechargeUSDMinor, agg.googleRechargeUSDMinor, agg.coinSellerTransferCoin); err != nil { + agg.coinSellerStockCoin, agg.mifaPayRechargeUSDMinor, agg.googleRechargeUSDMinor, agg.coinSellerTransferCoin, + agg.coinTotal, agg.consumedCoin, agg.platformGrantCoin, agg.manualGrantCoin, agg.salaryUSDMinor, agg.salaryTransferCoin, agg.micOnlineMS, agg.micOnlineUsers); err != nil { return err } } @@ -679,6 +1052,13 @@ func insertStageRows(ctx context.Context, tx *sql.Tx, state *rebuildState, app s return err } } + for _, row := range state.micByKey { + if _, err := tx.ExecContext(ctx, ` + INSERT INTO tmp_stat_backfill_mic (app_code, stat_tz, stat_day, country_id, region_id, user_id, mic_online_ms) + VALUES (?, ?, ?, ?, ?, ?, ?)`, app, statTZ, row.day, row.dim.countryID, row.dim.regionID, row.userID, row.micOnlineMS); err != nil { + return err + } + } return nil } @@ -687,7 +1067,9 @@ func validateStage(ctx context.Context, tx *sql.Tx) error { if err := tx.QueryRowContext(ctx, ` SELECT COUNT(*) FROM tmp_stat_backfill_app_day - WHERE new_users < 0 OR active_users < 0 OR paid_users < 0 OR recharge_usd_minor < 0 OR coin_seller_transfer_coin < 0`).Scan(&negativeCount); err != nil { + WHERE new_users < 0 OR active_users < 0 OR paid_users < 0 OR recharge_usd_minor < 0 OR coin_seller_transfer_coin < 0 + OR coin_total < 0 OR consumed_coin < 0 OR platform_grant_coin < 0 OR manual_grant_coin < 0 + OR salary_usd_minor < 0 OR salary_transfer_coin < 0 OR mic_online_ms < 0 OR mic_online_users < 0`).Scan(&negativeCount); err != nil { return err } if negativeCount > 0 { diff --git a/services/statistics-service/cmd/databi-real-flow-audit/main.go b/services/statistics-service/cmd/databi-real-flow-audit/main.go new file mode 100644 index 00000000..d0f84c9f --- /dev/null +++ b/services/statistics-service/cmd/databi-real-flow-audit/main.go @@ -0,0 +1,294 @@ +package main + +import ( + "context" + "database/sql" + "flag" + "fmt" + "log" + "os" + "strings" + "time" + + mysqlstorage "hyapp/services/statistics-service/internal/storage/mysql" + + _ "github.com/go-sql-driver/mysql" +) + +type auditWindow struct { + startDay string + endDay string + startMS int64 + endMS int64 +} + +type auditCheck struct { + key string + label string + source string + required bool + count int64 + err error +} + +func main() { + var ( + appCode = flag.String("app", envDefault("APP_CODE", "lalu"), "app_code to audit") + statTZ = flag.String("timezone", envDefault("STAT_TZ", mysqlstorage.StatTZAsiaShanghai), "stat_tz to audit: UTC or Asia/Shanghai") + startDay = flag.String("start-day", envDefault("START_DAY", todayIn(mysqlstorage.StatTZAsiaShanghai)), "inclusive stat day in selected timezone, yyyy-mm-dd") + endDay = flag.String("end-day", envDefault("END_DAY", todayIn(mysqlstorage.StatTZAsiaShanghai)), "inclusive stat day in selected timezone, yyyy-mm-dd") + statisticsDSN = flag.String("statistics-dsn", firstNonEmptyEnv("STATISTICS_MYSQL_DSN", "MYSQL_DSN"), "statistics mysql dsn") + userDSN = flag.String("user-dsn", firstNonEmptyEnv("USER_MYSQL_DSN", "MYSQL_USER_DSN"), "user-service mysql dsn") + walletDSN = flag.String("wallet-dsn", firstNonEmptyEnv("WALLET_MYSQL_DSN", "MYSQL_WALLET_DSN"), "wallet-service mysql dsn") + roomDSN = flag.String("room-dsn", firstNonEmptyEnv("ROOM_MYSQL_DSN", "MYSQL_ROOM_DSN"), "room-service mysql dsn") + gameDSN = flag.String("game-dsn", firstNonEmptyEnv("GAME_MYSQL_DSN", "MYSQL_GAME_DSN"), "game-service mysql dsn") + ) + flag.Parse() + + statTZValue, loc, err := normalizeStatTZ(*statTZ) + if err != nil { + log.Fatal(err) + } + window, err := resolveWindow(*startDay, *endDay, loc) + if err != nil { + log.Fatal(err) + } + ctx := context.Background() + dbs := map[string]*sql.DB{} + for name, dsn := range map[string]string{ + "statistics": *statisticsDSN, + "user": *userDSN, + "wallet": *walletDSN, + "room": *roomDSN, + "game": *gameDSN, + } { + if strings.TrimSpace(dsn) == "" { + log.Fatalf("%s dsn is required", name) + } + db, err := openDB(ctx, dsn) + if err != nil { + log.Fatalf("open %s db: %v", name, err) + } + defer db.Close() + dbs[name] = db + } + + sourceChecks := auditSources(ctx, dbs, *appCode, window) + statsChecks := auditStatistics(ctx, dbs["statistics"], *appCode, statTZValue, window) + failed := printChecks("owner outbox", sourceChecks) + failed += printChecks("statistics aggregate", statsChecks) + if failed > 0 { + log.Fatalf("databi real flow audit failed: %d required checks have no data", failed) + } + fmt.Printf("databi real flow audit passed app=%s stat_tz=%s days=%s..%s\n", *appCode, statTZValue, window.startDay, window.endDay) +} + +func auditSources(ctx context.Context, dbs map[string]*sql.DB, app string, window auditWindow) []auditCheck { + return []auditCheck{ + countCheck(ctx, dbs["user"], "new_users", "新增用户", "user_outbox.UserRegistered", true, ` + SELECT COUNT(*) FROM user_outbox + WHERE app_code = ? AND event_type = 'UserRegistered' AND created_at_ms >= ? AND created_at_ms < ?`, app, window.startMS, window.endMS), + countCheck(ctx, dbs["room"], "active_users", "活跃用户", "room_outbox.RoomUserJoined", true, ` + SELECT COUNT(*) FROM room_outbox + WHERE app_code = ? AND event_type = 'RoomUserJoined' AND created_at_ms >= ? AND created_at_ms < ?`, app, window.startMS, window.endMS), + countCheck(ctx, dbs["user"], "mic_online_ms", "麦上时长", "user_outbox.UserMicDailyStatsUpdated", true, ` + SELECT COUNT(*) FROM user_outbox + WHERE app_code = ? AND event_type = 'UserMicDailyStatsUpdated' AND created_at_ms >= ? AND created_at_ms < ?`, app, window.startMS, window.endMS), + countCheck(ctx, dbs["wallet"], "google_recharge_usd_minor", "谷歌充值", "wallet_outbox.WalletRechargeRecorded(recharge_type=google/google_play，可本地模拟)", true, ` + SELECT COUNT(*) FROM wallet_outbox + WHERE app_code = ? AND event_type = 'WalletRechargeRecorded' AND created_at_ms >= ? AND created_at_ms < ? + AND LOWER(COALESCE(JSON_UNQUOTE(JSON_EXTRACT(payload, '$.recharge_type')), JSON_UNQUOTE(JSON_EXTRACT(payload, '$.channel')), '')) IN ('google','google_play')`, app, window.startMS, window.endMS), + countCheck(ctx, dbs["wallet"], "mifapay_recharge_usd_minor", "三方充值", "wallet_outbox.WalletRechargeRecorded(recharge_type=mifapay/v5pay，可本地模拟)", true, ` + SELECT COUNT(*) FROM wallet_outbox + WHERE app_code = ? AND event_type = 'WalletRechargeRecorded' AND created_at_ms >= ? AND created_at_ms < ? + AND LOWER(COALESCE(JSON_UNQUOTE(JSON_EXTRACT(payload, '$.recharge_type')), JSON_UNQUOTE(JSON_EXTRACT(payload, '$.channel')), '')) IN ('mifapay','v5pay')`, app, window.startMS, window.endMS), + countCheck(ctx, dbs["wallet"], "coin_seller_stock_coin", "币商充值金币", "wallet_outbox.WalletCoinSellerStockPurchased", true, ` + SELECT COUNT(*) FROM wallet_outbox + WHERE app_code = ? AND event_type = 'WalletCoinSellerStockPurchased' AND created_at_ms >= ? AND created_at_ms < ?`, app, window.startMS, window.endMS), + countCheck(ctx, dbs["wallet"], "coin_seller_transfer_coin", "币商出货金币", "wallet_outbox.WalletRechargeRecorded(recharge_type=coin_seller_transfer)", true, ` + SELECT COUNT(*) FROM wallet_outbox + WHERE app_code = ? AND event_type = 'WalletRechargeRecorded' AND created_at_ms >= ? AND created_at_ms < ? + AND LOWER(COALESCE(JSON_UNQUOTE(JSON_EXTRACT(payload, '$.recharge_type')), '')) IN ('coin_seller_transfer','coin_seller')`, app, window.startMS, window.endMS), + countCheck(ctx, dbs["wallet"], "coin_total", "金币数量总和", "wallet_outbox.WalletBalanceChanged(asset=COIN)", true, ` + SELECT COUNT(*) FROM wallet_outbox + WHERE app_code = ? AND event_type = 'WalletBalanceChanged' AND asset_type = 'COIN' AND created_at_ms >= ? AND created_at_ms < ?`, app, window.startMS, window.endMS), + countCheck(ctx, dbs["wallet"], "consumed_coin", "消耗金币", "wallet_outbox.WalletBalanceChanged(COIN消费biz_type)", true, ` + SELECT COUNT(*) FROM wallet_outbox + WHERE app_code = ? AND event_type = 'WalletBalanceChanged' AND asset_type = 'COIN' AND available_delta + frozen_delta < 0 + AND created_at_ms >= ? AND created_at_ms < ? + AND LOWER(COALESCE(JSON_UNQUOTE(JSON_EXTRACT(payload, '$.biz_type')), '')) IN ('gift_debit','direct_gift_debit','vip_purchase','resource_shop_purchase','red_packet_create','game_debit','wheel_draw_debit','cp_breakup_fee')`, app, window.startMS, window.endMS), + countCheck(ctx, dbs["wallet"], "platform_grant_coin", "平台发放金币", "wallet_outbox.Wallet*RewardCredited", true, ` + SELECT COUNT(*) FROM wallet_outbox + WHERE app_code = ? AND event_type IN ('WalletTaskRewardCredited','WalletLuckyGiftRewardCredited','WalletWheelRewardCredited','WalletRoomTurnoverRewardCredited','WalletInviteActivityRewardCredited','WalletAgencyOpeningRewardCredited') + AND created_at_ms >= ? AND created_at_ms < ?`, app, window.startMS, window.endMS), + countCheck(ctx, dbs["wallet"], "manual_grant_coin", "人工发放金币", "wallet_outbox.WalletBalanceChanged(biz_type=manual_credit)", true, ` + SELECT COUNT(*) FROM wallet_outbox + WHERE app_code = ? AND event_type = 'WalletBalanceChanged' AND asset_type = 'COIN' AND available_delta + frozen_delta > 0 + AND created_at_ms >= ? AND created_at_ms < ? + AND LOWER(COALESCE(JSON_UNQUOTE(JSON_EXTRACT(payload, '$.biz_type')), '')) = 'manual_credit'`, app, window.startMS, window.endMS), + countCheck(ctx, dbs["wallet"], "salary_usd_minor", "工资总和", "wallet_outbox.WalletBalanceChanged(biz_type=host_salary_settlement)", true, ` + SELECT COUNT(*) FROM wallet_outbox + WHERE app_code = ? AND event_type = 'WalletBalanceChanged' AND asset_type IN ('HOST_SALARY_USD','AGENCY_SALARY_USD') + AND available_delta + frozen_delta > 0 AND created_at_ms >= ? AND created_at_ms < ? + AND LOWER(COALESCE(JSON_UNQUOTE(JSON_EXTRACT(payload, '$.biz_type')), '')) = 'host_salary_settlement'`, app, window.startMS, window.endMS), + countCheck(ctx, dbs["wallet"], "salary_transfer_coin", "工资兑换金币", "wallet_outbox.WalletBalanceChanged(biz_type=salary_transfer_to_coin_seller)", true, ` + SELECT COUNT(*) FROM wallet_outbox + WHERE app_code = ? AND event_type = 'WalletBalanceChanged' AND asset_type = 'COIN_SELLER_COIN' + AND available_delta + frozen_delta > 0 AND created_at_ms >= ? AND created_at_ms < ? + AND LOWER(COALESCE(JSON_UNQUOTE(JSON_EXTRACT(payload, '$.biz_type')), '')) = 'salary_transfer_to_coin_seller'`, app, window.startMS, window.endMS), + countCheck(ctx, dbs["room"], "gift_coin_spent", "礼物/幸运礼物流水", "room_outbox.RoomGiftSent", true, ` + SELECT COUNT(*) FROM room_outbox + WHERE app_code = ? AND event_type = 'RoomGiftSent' AND created_at_ms >= ? AND created_at_ms < ?`, app, window.startMS, window.endMS), + countCheck(ctx, dbs["wallet"], "lucky_gift_payout", "幸运礼物返奖", "wallet_outbox.WalletLuckyGiftRewardCredited", true, ` + SELECT COUNT(*) FROM wallet_outbox + WHERE app_code = ? AND event_type = 'WalletLuckyGiftRewardCredited' AND created_at_ms >= ? AND created_at_ms < ?`, app, window.startMS, window.endMS), + countCheck(ctx, dbs["game"], "game_turnover", "游戏流水/利润", "game_outbox.GameOrderSettled", true, ` + SELECT COUNT(*) FROM game_outbox + WHERE app_code = ? AND event_type = 'GameOrderSettled' AND created_at_ms >= ? AND created_at_ms < ?`, app, window.startMS, window.endMS), + } +} + +func auditStatistics(ctx context.Context, db *sql.DB, app string, statTZ string, window auditWindow) []auditCheck { + checks := []auditCheck{} + for _, spec := range []struct { + key string + label string + expr string + }{ + {"new_users", "新增用户", "SUM(new_users)"}, + {"active_users", "活跃用户", "SUM(active_users)"}, + {"paid_users", "付费用户", "SUM(paid_users)"}, + {"recharge_usd_minor", "总充值", "SUM(recharge_usd_minor)"}, + {"coin_seller_recharge_usd_minor", "币商充值", "SUM(coin_seller_recharge_usd_minor)"}, + {"coin_seller_stock_coin", "币商充值金币", "SUM(coin_seller_stock_coin)"}, + {"coin_seller_transfer_coin", "币商出货金币", "SUM(coin_seller_transfer_coin)"}, + {"google_recharge_usd_minor", "谷歌充值", "SUM(google_recharge_usd_minor)"}, + {"mifapay_recharge_usd_minor", "三方充值", "SUM(mifapay_recharge_usd_minor)"}, + {"coin_total", "金币数量总和", "SUM(coin_total)"}, + {"consumed_coin", "消耗金币", "SUM(consumed_coin)"}, + {"platform_grant_coin", "平台发放金币", "SUM(platform_grant_coin)"}, + {"manual_grant_coin", "人工发放金币", "SUM(manual_grant_coin)"}, + {"salary_usd_minor", "工资总和", "SUM(salary_usd_minor)"}, + {"salary_transfer_coin", "工资兑换金币", "SUM(salary_transfer_coin)"}, + {"mic_online_ms", "麦上总时长", "SUM(mic_online_ms)"}, + {"mic_online_users", "平均麦上时间用户数", "SUM(mic_online_users)"}, + {"gift_coin_spent", "礼物流水", "SUM(gift_coin_spent)"}, + {"lucky_gift_turnover", "幸运礼物流水", "SUM(lucky_gift_turnover)"}, + {"lucky_gift_payout", "幸运礼物返奖", "SUM(lucky_gift_payout)"}, + {"game_turnover", "游戏流水", "SUM(game_turnover)"}, + {"game_payout", "游戏返奖", "SUM(game_payout)"}, + } { + checks = append(checks, countCheck(ctx, db, spec.key, spec.label, "stat_app_day_country."+spec.key, true, + "SELECT COALESCE("+spec.expr+",0) FROM stat_app_day_country WHERE app_code = ? AND stat_tz = ? AND stat_day BETWEEN ? AND ?", + app, statTZ, window.startDay, window.endDay)) + } + checks = append(checks, + countCheck(ctx, db, "super_lucky_gift_turnover", "超级幸运礼物流水", "stat_lucky_gift_pool_day_country(pool_id LIKE %super_lucky%)", true, ` + SELECT COALESCE(SUM(turnover_coin),0) FROM stat_lucky_gift_pool_day_country + WHERE app_code = ? AND stat_tz = ? AND stat_day BETWEEN ? AND ? AND LOWER(pool_id) LIKE '%super_lucky%'`, + app, statTZ, window.startDay, window.endDay), + countCheck(ctx, db, "super_lucky_gift_payout", "超级幸运礼物返奖", "stat_lucky_gift_pool_day_country(pool_id LIKE %super_lucky%)", true, ` + SELECT COALESCE(SUM(payout_coin),0) FROM stat_lucky_gift_pool_day_country + WHERE app_code = ? AND stat_tz = ? AND stat_day BETWEEN ? AND ? AND LOWER(pool_id) LIKE '%super_lucky%'`, + app, statTZ, window.startDay, window.endDay), + ) + return checks +} + +func countCheck(ctx context.Context, db *sql.DB, key string, label string, source string, required bool, query string, args ...any) auditCheck { + check := auditCheck{key: key, label: label, source: source, required: required} + if db == nil { + check.err = fmt.Errorf("db is nil") + return check + } + err := db.QueryRowContext(ctx, query, args...).Scan(&check.count) + if err != nil { + check.err = err + } + return check +} + +func printChecks(title string, checks []auditCheck) int { + fmt.Printf("\n%s\n", title) + failed := 0 + for _, check := range checks { + status := "OK" + if check.err != nil || (check.required && check.count <= 0) { + status = "MISSING" + failed++ + } + if check.err != nil { + fmt.Printf(" %-8s %-34s count=%d source=%s err=%v\n", status, check.label, check.count, check.source, check.err) + continue + } + fmt.Printf(" %-8s %-34s count=%d source=%s\n", status, check.label, check.count, check.source) + } + return failed +} + +func normalizeStatTZ(value string) (string, *time.Location, error) { + switch strings.TrimSpace(value) { + case mysqlstorage.StatTZUTC: + return mysqlstorage.StatTZUTC, time.UTC, nil + case mysqlstorage.StatTZAsiaShanghai: + return mysqlstorage.StatTZAsiaShanghai, time.FixedZone(mysqlstorage.StatTZAsiaShanghai, 8*60*60), nil + default: + return "", nil, fmt.Errorf("unsupported stat_tz %q; only UTC and Asia/Shanghai are supported", value) + } +} + +func resolveWindow(startDay string, endDay string, loc *time.Location) (auditWindow, error) { + start, err := time.ParseInLocation("2006-01-02", strings.TrimSpace(startDay), loc) + if err != nil { + return auditWindow{}, fmt.Errorf("parse start-day: %w", err) + } + end, err := time.ParseInLocation("2006-01-02", strings.TrimSpace(endDay), loc) + if err != nil { + return auditWindow{}, fmt.Errorf("parse end-day: %w", err) + } + if end.Before(start) { + return auditWindow{}, fmt.Errorf("end-day %s is before start-day %s", endDay, startDay) + } + return auditWindow{ + startDay: start.Format("2006-01-02"), + endDay: end.Format("2006-01-02"), + startMS: start.UnixMilli(), + endMS: end.AddDate(0, 0, 1).UnixMilli(), + }, nil +} + +func openDB(ctx context.Context, dsn string) (*sql.DB, error) { + db, err := sql.Open("mysql", strings.TrimSpace(dsn)) + if err != nil { + return nil, err + } + if err := db.PingContext(ctx); err != nil { + _ = db.Close() + return nil, err + } + return db, nil +} + +func todayIn(statTZ string) string { + _, loc, err := normalizeStatTZ(statTZ) + if err != nil { + loc = time.FixedZone(mysqlstorage.StatTZAsiaShanghai, 8*60*60) + } + return time.Now().In(loc).Format("2006-01-02") +} + +func envDefault(key string, fallback string) string { + if value := strings.TrimSpace(os.Getenv(key)); value != "" { + return value + } + return fallback +} + +func firstNonEmptyEnv(keys ...string) string { + for _, key := range keys { + if value := strings.TrimSpace(os.Getenv(key)); value != "" { + return value + } + } + return "" +} diff --git a/services/statistics-service/cmd/replay-stat-tz/main.go b/services/statistics-service/cmd/replay-stat-tz/main.go index 7b972116..66aee9c1 100644 --- a/services/statistics-service/cmd/replay-stat-tz/main.go +++ b/services/statistics-service/cmd/replay-stat-tz/main.go @@ -15,6 +15,7 @@ import ( "google.golang.org/protobuf/proto" roomeventsv1 "hyapp.local/api/proto/events/room/v1" + "hyapp/pkg/appcode" "hyapp/pkg/gamemq" "hyapp/pkg/roommq" "hyapp/pkg/usermq" @@ -107,6 +108,10 @@ func main() { if err := resetReplayTarget(ctx, statsDB, *appCode, statTZValue, window); err != nil { log.Fatalf("reset target: %v", err) } + dimensionRows, err := primeUserDimensions(ctx, statsDB, dbs["user"], *appCode) + if err != nil { + log.Fatalf("prime user dimensions: %v", err) + } scopeCtx := mysqlstorage.WithStatTZScope(ctx, statTZValue) if err := replayUser(ctx, scopeCtx, repo, dbs["user"], *appCode, statTZValue, window); err != nil { log.Fatalf("replay user outbox: %v", err) @@ -120,8 +125,8 @@ func main() { if err := replayGame(ctx, scopeCtx, repo, dbs["game"], *appCode, statTZValue, window); err != nil { log.Fatalf("replay game outbox: %v", err) } - fmt.Printf("applied app=%s stat_tz=%s days=%s..%s source_rows user=%d wallet=%d room=%d game=%d\n", - *appCode, statTZValue, window.startDay, window.endDay, summary.userRows, summary.walletRows, summary.roomRows, summary.gameRows) + fmt.Printf("applied app=%s stat_tz=%s days=%s..%s source_rows user=%d wallet=%d room=%d game=%d user_dimension_rows=%d\n", + *appCode, statTZValue, window.startDay, window.endDay, summary.userRows, summary.walletRows, summary.roomRows, summary.gameRows, dimensionRows) } func normalizeReplayStatTZ(value string) (string, *time.Location, error) { @@ -179,13 +184,13 @@ func countSources(ctx context.Context, dbs map[string]*sql.DB, app string, windo var out replaySummary var err error if db := dbs["user"]; db != nil { - out.userRows, err = countOutboxRows(ctx, db, "user_outbox", app, []string{"UserRegistered"}, window) + out.userRows, err = countOutboxRows(ctx, db, "user_outbox", app, userReplayEventTypes(), window) if err != nil { return replaySummary{}, err } } if db := dbs["wallet"]; db != nil { - out.walletRows, err = countOutboxRows(ctx, db, "wallet_outbox", app, []string{"WalletRechargeRecorded", "WalletCoinSellerStockPurchased", "WalletLuckyGiftRewardCredited"}, window) + out.walletRows, err = countOutboxRows(ctx, db, "wallet_outbox", app, walletReplayEventTypes(), window) if err != nil { return replaySummary{}, err } @@ -226,6 +231,7 @@ func resetReplayTarget(ctx context.Context, db *sql.DB, app string, statTZ strin {"stat_app_day_country", "stat_day"}, {"stat_user_day_activity", "stat_day"}, {"stat_user_registration", "registered_day"}, + {"stat_user_day_mic", "stat_day"}, {"stat_recharge_day_payers", "stat_day"}, {"stat_lucky_gift_day_payers", "stat_day"}, {"stat_lucky_gift_pool_day_country", "stat_day"}, @@ -254,11 +260,10 @@ func resetReplayTarget(ctx context.Context, db *sql.DB, app string, statTZ strin } func replayUser(ctx context.Context, scopeCtx context.Context, repo *mysqlstorage.Repository, db *sql.DB, app string, statTZ string, window replayWindow) error { - rows, err := db.QueryContext(ctx, ` - SELECT app_code, event_id, event_type, aggregate_type, aggregate_id, CAST(payload_json AS CHAR), created_at_ms - FROM user_outbox - WHERE app_code = ? AND event_type = 'UserRegistered' AND created_at_ms >= ? AND created_at_ms < ? - ORDER BY created_at_ms ASC, event_id ASC`, app, window.startMS, window.endMS) + query, args := outboxWindowQuery(` + app_code, event_id, event_type, aggregate_type, aggregate_id, CAST(payload_json AS CHAR), created_at_ms`, "user_outbox", app, + userReplayEventTypes(), window, "ORDER BY created_at_ms ASC, event_id ASC") + rows, err := db.QueryContext(ctx, query, args...) if err != nil { return err } @@ -280,11 +285,66 @@ func replayUser(ctx context.Context, scopeCtx context.Context, repo *mysqlstorag return rows.Err() } +func primeUserDimensions(ctx context.Context, statsDB *sql.DB, userDB *sql.DB, app string) (int64, error) { + rows, err := userDB.QueryContext(ctx, ` + SELECT u.user_id, COALESCE(c.country_id, 0), COALESCE(u.region_id, 0) + FROM users u + LEFT JOIN countries c ON c.app_code = u.app_code AND c.country_code = u.country + WHERE u.app_code = ? + AND u.user_id > 0 + AND (COALESCE(c.country_id, 0) > 0 OR COALESCE(u.region_id, 0) > 0)`, + appcode.Normalize(app)) + if err != nil { + return 0, err + } + defer rows.Close() + + tx, err := statsDB.BeginTx(ctx, nil) + if err != nil { + return 0, err + } + defer tx.Rollback() + + stmt, err := tx.PrepareContext(ctx, ` + INSERT INTO stat_user_dimension (app_code, user_id, country_id, region_id, updated_at_ms) + VALUES (?, ?, ?, ?, ?) + ON DUPLICATE KEY UPDATE + country_id = CASE WHEN VALUES(country_id) > 0 THEN VALUES(country_id) ELSE country_id END, + region_id = CASE WHEN VALUES(region_id) > 0 THEN VALUES(region_id) ELSE region_id END, + updated_at_ms = VALUES(updated_at_ms)`) + if err != nil { + return 0, err + } + defer stmt.Close() + + nowMS := time.Now().UnixMilli() + var count int64 + for rows.Next() { + var userID int64 + var countryID int64 + var regionID int64 + if err := rows.Scan(&userID, &countryID, ®ionID); err != nil { + return 0, err + } + if _, err := stmt.ExecContext(ctx, appcode.Normalize(app), userID, countryID, regionID, nowMS); err != nil { + return 0, err + } + count++ + } + if err := rows.Err(); err != nil { + return 0, err + } + if err := tx.Commit(); err != nil { + return 0, err + } + return count, nil +} + func replayWallet(ctx context.Context, scopeCtx context.Context, repo *mysqlstorage.Repository, db *sql.DB, app string, statTZ string, window replayWindow) error { query, args := outboxWindowQuery(` app_code, event_id, event_type, transaction_id, command_id, user_id, asset_type, available_delta, frozen_delta, CAST(payload AS CHAR), created_at_ms`, "wallet_outbox", app, - []string{"WalletRechargeRecorded", "WalletCoinSellerStockPurchased", "WalletLuckyGiftRewardCredited"}, window, "ORDER BY created_at_ms ASC, event_id ASC") + walletReplayEventTypes(), window, "ORDER BY created_at_ms ASC, event_id ASC") rows, err := db.QueryContext(ctx, query, args...) if err != nil { return err @@ -371,6 +431,28 @@ func outboxWindowQuery(selectExpr string, table string, app string, eventTypes [ return fmt.Sprintf("SELECT %s FROM %s WHERE app_code = ? AND event_type IN (%s) AND created_at_ms >= ? AND created_at_ms < ? %s", selectExpr, table, placeholders, suffix), args } +func userReplayEventTypes() []string { + return []string{ + "UserRegistered", + "UserRegionChanged", + "UserMicDailyStatsUpdated", + } +} + +func walletReplayEventTypes() []string { + return []string{ + "WalletRechargeRecorded", + "WalletCoinSellerStockPurchased", + "WalletBalanceChanged", + "WalletTaskRewardCredited", + "WalletLuckyGiftRewardCredited", + "WalletWheelRewardCredited", + "WalletRoomTurnoverRewardCredited", + "WalletInviteActivityRewardCredited", + "WalletAgencyOpeningRewardCredited", + } +} + func gameReplayEventTypes() []string { return []string{ gamemq.EventTypeGameOrderSettled, diff --git a/services/statistics-service/deploy/mysql/initdb/001_statistics_service.sql b/services/statistics-service/deploy/mysql/initdb/001_statistics_service.sql index d8000b4f..9c5cce3f 100644 --- a/services/statistics-service/deploy/mysql/initdb/001_statistics_service.sql +++ b/services/statistics-service/deploy/mysql/initdb/001_statistics_service.sql @@ -29,9 +29,18 @@ CREATE TABLE IF NOT EXISTS stat_app_day_country ( recharge_usd_minor BIGINT NOT NULL DEFAULT 0 COMMENT '总充值美元最小单位', new_user_recharge_usd_minor BIGINT NOT NULL DEFAULT 0 COMMENT '新增充值美元最小单位', coin_seller_recharge_usd_minor BIGINT NOT NULL DEFAULT 0 COMMENT '币商充值美元最小单位', + coin_seller_stock_coin BIGINT NOT NULL DEFAULT 0 COMMENT '币商库存充值金币数量', mifapay_recharge_usd_minor BIGINT NOT NULL DEFAULT 0 COMMENT 'MifaPay 充值美元最小单位', google_recharge_usd_minor BIGINT NOT NULL DEFAULT 0 COMMENT 'Google 充值美元最小单位', coin_seller_transfer_coin BIGINT NOT NULL DEFAULT 0 COMMENT '币商向用户转普通金币数量', + coin_total BIGINT NOT NULL DEFAULT 0 COMMENT '普通用户 COIN 钱包日终余额总和', + consumed_coin BIGINT NOT NULL DEFAULT 0 COMMENT '消费类普通金币扣减', + platform_grant_coin BIGINT NOT NULL DEFAULT 0 COMMENT '系统奖励金币发放', + manual_grant_coin BIGINT NOT NULL DEFAULT 0 COMMENT '后台人工发放金币', + salary_usd_minor BIGINT NOT NULL DEFAULT 0 COMMENT '主播和代理工资 USD 最小单位', + salary_transfer_coin BIGINT NOT NULL DEFAULT 0 COMMENT '工资转币商入账金币数量', + mic_online_ms BIGINT NOT NULL DEFAULT 0 COMMENT '麦上总时长毫秒', + mic_online_users BIGINT NOT NULL DEFAULT 0 COMMENT '麦上用户数', gift_coin_spent BIGINT NOT NULL DEFAULT 0 COMMENT '礼物消耗金币', lucky_gift_turnover BIGINT NOT NULL DEFAULT 0 COMMENT '幸运礼物流水', lucky_gift_payout BIGINT NOT NULL DEFAULT 0 COMMENT '幸运礼物返奖', @@ -71,6 +80,31 @@ CREATE TABLE IF NOT EXISTS stat_user_registration ( KEY idx_stat_user_registration_region (app_code, stat_tz, registered_day, region_id) ) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_unicode_ci COMMENT='用户注册日留存 cohort 表'; +CREATE TABLE IF NOT EXISTS stat_user_dimension ( + app_code VARCHAR(32) NOT NULL, + user_id BIGINT NOT NULL, + country_id BIGINT NOT NULL DEFAULT 0, + region_id BIGINT NOT NULL DEFAULT 0, + updated_at_ms BIGINT NOT NULL, + PRIMARY KEY (app_code, user_id), + KEY idx_stat_user_dimension_country (app_code, country_id), + KEY idx_stat_user_dimension_region (app_code, region_id) +) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_unicode_ci COMMENT='统计服务用户国家区域维度投影'; + +CREATE TABLE IF NOT EXISTS stat_user_day_mic ( + app_code VARCHAR(32) NOT NULL, + stat_tz VARCHAR(64) NOT NULL DEFAULT 'UTC' COMMENT '统计时区，UTC 或 Asia/Shanghai', + stat_day DATE NOT NULL, + country_id BIGINT NOT NULL DEFAULT 0, + region_id BIGINT NOT NULL DEFAULT 0, + user_id BIGINT NOT NULL, + mic_online_ms BIGINT NOT NULL DEFAULT 0, + updated_at_ms BIGINT NOT NULL, + PRIMARY KEY (app_code, stat_tz, stat_day, user_id), + KEY idx_stat_user_day_mic_country (app_code, stat_tz, stat_day, country_id), + KEY idx_stat_user_day_mic_region (app_code, stat_tz, stat_day, region_id) +) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_unicode_ci COMMENT='用户每日麦上时长去重表'; + CREATE TABLE IF NOT EXISTS stat_recharge_day_payers ( app_code VARCHAR(32) NOT NULL, stat_tz VARCHAR(64) NOT NULL DEFAULT 'UTC' COMMENT '统计时区，UTC 或 Asia/Shanghai', diff --git a/services/statistics-service/internal/app/app.go b/services/statistics-service/internal/app/app.go index 7b136665..fbb642d3 100644 --- a/services/statistics-service/internal/app/app.go +++ b/services/statistics-service/internal/app/app.go @@ -168,34 +168,56 @@ func newConsumers(cfg config.Config, repo *mysqlstorage.Repository) ([]*rocketmq // ConsumeUserOutboxBody 是 user outbox 的唯一统计投影入口；实时 MQ 和离线回灌共用它，避免同一 payload 出现两套解析口径。 func ConsumeUserOutboxBody(ctx context.Context, repo *mysqlstorage.Repository, body []byte) error { - event, ok, err := userRegisteredEvent(body) + if event, ok, err := userRegisteredEvent(body); err != nil || ok { + if err != nil || !ok { + return err + } + return repo.ConsumeUserRegistered(appcode.WithContext(ctx, event.AppCode), event) + } + if event, ok, err := userRegionChangedEvent(body); err != nil || ok { + if err != nil || !ok { + return err + } + return repo.ConsumeUserRegionChanged(appcode.WithContext(ctx, event.AppCode), event) + } + event, ok, err := userMicDailyStatsUpdatedEvent(body) if err != nil || !ok { return err } - return repo.ConsumeUserRegistered(appcode.WithContext(ctx, event.AppCode), event) + return repo.ConsumeUserMicDailyStatsUpdated(appcode.WithContext(ctx, event.AppCode), event) } -// ConsumeWalletOutboxBody 按钱包事实类型分派充值、币商补货和幸运礼物返奖；未知事件只提交消费位点，不写统计读模型。 +// ConsumeWalletOutboxBody 按钱包事实类型分派充值、币商补货、奖励和余额变动；未知事件只提交消费位点，不写统计读模型。 func ConsumeWalletOutboxBody(ctx context.Context, repo *mysqlstorage.Repository, body []byte) error { - event, ok, err := rechargeEvent(body) - if err != nil { - return err - } - if ok { + if event, ok, err := rechargeEvent(body); err != nil || ok { + if err != nil || !ok { + return err + } return repo.ConsumeRecharge(appcode.WithContext(ctx, event.AppCode), event) } - stock, ok, err := coinSellerStockEvent(body) - if err != nil { - return err - } - if ok { + if stock, ok, err := coinSellerStockEvent(body); err != nil || ok { + if err != nil || !ok { + return err + } return repo.ConsumeCoinSellerStock(appcode.WithContext(ctx, stock.AppCode), stock) } - reward, ok, err := luckyGiftRewardEvent(body) + if reward, ok, err := luckyGiftRewardEvent(body); err != nil || ok { + if err != nil || !ok { + return err + } + return repo.ConsumeLuckyGiftReward(appcode.WithContext(ctx, reward.AppCode), reward) + } + if grant, ok, err := platformGrantCoinEvent(body); err != nil || ok { + if err != nil || !ok { + return err + } + return repo.ConsumePlatformGrantCoin(appcode.WithContext(ctx, grant.AppCode), grant) + } + event, ok, err := walletBalanceChangedEvent(body) if err != nil || !ok { return err } - return repo.ConsumeLuckyGiftReward(appcode.WithContext(ctx, reward.AppCode), reward) + return repo.ConsumeWalletBalanceChanged(appcode.WithContext(ctx, event.AppCode), event) } // ConsumeRoomOutboxBody 只投影房间统计需要的礼物和真人进房事件；礼物事实额外产生 sender 当日活跃，保持实时和回灌一致。 @@ -262,6 +284,44 @@ func userRegisteredEvent(body []byte) (mysqlstorage.UserRegisteredEvent, bool, e }, true, nil } +func userRegionChangedEvent(body []byte) (mysqlstorage.UserRegionChangedEvent, bool, error) { + message, err := usermq.DecodeUserOutboxMessage(body) + if err != nil { + return mysqlstorage.UserRegionChangedEvent{}, false, err + } + if message.EventType != "UserRegionChanged" { + return mysqlstorage.UserRegionChangedEvent{}, false, nil + } + payload := mysqlstorage.DecodeJSON(message.PayloadJSON) + return mysqlstorage.UserRegionChangedEvent{ + AppCode: message.AppCode, + EventID: message.EventID, + UserID: firstNonZeroInt64(mysqlstorage.Int64(payload, "user_id"), message.AggregateID), + CountryID: firstNonZeroInt64(mysqlstorage.Int64(payload, "country_id"), mysqlstorage.Int64(payload, "new_country_id")), + RegionID: firstNonZeroInt64(mysqlstorage.Int64(payload, "region_id"), mysqlstorage.Int64(payload, "new_region_id")), + OccurredAtMS: firstNonZeroInt64(mysqlstorage.Int64(payload, "changed_at_ms"), message.OccurredAtMS), + }, true, nil +} + +func userMicDailyStatsUpdatedEvent(body []byte) (mysqlstorage.UserMicDailyStatsUpdatedEvent, bool, error) { + message, err := usermq.DecodeUserOutboxMessage(body) + if err != nil { + return mysqlstorage.UserMicDailyStatsUpdatedEvent{}, false, err + } + if message.EventType != "UserMicDailyStatsUpdated" { + return mysqlstorage.UserMicDailyStatsUpdatedEvent{}, false, nil + } + payload := mysqlstorage.DecodeJSON(message.PayloadJSON) + return mysqlstorage.UserMicDailyStatsUpdatedEvent{ + AppCode: message.AppCode, + EventID: message.EventID, + UserID: firstNonZeroInt64(mysqlstorage.Int64(payload, "user_id"), message.AggregateID), + StatDate: mysqlstorage.String(payload, "stat_date"), + MicOnlineDeltaMS: mysqlstorage.Int64(payload, "mic_online_delta_ms"), + OccurredAtMS: firstNonZeroInt64(mysqlstorage.Int64(payload, "updated_at_ms"), message.OccurredAtMS), + }, true, nil +} + func rechargeEvent(body []byte) (mysqlstorage.RechargeEvent, bool, error) { message, err := walletmq.DecodeWalletOutboxMessage(body) if err != nil { @@ -384,6 +444,73 @@ func luckyGiftRewardEvent(body []byte) (mysqlstorage.LuckyGiftRewardEvent, bool, }, true, nil } +func platformGrantCoinEvent(body []byte) (mysqlstorage.PlatformGrantCoinEvent, bool, error) { + message, err := walletmq.DecodeWalletOutboxMessage(body) + if err != nil { + return mysqlstorage.PlatformGrantCoinEvent{}, false, err + } + if !isPlatformGrantCoinEventType(message.EventType) || message.EventType == "WalletLuckyGiftRewardCredited" { + return mysqlstorage.PlatformGrantCoinEvent{}, false, nil + } + payload := mysqlstorage.DecodeJSON(message.PayloadJSON) + return mysqlstorage.PlatformGrantCoinEvent{ + AppCode: message.AppCode, + EventID: message.EventID, + EventType: message.EventType, + UserID: firstNonZeroInt64(mysqlstorage.Int64(payload, "user_id"), message.UserID), + CountryID: firstNonZeroInt64(mysqlstorage.Int64(payload, "country_id"), mysqlstorage.Int64(payload, "target_country_id")), + RegionID: firstNonZeroInt64(mysqlstorage.Int64(payload, "visible_region_id"), mysqlstorage.Int64(payload, "region_id"), mysqlstorage.Int64(payload, "target_region_id")), + Amount: firstNonZeroInt64(mysqlstorage.Int64(payload, "amount"), message.AvailableDelta+message.FrozenDelta), + OccurredAtMS: message.OccurredAtMS, + }, true, nil +} + +func walletBalanceChangedEvent(body []byte) (mysqlstorage.WalletBalanceChangedEvent, bool, error) { + message, err := walletmq.DecodeWalletOutboxMessage(body) + if err != nil { + return mysqlstorage.WalletBalanceChangedEvent{}, false, err + } + if message.EventType != "WalletBalanceChanged" { + return mysqlstorage.WalletBalanceChangedEvent{}, false, nil + } + payload := mysqlstorage.DecodeJSON(message.PayloadJSON) + metadata := objectValue(payload, "metadata") + assetType := firstNonEmpty(mysqlstorage.String(payload, "asset_type"), message.AssetType) + bizType := firstNonEmpty(mysqlstorage.String(payload, "biz_type"), mysqlstorage.String(payload, "business_type")) + dimensionUserID := int64(0) + if strings.EqualFold(strings.TrimSpace(bizType), "salary_transfer_to_coin_seller") && strings.EqualFold(strings.TrimSpace(assetType), "COIN_SELLER_COIN") { + dimensionUserID = firstNonZeroInt64(mysqlstorage.Int64(payload, "dimension_user_id"), mysqlstorage.Int64(metadata, "source_user_id"), mysqlstorage.Int64(metadata, "sourceUserID")) + } + return mysqlstorage.WalletBalanceChangedEvent{ + AppCode: message.AppCode, + EventID: message.EventID, + EventType: message.EventType, + UserID: firstNonZeroInt64(mysqlstorage.Int64(payload, "user_id"), message.UserID), + DimensionUserID: dimensionUserID, + AssetType: assetType, + BizType: bizType, + AvailableDelta: firstNonZeroInt64(mysqlstorage.Int64(payload, "available_delta"), message.AvailableDelta), + FrozenDelta: firstNonZeroInt64(mysqlstorage.Int64(payload, "frozen_delta"), message.FrozenDelta), + CountryID: firstNonZeroInt64(mysqlstorage.Int64(payload, "country_id"), mysqlstorage.Int64(payload, "target_country_id")), + RegionID: firstNonZeroInt64(mysqlstorage.Int64(payload, "visible_region_id"), mysqlstorage.Int64(payload, "region_id"), mysqlstorage.Int64(payload, "target_region_id"), mysqlstorage.Int64(metadata, "region_id"), mysqlstorage.Int64(metadata, "regionId")), + OccurredAtMS: firstNonZeroInt64(mysqlstorage.Int64(payload, "created_at_ms"), message.OccurredAtMS), + }, true, nil +} + +func isPlatformGrantCoinEventType(value string) bool { + switch strings.TrimSpace(value) { + case "WalletTaskRewardCredited", + "WalletLuckyGiftRewardCredited", + "WalletWheelRewardCredited", + "WalletRoomTurnoverRewardCredited", + "WalletInviteActivityRewardCredited", + "WalletAgencyOpeningRewardCredited": + return true + default: + return false + } +} + func roomGiftEvent(body []byte) (mysqlstorage.RoomGiftEvent, bool, error) { envelope, _, err := roommq.DecodeRoomOutboxMessage(body) if err != nil { @@ -635,6 +762,13 @@ func firstNonZeroInt64(values ...int64) int64 { return 0 } +func objectValue(payload map[string]any, key string) map[string]any { + if value, ok := payload[key].(map[string]any); ok { + return value + } + return map[string]any{} +} + func boolValue(payload map[string]any, key string) bool { switch value := payload[key].(type) { case bool: diff --git a/services/statistics-service/internal/app/local_flow_test.go b/services/statistics-service/internal/app/local_flow_test.go index 8585a685..cf376691 100644 --- a/services/statistics-service/internal/app/local_flow_test.go +++ b/services/statistics-service/internal/app/local_flow_test.go @@ -17,7 +17,9 @@ import ( mysqlstorage "hyapp/services/statistics-service/internal/storage/mysql" ) -func TestLocalStatisticsDataScreenFlow(t *testing.T) { +// 这个用例只覆盖 statistics-service 对 owner outbox payload 的解析和聚合边界； +// 本地真实业务链路用 scripts/databi-real-flow-check.sh 从各 owner service 的 outbox 重放验证。 +func TestConsumeOutboxMessagesProjectsDataScreenMetrics(t *testing.T) { ctx := context.Background() _, file, _, _ := runtime.Caller(0) schema := mysqlschema.New(t, mysqlschema.Config{ @@ -231,7 +233,7 @@ func TestLocalStatisticsDataScreenFlow(t *testing.T) { if err != nil { t.Fatalf("query overview failed: %v", err) } - if overview.GoogleRechargeUSDMinor != 150 || overview.CoinSellerRechargeUSDMinor != 350 || overview.CoinSellerTransferCoin != 160_000 || overview.RechargeUSDMinor != 500 || overview.NewUserRechargeUSDMinor != 150 || overview.PaidUsers != 2 { + if overview.GoogleRechargeUSDMinor != 150 || overview.CoinSellerRechargeUSDMinor != 350 || overview.CoinSellerStockCoin != 350_000 || overview.CoinSellerTransferCoin != 160_000 || overview.RechargeUSDMinor != 500 || overview.NewUserRechargeUSDMinor != 150 || overview.PaidUsers != 2 { t.Fatalf("google recharge metrics mismatch: %+v", overview) } if overview.LuckyGiftTurnover != 1_250 || overview.LuckyGiftPayout != 300 || overview.LuckyGiftProfit != 950 || overview.LuckyGiftPayers != 1 { @@ -252,6 +254,103 @@ func TestLocalStatisticsDataScreenFlow(t *testing.T) { } } +func TestConsumeWalletOutboxBodyProjectsBalanceChangedReportMetrics(t *testing.T) { + ctx := context.Background() + _, file, _, _ := runtime.Caller(0) + schema := mysqlschema.New(t, mysqlschema.Config{ + InitDBPath: mysqlschema.InitDBPath(t, file, "../../deploy/mysql/initdb/001_statistics_service.sql"), + DatabasePrefix: "hy_stats_wallet_balance_flow", + }) + repo, err := mysqlstorage.Open(ctx, schema.DSN) + if err != nil { + t.Fatalf("open statistics repository failed: %v", err) + } + t.Cleanup(func() { _ = repo.Close() }) + + occurredAt := time.Date(2026, 6, 22, 8, 0, 0, 0, time.UTC).UnixMilli() + for _, message := range []walletmq.WalletOutboxMessage{ + { + AppCode: "lalu", + EventID: "WalletBalanceChanged:coin:topup", + EventType: "WalletBalanceChanged", + TransactionID: "wallet_tx_topup", + CommandID: "topup", + UserID: 88001, + AssetType: "COIN", + PayloadJSON: mustJSON(t, map[string]any{ + "user_id": int64(88001), + "asset_type": "COIN", + "biz_type": "google_play_recharge", + "available_delta": int64(100), + "country_id": int64(15), + "region_id": int64(10), + "created_at_ms": occurredAt, + }), + OccurredAtMS: occurredAt, + }, + { + AppCode: "lalu", + EventID: "WalletBalanceChanged:coin:gift", + EventType: "WalletBalanceChanged", + TransactionID: "wallet_tx_gift", + CommandID: "gift", + UserID: 88001, + AssetType: "COIN", + PayloadJSON: mustJSON(t, map[string]any{ + "user_id": int64(88001), + "asset_type": "COIN", + "biz_type": "gift_debit", + "available_delta": int64(-88), + "country_id": int64(15), + "region_id": int64(10), + "created_at_ms": occurredAt, + }), + OccurredAtMS: occurredAt, + }, + { + AppCode: "lalu", + EventID: "WalletBalanceChanged:salary:coin-seller", + EventType: "WalletBalanceChanged", + TransactionID: "wallet_tx_salary_seller", + CommandID: "salary-seller", + UserID: 99001, + AssetType: "COIN_SELLER_COIN", + PayloadJSON: mustJSON(t, map[string]any{ + "user_id": int64(99001), + "asset_type": "COIN_SELLER_COIN", + "biz_type": "salary_transfer_to_coin_seller", + "available_delta": int64(660), + "country_id": int64(15), + "region_id": int64(10), + "metadata": map[string]any{ + "source_user_id": int64(88001), + "region_id": int64(10), + }, + "created_at_ms": occurredAt, + }), + OccurredAtMS: occurredAt, + }, + } { + if err := ConsumeWalletOutboxBody(ctx, repo, walletMessage(t, message)); err != nil { + t.Fatalf("consume wallet balance message %s failed: %v", message.EventID, err) + } + } + + start := time.Date(2026, 6, 22, 0, 0, 0, 0, time.UTC).UnixMilli() + overview, err := repo.QueryOverview(ctx, mysqlstorage.OverviewQuery{ + AppCode: "lalu", + StartMS: start, + EndMS: start + int64(24*time.Hour/time.Millisecond), + CountryID: 15, + }) + if err != nil { + t.Fatalf("query overview failed: %v", err) + } + if overview.CoinTotal != 12 || overview.ConsumedCoin != 88 || overview.SalaryTransferCoin != 660 { + t.Fatalf("wallet balance report metrics mismatch: %+v", overview) + } +} + func TestRoomJoinActiveEventSkipsRobotJoin(t *testing.T) { occurredAt := time.Date(2026, 6, 25, 8, 0, 0, 0, time.UTC).UnixMilli() diff --git a/services/statistics-service/internal/storage/mysql/query.go b/services/statistics-service/internal/storage/mysql/query.go index eda99d18..7b972cec 100644 --- a/services/statistics-service/internal/storage/mysql/query.go +++ b/services/statistics-service/internal/storage/mysql/query.go @@ -6,6 +6,7 @@ import ( "math" "sort" "strconv" + "strings" "time" "hyapp/pkg/appcode" @@ -23,86 +24,116 @@ type OverviewQuery struct { } type Overview struct { - AppCode string `json:"app_code"` - StatTZ string `json:"stat_tz"` - StartMS int64 `json:"start_ms"` - EndMS int64 `json:"end_ms"` - CountryID int64 `json:"country_id"` - NewUsers int64 `json:"new_users"` - ActiveUsers int64 `json:"active_users"` - PaidUsers int64 `json:"paid_users"` - NewPaidUsers int64 `json:"new_paid_users"` - RechargeUSDMinor int64 `json:"recharge_usd_minor"` - NewUserRechargeUSDMinor int64 `json:"new_user_recharge_usd_minor"` - CoinSellerRechargeUSDMinor int64 `json:"coin_seller_recharge_usd_minor"` - MifaPayRechargeUSDMinor int64 `json:"mifapay_recharge_usd_minor"` - GoogleRechargeUSDMinor int64 `json:"google_recharge_usd_minor"` - CoinSellerTransferCoin int64 `json:"coin_seller_transfer_coin"` - GiftCoinSpent int64 `json:"gift_coin_spent"` - RealRoomRobotNormalGiftCoin int64 `json:"real_room_robot_normal_gift_coin"` - RealRoomRobotLuckyGiftCoin int64 `json:"real_room_robot_lucky_gift_coin"` - RealRoomRobotSuperGiftCoin int64 `json:"real_room_robot_super_gift_coin"` - RealRoomRobotTotalGiftCoin int64 `json:"real_room_robot_total_gift_coin"` - RealRoomRobotGiftRoomCount int64 `json:"real_room_robot_gift_room_count"` - RealRoomRobotAvgRoomGiftCoin int64 `json:"real_room_robot_avg_room_gift_coin"` - LuckyGiftTurnover int64 `json:"lucky_gift_turnover"` - LuckyGiftPayout int64 `json:"lucky_gift_payout"` - LuckyGiftPayers int64 `json:"lucky_gift_payers"` - LuckyGiftProfit int64 `json:"lucky_gift_profit"` - LuckyGiftPayoutRate float64 `json:"lucky_gift_payout_rate"` - LuckyGiftProfitRate float64 `json:"lucky_gift_profit_rate"` - GameTurnover int64 `json:"game_turnover"` - GamePayout int64 `json:"game_payout"` - GameRefund int64 `json:"game_refund"` - GamePlayers int64 `json:"game_players"` - GameProfit int64 `json:"game_profit"` - GameProfitRate float64 `json:"game_profit_rate"` - ARPUUSDMinor int64 `json:"arpu_usd_minor"` - ARPPUUSDMinor int64 `json:"arppu_usd_minor"` - UPValueUSDMinor int64 `json:"up_value_usd_minor"` - RechargeDeltaRate float64 `json:"recharge_delta_rate"` - NewUserRechargeDeltaRate float64 `json:"new_user_recharge_delta_rate"` - ActiveUsersDeltaRate float64 `json:"active_users_delta_rate"` - PaidUsersDeltaRate float64 `json:"paid_users_delta_rate"` - NewPaidUsersDeltaRate float64 `json:"new_paid_users_delta_rate"` - ARPUDeltaRate float64 `json:"arpu_delta_rate"` - ARPPUDeltaRate float64 `json:"arppu_delta_rate"` - GiftCoinSpentDeltaRate float64 `json:"gift_coin_spent_delta_rate"` - CoinSellerTransferDeltaRate float64 `json:"coin_seller_transfer_coin_delta_rate"` - LuckyGiftTurnoverDeltaRate float64 `json:"lucky_gift_turnover_delta_rate"` - LuckyGiftPayoutDeltaRate float64 `json:"lucky_gift_payout_delta_rate"` - LuckyGiftProfitDeltaRate float64 `json:"lucky_gift_profit_delta_rate"` - GameTurnoverDeltaRate float64 `json:"game_turnover_delta_rate"` - GameProfitDeltaRate float64 `json:"game_profit_delta_rate"` - Retention Retention `json:"retention"` - DailySeries []DailySeriesItem `json:"daily_series"` - GameRanking []GameRank `json:"game_ranking"` - LuckyGiftPools []LuckyGiftPoolStat `json:"lucky_gift_pools"` - CountryBreakdown []CountryBreakdown `json:"country_breakdown"` + AppCode string `json:"app_code"` + StatTZ string `json:"stat_tz"` + StartMS int64 `json:"start_ms"` + EndMS int64 `json:"end_ms"` + CountryID int64 `json:"country_id"` + NewUsers int64 `json:"new_users"` + ActiveUsers int64 `json:"active_users"` + PaidUsers int64 `json:"paid_users"` + NewPaidUsers int64 `json:"new_paid_users"` + RechargeUSDMinor int64 `json:"recharge_usd_minor"` + NewUserRechargeUSDMinor int64 `json:"new_user_recharge_usd_minor"` + CoinSellerRechargeUSDMinor int64 `json:"coin_seller_recharge_usd_minor"` + CoinSellerStockCoin int64 `json:"coin_seller_stock_coin"` + MifaPayRechargeUSDMinor int64 `json:"mifapay_recharge_usd_minor"` + GoogleRechargeUSDMinor int64 `json:"google_recharge_usd_minor"` + CoinSellerTransferCoin int64 `json:"coin_seller_transfer_coin"` + CoinTotal int64 `json:"coin_total"` + ConsumedCoin int64 `json:"consumed_coin"` + PlatformGrantCoin int64 `json:"platform_grant_coin"` + ManualGrantCoin int64 `json:"manual_grant_coin"` + SalaryUSDMinor int64 `json:"salary_usd_minor"` + SalaryTransferCoin int64 `json:"salary_transfer_coin"` + MicOnlineMS int64 `json:"mic_online_ms"` + MicOnlineUsers int64 `json:"mic_online_users"` + AvgMicOnlineMS int64 `json:"avg_mic_online_ms"` + GiftCoinSpent int64 `json:"gift_coin_spent"` + RealRoomRobotNormalGiftCoin int64 `json:"real_room_robot_normal_gift_coin"` + RealRoomRobotLuckyGiftCoin int64 `json:"real_room_robot_lucky_gift_coin"` + RealRoomRobotSuperGiftCoin int64 `json:"real_room_robot_super_gift_coin"` + RealRoomRobotTotalGiftCoin int64 `json:"real_room_robot_total_gift_coin"` + RealRoomRobotGiftRoomCount int64 `json:"real_room_robot_gift_room_count"` + RealRoomRobotAvgRoomGiftCoin int64 `json:"real_room_robot_avg_room_gift_coin"` + LuckyGiftTurnover int64 `json:"lucky_gift_turnover"` + LuckyGiftPayout int64 `json:"lucky_gift_payout"` + LuckyGiftPayers int64 `json:"lucky_gift_payers"` + LuckyGiftProfit int64 `json:"lucky_gift_profit"` + LuckyGiftPayoutRate float64 `json:"lucky_gift_payout_rate"` + LuckyGiftProfitRate float64 `json:"lucky_gift_profit_rate"` + GameTurnover int64 `json:"game_turnover"` + GamePayout int64 `json:"game_payout"` + GameRefund int64 `json:"game_refund"` + GamePlayers int64 `json:"game_players"` + GameProfit int64 `json:"game_profit"` + GameProfitRate float64 `json:"game_profit_rate"` + SuperLuckyGiftTurnover int64 `json:"super_lucky_gift_turnover"` + SuperLuckyGiftPayout int64 `json:"super_lucky_gift_payout"` + SuperLuckyGiftProfit int64 `json:"super_lucky_gift_profit"` + SuperLuckyGiftProfitRate float64 `json:"super_lucky_gift_profit_rate"` + ARPUUSDMinor int64 `json:"arpu_usd_minor"` + ARPPUUSDMinor int64 `json:"arppu_usd_minor"` + UPValueUSDMinor int64 `json:"up_value_usd_minor"` + RechargeDeltaRate float64 `json:"recharge_delta_rate"` + NewUserRechargeDeltaRate float64 `json:"new_user_recharge_delta_rate"` + ActiveUsersDeltaRate float64 `json:"active_users_delta_rate"` + PaidUsersDeltaRate float64 `json:"paid_users_delta_rate"` + NewPaidUsersDeltaRate float64 `json:"new_paid_users_delta_rate"` + ARPUDeltaRate float64 `json:"arpu_delta_rate"` + ARPPUDeltaRate float64 `json:"arppu_delta_rate"` + GiftCoinSpentDeltaRate float64 `json:"gift_coin_spent_delta_rate"` + CoinSellerTransferDeltaRate float64 `json:"coin_seller_transfer_coin_delta_rate"` + LuckyGiftTurnoverDeltaRate float64 `json:"lucky_gift_turnover_delta_rate"` + LuckyGiftPayoutDeltaRate float64 `json:"lucky_gift_payout_delta_rate"` + LuckyGiftProfitDeltaRate float64 `json:"lucky_gift_profit_delta_rate"` + GameTurnoverDeltaRate float64 `json:"game_turnover_delta_rate"` + GameProfitDeltaRate float64 `json:"game_profit_delta_rate"` + Retention Retention `json:"retention"` + DailySeries []DailySeriesItem `json:"daily_series"` + DailyCountryBreakdown []DailyCountryStat `json:"daily_country_breakdown"` + GameRanking []GameRank `json:"game_ranking"` + LuckyGiftPools []LuckyGiftPoolStat `json:"lucky_gift_pools"` + CountryBreakdown []CountryBreakdown `json:"country_breakdown"` + ReportMetricSources []ReportMetricSource `json:"report_metric_sources"` } type DailySeriesItem struct { - StatDay string `json:"stat_day"` - Label string `json:"label"` - NewUsers int64 `json:"new_users"` - ActiveUsers int64 `json:"active_users"` - PaidUsers int64 `json:"paid_users"` - NewPaidUsers int64 `json:"new_paid_users"` - RechargeUsers int64 `json:"recharge_users"` - RechargeUSDMinor int64 `json:"recharge_usd_minor"` - NewUserRechargeUSDMinor int64 `json:"new_user_recharge_usd_minor"` - CoinSellerRechargeUSDMinor int64 `json:"coin_seller_recharge_usd_minor"` - MifaPayRechargeUSDMinor int64 `json:"mifapay_recharge_usd_minor"` - GoogleRechargeUSDMinor int64 `json:"google_recharge_usd_minor"` - CoinSellerTransferCoin int64 `json:"coin_seller_transfer_coin"` - GiftCoinSpent int64 `json:"gift_coin_spent"` - LuckyGiftTurnover int64 `json:"lucky_gift_turnover"` - LuckyGiftPayout int64 `json:"lucky_gift_payout"` - LuckyGiftProfit int64 `json:"lucky_gift_profit"` - GameTurnover int64 `json:"game_turnover"` - GameProfit int64 `json:"game_profit"` - ARPUUSDMinor int64 `json:"arpu_usd_minor"` - ARPPUUSDMinor int64 `json:"arppu_usd_minor"` + StatDay string `json:"stat_day"` + Label string `json:"label"` + NewUsers int64 `json:"new_users"` + ActiveUsers int64 `json:"active_users"` + PaidUsers int64 `json:"paid_users"` + NewPaidUsers int64 `json:"new_paid_users"` + RechargeUsers int64 `json:"recharge_users"` + RechargeUSDMinor int64 `json:"recharge_usd_minor"` + NewUserRechargeUSDMinor int64 `json:"new_user_recharge_usd_minor"` + CoinSellerRechargeUSDMinor int64 `json:"coin_seller_recharge_usd_minor"` + CoinSellerStockCoin int64 `json:"coin_seller_stock_coin"` + MifaPayRechargeUSDMinor int64 `json:"mifapay_recharge_usd_minor"` + GoogleRechargeUSDMinor int64 `json:"google_recharge_usd_minor"` + CoinSellerTransferCoin int64 `json:"coin_seller_transfer_coin"` + CoinTotal int64 `json:"coin_total"` + ConsumedCoin int64 `json:"consumed_coin"` + PlatformGrantCoin int64 `json:"platform_grant_coin"` + ManualGrantCoin int64 `json:"manual_grant_coin"` + SalaryUSDMinor int64 `json:"salary_usd_minor"` + SalaryTransferCoin int64 `json:"salary_transfer_coin"` + MicOnlineMS int64 `json:"mic_online_ms"` + MicOnlineUsers int64 `json:"mic_online_users"` + AvgMicOnlineMS int64 `json:"avg_mic_online_ms"` + GiftCoinSpent int64 `json:"gift_coin_spent"` + LuckyGiftTurnover int64 `json:"lucky_gift_turnover"` + LuckyGiftPayout int64 `json:"lucky_gift_payout"` + LuckyGiftProfit int64 `json:"lucky_gift_profit"` + GameTurnover int64 `json:"game_turnover"` + GameProfit int64 `json:"game_profit"` + SuperLuckyGiftTurnover int64 `json:"super_lucky_gift_turnover"` + SuperLuckyGiftPayout int64 `json:"super_lucky_gift_payout"` + SuperLuckyGiftProfit int64 `json:"super_lucky_gift_profit"` + SuperLuckyGiftProfitRate float64 `json:"super_lucky_gift_profit_rate"` + ARPUUSDMinor int64 `json:"arpu_usd_minor"` + ARPPUUSDMinor int64 `json:"arppu_usd_minor"` } type Retention struct { @@ -135,6 +166,11 @@ type LuckyGiftPoolStat struct { ProfitRate float64 `json:"profit_rate"` } +type superLuckyAggregate struct { + Turnover int64 + Payout int64 +} + type CountryBreakdown struct { CountryID int64 `json:"country_id"` RegionID int64 `json:"region_id"` @@ -144,9 +180,19 @@ type CountryBreakdown struct { RechargeUSDMinor int64 `json:"recharge_usd_minor"` NewUserRechargeUSDMinor int64 `json:"new_user_recharge_usd_minor"` CoinSellerRechargeUSDMinor int64 `json:"coin_seller_recharge_usd_minor"` + CoinSellerStockCoin int64 `json:"coin_seller_stock_coin"` MifaPayRechargeUSDMinor int64 `json:"mifapay_recharge_usd_minor"` GoogleRechargeUSDMinor int64 `json:"google_recharge_usd_minor"` CoinSellerTransferCoin int64 `json:"coin_seller_transfer_coin"` + CoinTotal int64 `json:"coin_total"` + ConsumedCoin int64 `json:"consumed_coin"` + PlatformGrantCoin int64 `json:"platform_grant_coin"` + ManualGrantCoin int64 `json:"manual_grant_coin"` + SalaryUSDMinor int64 `json:"salary_usd_minor"` + SalaryTransferCoin int64 `json:"salary_transfer_coin"` + MicOnlineMS int64 `json:"mic_online_ms"` + MicOnlineUsers int64 `json:"mic_online_users"` + AvgMicOnlineMS int64 `json:"avg_mic_online_ms"` GiftCoinSpent int64 `json:"gift_coin_spent"` LuckyGiftTurnover int64 `json:"lucky_gift_turnover"` LuckyGiftPayout int64 `json:"lucky_gift_payout"` @@ -160,10 +206,29 @@ type CountryBreakdown struct { GamePlayers int64 `json:"game_players"` GameProfit int64 `json:"game_profit"` GameProfitRate float64 `json:"game_profit_rate"` + SuperLuckyGiftTurnover int64 `json:"super_lucky_gift_turnover"` + SuperLuckyGiftPayout int64 `json:"super_lucky_gift_payout"` + SuperLuckyGiftProfit int64 `json:"super_lucky_gift_profit"` + SuperLuckyGiftProfitRate float64 `json:"super_lucky_gift_profit_rate"` ARPUUSDMinor int64 `json:"arpu_usd_minor"` ARPPUUSDMinor int64 `json:"arppu_usd_minor"` } +type DailyCountryStat struct { + StatDay string `json:"stat_day"` + Label string `json:"label"` + CountryBreakdown +} + +type ReportMetricSource struct { + Key string `json:"key"` + Label string `json:"label"` + Status string `json:"status"` + Source string `json:"source"` + Definition string `json:"definition"` + MissingReason string `json:"missing_reason,omitempty"` +} + type SelfGameOverviewQuery struct { AppCode string StatTZ string @@ -361,6 +426,7 @@ func (r *Repository) QueryOverview(ctx context.Context, query OverviewQuery) (Ov return Overview{}, err } overview.LuckyGiftPools = pools + overview.SuperLuckyGiftTurnover, overview.SuperLuckyGiftPayout = sumSuperLuckyGiftPools(pools) if statTZ == StatTZUTC { // activity-service 的 lucky_draw_pool_day_stats 仍是 UTC 日表；只有 UTC 查询能拿它覆盖旧统计行，北京日必须使用本服务 stat_tz=Asia/Shanghai 的读模型。 if activityPools, ok, err := r.queryLuckyGiftPoolsFromActivityStats(ctx, app, startDay, endDay, query.CountryID, query.RegionID); err != nil { @@ -368,6 +434,7 @@ func (r *Repository) QueryOverview(ctx context.Context, query OverviewQuery) (Ov } else if ok { overview.LuckyGiftPools = activityPools overview.LuckyGiftTurnover, overview.LuckyGiftPayout = sumLuckyGiftPools(activityPools) + overview.SuperLuckyGiftTurnover, overview.SuperLuckyGiftPayout = sumSuperLuckyGiftPools(activityPools) } // 上一周期也使用相同的日聚合口径，保证“与昨日相比/前周期相比”不额外查询明细表。 if activityPools, ok, err := r.queryLuckyGiftPoolsFromActivityStats(ctx, app, previousStartDay, previousEndDay, query.CountryID, query.RegionID); err != nil { @@ -395,6 +462,11 @@ func (r *Repository) QueryOverview(ctx context.Context, query OverviewQuery) (Ov return Overview{}, err } overview.CountryBreakdown = breakdown + dailyCountryBreakdown, err := r.queryDailyCountryBreakdown(ctx, app, statTZ, startDay, endDay, query.RegionID) + if err != nil { + return Overview{}, err + } + overview.DailyCountryBreakdown = dailyCountryBreakdown } seriesStartDay, seriesEndDay := startDay, endDay if query.SeriesStartMS > 0 && query.SeriesEndMS > query.SeriesStartMS { @@ -407,6 +479,7 @@ func (r *Repository) QueryOverview(ctx context.Context, query OverviewQuery) (Ov return Overview{}, err } overview.DailySeries = dailySeries + overview.ReportMetricSources = reportMetricSources() return overview, nil } @@ -483,8 +556,11 @@ func (r *Repository) queryAppDayOverviewTotals(ctx context.Context, app, statTZ, row := r.db.QueryRowContext(ctx, ` SELECT COALESCE(SUM(new_users),0), COALESCE(SUM(active_users),0), COALESCE(SUM(paid_users),0), COALESCE(SUM(recharge_usd_minor),0), COALESCE(SUM(new_user_recharge_usd_minor),0), - COALESCE(SUM(coin_seller_recharge_usd_minor),0), COALESCE(SUM(mifapay_recharge_usd_minor),0), + COALESCE(SUM(coin_seller_recharge_usd_minor),0), COALESCE(SUM(coin_seller_stock_coin),0), COALESCE(SUM(mifapay_recharge_usd_minor),0), COALESCE(SUM(google_recharge_usd_minor),0), COALESCE(SUM(coin_seller_transfer_coin),0), + COALESCE(SUM(coin_total),0), COALESCE(SUM(consumed_coin),0), COALESCE(SUM(platform_grant_coin),0), + COALESCE(SUM(manual_grant_coin),0), COALESCE(SUM(salary_usd_minor),0), COALESCE(SUM(salary_transfer_coin),0), + COALESCE(SUM(mic_online_ms),0), COALESCE(SUM(mic_online_users),0), COALESCE(SUM(gift_coin_spent),0), COALESCE(SUM(lucky_gift_turnover),0), COALESCE(SUM(lucky_gift_payout),0), COALESCE(SUM(lucky_gift_payers),0), COALESCE(SUM(game_turnover),0), COALESCE(SUM(game_payout),0), COALESCE(SUM(game_refund),0), @@ -492,7 +568,7 @@ func (r *Repository) queryAppDayOverviewTotals(ctx context.Context, app, statTZ, FROM stat_app_day_country WHERE app_code = ? AND stat_tz = ? AND stat_day BETWEEN ? AND ?`+filter, args...) var overview Overview - if err := row.Scan(&overview.NewUsers, &overview.ActiveUsers, &overview.PaidUsers, &overview.RechargeUSDMinor, &overview.NewUserRechargeUSDMinor, &overview.CoinSellerRechargeUSDMinor, &overview.MifaPayRechargeUSDMinor, &overview.GoogleRechargeUSDMinor, &overview.CoinSellerTransferCoin, &overview.GiftCoinSpent, &overview.LuckyGiftTurnover, &overview.LuckyGiftPayout, &overview.LuckyGiftPayers, &overview.GameTurnover, &overview.GamePayout, &overview.GameRefund, &overview.GamePlayers); err != nil { + if err := row.Scan(&overview.NewUsers, &overview.ActiveUsers, &overview.PaidUsers, &overview.RechargeUSDMinor, &overview.NewUserRechargeUSDMinor, &overview.CoinSellerRechargeUSDMinor, &overview.CoinSellerStockCoin, &overview.MifaPayRechargeUSDMinor, &overview.GoogleRechargeUSDMinor, &overview.CoinSellerTransferCoin, &overview.CoinTotal, &overview.ConsumedCoin, &overview.PlatformGrantCoin, &overview.ManualGrantCoin, &overview.SalaryUSDMinor, &overview.SalaryTransferCoin, &overview.MicOnlineMS, &overview.MicOnlineUsers, &overview.GiftCoinSpent, &overview.LuckyGiftTurnover, &overview.LuckyGiftPayout, &overview.LuckyGiftPayers, &overview.GameTurnover, &overview.GamePayout, &overview.GameRefund, &overview.GamePlayers); err != nil { return Overview{}, err } return overview, nil @@ -565,8 +641,11 @@ func (r *Repository) queryDailySeries(ctx context.Context, app, statTZ, startDay SELECT DATE_FORMAT(stat_day, '%Y-%m-%d'), COALESCE(SUM(new_users),0), COALESCE(SUM(active_users),0), COALESCE(SUM(paid_users),0), COALESCE(SUM(recharge_usd_minor),0), COALESCE(SUM(new_user_recharge_usd_minor),0), - COALESCE(SUM(coin_seller_recharge_usd_minor),0), COALESCE(SUM(mifapay_recharge_usd_minor),0), + COALESCE(SUM(coin_seller_recharge_usd_minor),0), COALESCE(SUM(coin_seller_stock_coin),0), COALESCE(SUM(mifapay_recharge_usd_minor),0), COALESCE(SUM(google_recharge_usd_minor),0), COALESCE(SUM(coin_seller_transfer_coin),0), + COALESCE(SUM(coin_total),0), COALESCE(SUM(consumed_coin),0), COALESCE(SUM(platform_grant_coin),0), + COALESCE(SUM(manual_grant_coin),0), COALESCE(SUM(salary_usd_minor),0), COALESCE(SUM(salary_transfer_coin),0), + COALESCE(SUM(mic_online_ms),0), COALESCE(SUM(mic_online_users),0), COALESCE(SUM(gift_coin_spent),0), COALESCE(SUM(lucky_gift_turnover),0), COALESCE(SUM(lucky_gift_payout),0), COALESCE(SUM(game_turnover),0), COALESCE(SUM(game_payout),0), COALESCE(SUM(game_refund),0) @@ -582,7 +661,7 @@ func (r *Repository) queryDailySeries(ctx context.Context, app, statTZ, startDay for rows.Next() { var item DailySeriesItem var gamePayout, gameRefund int64 - if err := rows.Scan(&item.StatDay, &item.NewUsers, &item.ActiveUsers, &item.PaidUsers, &item.RechargeUSDMinor, &item.NewUserRechargeUSDMinor, &item.CoinSellerRechargeUSDMinor, &item.MifaPayRechargeUSDMinor, &item.GoogleRechargeUSDMinor, &item.CoinSellerTransferCoin, &item.GiftCoinSpent, &item.LuckyGiftTurnover, &item.LuckyGiftPayout, &item.GameTurnover, &gamePayout, &gameRefund); err != nil { + if err := rows.Scan(&item.StatDay, &item.NewUsers, &item.ActiveUsers, &item.PaidUsers, &item.RechargeUSDMinor, &item.NewUserRechargeUSDMinor, &item.CoinSellerRechargeUSDMinor, &item.CoinSellerStockCoin, &item.MifaPayRechargeUSDMinor, &item.GoogleRechargeUSDMinor, &item.CoinSellerTransferCoin, &item.CoinTotal, &item.ConsumedCoin, &item.PlatformGrantCoin, &item.ManualGrantCoin, &item.SalaryUSDMinor, &item.SalaryTransferCoin, &item.MicOnlineMS, &item.MicOnlineUsers, &item.GiftCoinSpent, &item.LuckyGiftTurnover, &item.LuckyGiftPayout, &item.GameTurnover, &gamePayout, &gameRefund); err != nil { return nil, err } item.Label = item.StatDay @@ -600,6 +679,17 @@ func (r *Repository) queryDailySeries(ctx context.Context, app, statTZ, startDay for index := range out { out[index].NewPaidUsers = newPaidUsersByDay[out[index].StatDay] } + superByDay, err := r.querySuperLuckyGiftDaily(ctx, app, statTZ, startDay, endDay, countryID, regionID) + if err != nil { + return nil, err + } + for index := range out { + if super, ok := superByDay[out[index].StatDay]; ok { + out[index].SuperLuckyGiftTurnover = super.Turnover + out[index].SuperLuckyGiftPayout = super.Payout + applyDailySeriesDerivedMetrics(&out[index]) + } + } if normalizeStatTZ(statTZ) == StatTZUTC { if activityDaily, ok, err := r.queryLuckyGiftDailyFromActivityStats(ctx, app, startDay, endDay, countryID, regionID); err != nil { return nil, err @@ -654,8 +744,11 @@ func applyDailySeriesDerivedMetrics(item *DailySeriesItem) { } item.RechargeUsers = item.PaidUsers item.LuckyGiftProfit = item.LuckyGiftTurnover - item.LuckyGiftPayout + item.SuperLuckyGiftProfit = item.SuperLuckyGiftTurnover - item.SuperLuckyGiftPayout + item.SuperLuckyGiftProfitRate = ratio(item.SuperLuckyGiftProfit, item.SuperLuckyGiftTurnover) item.ARPUUSDMinor = div(item.RechargeUSDMinor, item.ActiveUsers) item.ARPPUUSDMinor = div(item.RechargeUSDMinor, item.PaidUsers) + item.AvgMicOnlineMS = div(item.MicOnlineMS, item.MicOnlineUsers) } func applyOverviewDerivedMetrics(overview *Overview) { @@ -672,9 +765,12 @@ func applyOverviewDerivedMetrics(overview *Overview) { overview.ARPUUSDMinor = div(overview.RechargeUSDMinor, overview.ActiveUsers) overview.ARPPUUSDMinor = div(overview.RechargeUSDMinor, overview.PaidUsers) overview.UPValueUSDMinor = overview.ARPPUUSDMinor + overview.AvgMicOnlineMS = div(overview.MicOnlineMS, overview.MicOnlineUsers) overview.LuckyGiftProfit = overview.LuckyGiftTurnover - overview.LuckyGiftPayout overview.LuckyGiftPayoutRate = ratio(overview.LuckyGiftPayout, overview.LuckyGiftTurnover) overview.LuckyGiftProfitRate = ratio(overview.LuckyGiftProfit, overview.LuckyGiftTurnover) + overview.SuperLuckyGiftProfit = overview.SuperLuckyGiftTurnover - overview.SuperLuckyGiftPayout + overview.SuperLuckyGiftProfitRate = ratio(overview.SuperLuckyGiftProfit, overview.SuperLuckyGiftTurnover) } func (overview *Overview) applyDeltaRates(previous Overview) { @@ -772,8 +868,11 @@ func (r *Repository) queryCountryBreakdown(ctx context.Context, app, statTZ, sta SELECT country_id, COALESCE(MAX(region_id),0), COALESCE(SUM(new_users),0), COALESCE(SUM(active_users),0), COALESCE(SUM(paid_users),0), COALESCE(SUM(recharge_usd_minor),0), COALESCE(SUM(new_user_recharge_usd_minor),0), - COALESCE(SUM(coin_seller_recharge_usd_minor),0), COALESCE(SUM(mifapay_recharge_usd_minor),0), + COALESCE(SUM(coin_seller_recharge_usd_minor),0), COALESCE(SUM(coin_seller_stock_coin),0), COALESCE(SUM(mifapay_recharge_usd_minor),0), COALESCE(SUM(google_recharge_usd_minor),0), COALESCE(SUM(coin_seller_transfer_coin),0), + COALESCE(SUM(coin_total),0), COALESCE(SUM(consumed_coin),0), COALESCE(SUM(platform_grant_coin),0), + COALESCE(SUM(manual_grant_coin),0), COALESCE(SUM(salary_usd_minor),0), COALESCE(SUM(salary_transfer_coin),0), + COALESCE(SUM(mic_online_ms),0), COALESCE(SUM(mic_online_users),0), COALESCE(SUM(gift_coin_spent),0), COALESCE(SUM(lucky_gift_turnover),0), COALESCE(SUM(lucky_gift_payout),0), COALESCE(SUM(lucky_gift_payers),0), COALESCE(SUM(game_turnover),0), COALESCE(SUM(game_payout),0), COALESCE(SUM(game_refund),0), @@ -790,13 +889,83 @@ func (r *Repository) queryCountryBreakdown(ctx context.Context, app, statTZ, sta out := []CountryBreakdown{} for rows.Next() { var item CountryBreakdown - if err := rows.Scan(&item.CountryID, &item.RegionID, &item.NewUsers, &item.ActiveUsers, &item.PaidUsers, &item.RechargeUSDMinor, &item.NewUserRechargeUSDMinor, &item.CoinSellerRechargeUSDMinor, &item.MifaPayRechargeUSDMinor, &item.GoogleRechargeUSDMinor, &item.CoinSellerTransferCoin, &item.GiftCoinSpent, &item.LuckyGiftTurnover, &item.LuckyGiftPayout, &item.LuckyGiftPayers, &item.GameTurnover, &item.GamePayout, &item.GameRefund, &item.GamePlayers); err != nil { + if err := rows.Scan(&item.CountryID, &item.RegionID, &item.NewUsers, &item.ActiveUsers, &item.PaidUsers, &item.RechargeUSDMinor, &item.NewUserRechargeUSDMinor, &item.CoinSellerRechargeUSDMinor, &item.CoinSellerStockCoin, &item.MifaPayRechargeUSDMinor, &item.GoogleRechargeUSDMinor, &item.CoinSellerTransferCoin, &item.CoinTotal, &item.ConsumedCoin, &item.PlatformGrantCoin, &item.ManualGrantCoin, &item.SalaryUSDMinor, &item.SalaryTransferCoin, &item.MicOnlineMS, &item.MicOnlineUsers, &item.GiftCoinSpent, &item.LuckyGiftTurnover, &item.LuckyGiftPayout, &item.LuckyGiftPayers, &item.GameTurnover, &item.GamePayout, &item.GameRefund, &item.GamePlayers); err != nil { return nil, err } applyCountryBreakdownDerivedMetrics(&item) out = append(out, item) } - return out, rows.Err() + if err := rows.Err(); err != nil { + return nil, err + } + superByCountry, err := r.querySuperLuckyGiftCountry(ctx, app, statTZ, startDay, endDay, regionID) + if err != nil { + return nil, err + } + for index := range out { + if super, ok := superByCountry[out[index].CountryID]; ok { + out[index].SuperLuckyGiftTurnover = super.Turnover + out[index].SuperLuckyGiftPayout = super.Payout + applyCountryBreakdownDerivedMetrics(&out[index]) + } + } + return out, nil +} + +func (r *Repository) queryDailyCountryBreakdown(ctx context.Context, app, statTZ, startDay, endDay string, regionID int64) ([]DailyCountryStat, error) { + args := []any{app, normalizeStatTZ(statTZ), startDay, endDay} + filter := "" + if regionID > 0 { + filter = " AND region_id = ?" + args = append(args, regionID) + } + rows, err := r.db.QueryContext(ctx, ` + SELECT DATE_FORMAT(stat_day, '%Y-%m-%d'), country_id, COALESCE(MAX(region_id),0), + COALESCE(SUM(new_users),0), COALESCE(SUM(active_users),0), COALESCE(SUM(paid_users),0), + COALESCE(SUM(recharge_usd_minor),0), COALESCE(SUM(new_user_recharge_usd_minor),0), + COALESCE(SUM(coin_seller_recharge_usd_minor),0), COALESCE(SUM(coin_seller_stock_coin),0), COALESCE(SUM(mifapay_recharge_usd_minor),0), + COALESCE(SUM(google_recharge_usd_minor),0), COALESCE(SUM(coin_seller_transfer_coin),0), + COALESCE(SUM(coin_total),0), COALESCE(SUM(consumed_coin),0), COALESCE(SUM(platform_grant_coin),0), + COALESCE(SUM(manual_grant_coin),0), COALESCE(SUM(salary_usd_minor),0), COALESCE(SUM(salary_transfer_coin),0), + COALESCE(SUM(mic_online_ms),0), COALESCE(SUM(mic_online_users),0), + COALESCE(SUM(gift_coin_spent),0), + COALESCE(SUM(lucky_gift_turnover),0), COALESCE(SUM(lucky_gift_payout),0), COALESCE(SUM(lucky_gift_payers),0), + COALESCE(SUM(game_turnover),0), COALESCE(SUM(game_payout),0), COALESCE(SUM(game_refund),0), + COALESCE(SUM(game_players),0) + FROM stat_app_day_country + WHERE app_code = ? AND stat_tz = ? AND stat_day BETWEEN ? AND ?`+filter+` + GROUP BY stat_day, country_id + ORDER BY stat_day ASC, COALESCE(SUM(recharge_usd_minor),0) DESC, COALESCE(SUM(active_users),0) DESC, country_id ASC + LIMIT 5000`, args...) + if err != nil { + return nil, err + } + defer rows.Close() + out := []DailyCountryStat{} + for rows.Next() { + var item DailyCountryStat + if err := rows.Scan(&item.StatDay, &item.CountryID, &item.RegionID, &item.NewUsers, &item.ActiveUsers, &item.PaidUsers, &item.RechargeUSDMinor, &item.NewUserRechargeUSDMinor, &item.CoinSellerRechargeUSDMinor, &item.CoinSellerStockCoin, &item.MifaPayRechargeUSDMinor, &item.GoogleRechargeUSDMinor, &item.CoinSellerTransferCoin, &item.CoinTotal, &item.ConsumedCoin, &item.PlatformGrantCoin, &item.ManualGrantCoin, &item.SalaryUSDMinor, &item.SalaryTransferCoin, &item.MicOnlineMS, &item.MicOnlineUsers, &item.GiftCoinSpent, &item.LuckyGiftTurnover, &item.LuckyGiftPayout, &item.LuckyGiftPayers, &item.GameTurnover, &item.GamePayout, &item.GameRefund, &item.GamePlayers); err != nil { + return nil, err + } + item.Label = item.StatDay + applyCountryBreakdownDerivedMetrics(&item.CountryBreakdown) + out = append(out, item) + } + if err := rows.Err(); err != nil { + return nil, err + } + superByDayCountry, err := r.querySuperLuckyGiftDailyCountry(ctx, app, statTZ, startDay, endDay, regionID) + if err != nil { + return nil, err + } + for index := range out { + if super, ok := superByDayCountry[dailyCountryKey(out[index].StatDay, out[index].CountryID)]; ok { + out[index].SuperLuckyGiftTurnover = super.Turnover + out[index].SuperLuckyGiftPayout = super.Payout + applyCountryBreakdownDerivedMetrics(&out[index].CountryBreakdown) + } + } + return out, nil } func applyCountryBreakdownDerivedMetrics(item *CountryBreakdown) { @@ -811,9 +980,12 @@ func applyCountryBreakdownDerivedMetrics(item *CountryBreakdown) { item.GameProfitRate = ratio(item.GameProfit, item.GameTurnover) item.ARPUUSDMinor = div(item.RechargeUSDMinor, item.ActiveUsers) item.ARPPUUSDMinor = div(item.RechargeUSDMinor, item.PaidUsers) + item.AvgMicOnlineMS = div(item.MicOnlineMS, item.MicOnlineUsers) item.LuckyGiftProfit = item.LuckyGiftTurnover - item.LuckyGiftPayout item.LuckyGiftPayoutRate = ratio(item.LuckyGiftPayout, item.LuckyGiftTurnover) item.LuckyGiftProfitRate = ratio(item.LuckyGiftProfit, item.LuckyGiftTurnover) + item.SuperLuckyGiftProfit = item.SuperLuckyGiftTurnover - item.SuperLuckyGiftPayout + item.SuperLuckyGiftProfitRate = ratio(item.SuperLuckyGiftProfit, item.SuperLuckyGiftTurnover) } func (r *Repository) queryLuckyGiftPools(ctx context.Context, app, statTZ, startDay, endDay string, countryID int64, regionID int64) ([]LuckyGiftPoolStat, error) { @@ -966,6 +1138,141 @@ func sumLuckyGiftPools(pools []LuckyGiftPoolStat) (int64, int64) { return turnover, payout } +func sumSuperLuckyGiftPools(pools []LuckyGiftPoolStat) (int64, int64) { + var turnover, payout int64 + for _, pool := range pools { + if !isSuperLuckyPool(pool.PoolID) { + continue + } + turnover += pool.TurnoverCoin + payout += pool.PayoutCoin + } + return turnover, payout +} + +func (r *Repository) querySuperLuckyGiftDaily(ctx context.Context, app, statTZ, startDay, endDay string, countryID int64, regionID int64) (map[string]superLuckyAggregate, error) { + args := []any{app, normalizeStatTZ(statTZ), startDay, endDay} + filter := "" + if countryID > 0 { + filter += " AND country_id = ?" + args = append(args, countryID) + } + if regionID > 0 { + filter += " AND region_id = ?" + args = append(args, regionID) + } + rows, err := r.db.QueryContext(ctx, ` + SELECT DATE_FORMAT(stat_day, '%Y-%m-%d'), COALESCE(SUM(turnover_coin),0), COALESCE(SUM(payout_coin),0) + FROM stat_lucky_gift_pool_day_country + WHERE app_code = ? AND stat_tz = ? AND stat_day BETWEEN ? AND ? AND `+superLuckyPoolSQL()+filter+` + GROUP BY stat_day`, args...) + if err != nil { + return nil, err + } + defer rows.Close() + out := map[string]superLuckyAggregate{} + for rows.Next() { + var day string + var item superLuckyAggregate + if err := rows.Scan(&day, &item.Turnover, &item.Payout); err != nil { + return nil, err + } + out[day] = item + } + return out, rows.Err() +} + +func (r *Repository) querySuperLuckyGiftCountry(ctx context.Context, app, statTZ, startDay, endDay string, regionID int64) (map[int64]superLuckyAggregate, error) { + args := []any{app, normalizeStatTZ(statTZ), startDay, endDay} + filter := "" + if regionID > 0 { + filter = " AND region_id = ?" + args = append(args, regionID) + } + rows, err := r.db.QueryContext(ctx, ` + SELECT country_id, COALESCE(SUM(turnover_coin),0), COALESCE(SUM(payout_coin),0) + FROM stat_lucky_gift_pool_day_country + WHERE app_code = ? AND stat_tz = ? AND stat_day BETWEEN ? AND ? AND `+superLuckyPoolSQL()+filter+` + GROUP BY country_id`, args...) + if err != nil { + return nil, err + } + defer rows.Close() + out := map[int64]superLuckyAggregate{} + for rows.Next() { + var countryID int64 + var item superLuckyAggregate + if err := rows.Scan(&countryID, &item.Turnover, &item.Payout); err != nil { + return nil, err + } + out[countryID] = item + } + return out, rows.Err() +} + +func (r *Repository) querySuperLuckyGiftDailyCountry(ctx context.Context, app, statTZ, startDay, endDay string, regionID int64) (map[string]superLuckyAggregate, error) { + args := []any{app, normalizeStatTZ(statTZ), startDay, endDay} + filter := "" + if regionID > 0 { + filter = " AND region_id = ?" + args = append(args, regionID) + } + rows, err := r.db.QueryContext(ctx, ` + SELECT DATE_FORMAT(stat_day, '%Y-%m-%d'), country_id, COALESCE(SUM(turnover_coin),0), COALESCE(SUM(payout_coin),0) + FROM stat_lucky_gift_pool_day_country + WHERE app_code = ? AND stat_tz = ? AND stat_day BETWEEN ? AND ? AND `+superLuckyPoolSQL()+filter+` + GROUP BY stat_day, country_id`, args...) + if err != nil { + return nil, err + } + defer rows.Close() + out := map[string]superLuckyAggregate{} + for rows.Next() { + var day string + var countryID int64 + var item superLuckyAggregate + if err := rows.Scan(&day, &countryID, &item.Turnover, &item.Payout); err != nil { + return nil, err + } + out[dailyCountryKey(day, countryID)] = item + } + return out, rows.Err() +} + +func dailyCountryKey(day string, countryID int64) string { + return day + ":" + strconv.FormatInt(countryID, 10) +} + +func superLuckyPoolSQL() string { + return "LOWER(pool_id) LIKE '%super_lucky%'" +} + +func isSuperLuckyPool(poolID string) bool { + return strings.Contains(strings.ToLower(strings.TrimSpace(poolID)), "super_lucky") +} + +func reportMetricSources() []ReportMetricSource { + return []ReportMetricSource{ + {Key: "coin_seller_recharge_usd_minor", Label: "币商充值", Status: "available", Source: "WalletCoinSellerStockPurchased / WalletRechargeRecorded", Definition: "币商进货 USDT 和未识别为 Google/MifaPay/USDT-TRC20/币商转账的充值美元最小单位。"}, + {Key: "coin_seller_stock_coin", Label: "币商充值金币", Status: "available", Source: "WalletCoinSellerStockPurchased coin_amount", Definition: "币商库存进货金币数量，扣减类库存调整按负数冲减。"}, + {Key: "coin_seller_transfer_coin", Label: "币商出货金币", Status: "available", Source: "WalletRechargeRecorded recharge_type=coin_seller_transfer|coin_seller", Definition: "币商向用户转入普通金币数量。"}, + {Key: "google_recharge_usd_minor", Label: "谷歌充值", Status: "available", Source: "WalletRechargeRecorded recharge_type=google|google_play", Definition: "Google Play 充值美元最小单位。"}, + {Key: "mifapay_recharge_usd_minor", Label: "三方充值", Status: "available", Source: "WalletRechargeRecorded recharge_type=mifapay|v5pay", Definition: "当前三方充值列使用 MifaPay/V5Pay 美元最小单位。"}, + {Key: "game_turnover", Label: "游戏流水/利润率", Status: "available", Source: "GameOrderSettled", Definition: "debit/bet 为流水，credit/payout 为返奖，refund/reverse 为退款，利润率=(流水-返奖-退款)/流水。"}, + {Key: "lucky_gift_turnover", Label: "幸运礼物流水/利润率", Status: "available", Source: "RoomGiftSent + WalletLuckyGiftRewardCredited", Definition: "带 pool_id 的送礼金币为流水，钱包返奖为返奖，利润率=(流水-返奖)/流水。"}, + {Key: "super_lucky_gift_turnover", Label: "超级幸运礼物流水/利润率", Status: "available", Source: "stat_lucky_gift_pool_day_country", Definition: "pool_id 包含 super_lucky 的奖池流水和返奖派生，利润率=(流水-返奖)/流水。"}, + {Key: "gift_coin_spent", Label: "礼物流水", Status: "available", Source: "RoomGiftSent", Definition: "非机器人真实送礼 coin_spent，历史事件无 coin_spent 时回退 gift_value。"}, + {Key: "coin_total", Label: "金币数量(总和)", Status: "available", Source: "wallet_accounts / wallet_entries", Definition: "普通用户 COIN 钱包日终 available+frozen 余额总和；当天实时行按 WalletBalanceChanged 增量滚动，回填固化日终余额。"}, + {Key: "consumed_coin", Label: "消耗金币", Status: "available", Source: "WalletBalanceChanged biz_type allowlist", Definition: "仅统计消费类普通 COIN 扣减：送礼、VIP、资源商城、红包创建、游戏下注、转盘抽奖和 CP 分手费。"}, + {Key: "platform_grant_coin", Label: "平台发放金币", Status: "available", Source: "Wallet*RewardCredited", Definition: "系统奖励类 COIN 正向发放，包括任务、幸运礼物返奖、转盘、房间流水、邀请活动和代理开通奖励。"}, + {Key: "manual_grant_coin", Label: "人工发放金币", Status: "available", Source: "WalletBalanceChanged biz_type=manual_credit", Definition: "后台人工调账向普通 COIN 钱包正向发放的金币。"}, + {Key: "salary_usd_minor", Label: "工资总和(主播/代理)", Status: "available", Source: "host_salary_settlement_records / WalletBalanceChanged", Definition: "成功结算的主播工资和代理工资 USD minor 合计，不包含 residual、BD/Admin 工资。"}, + {Key: "salary_transfer_coin", Label: "工资兑换金币", Status: "available", Source: "WalletBalanceChanged biz_type=salary_transfer_to_coin_seller / wallet_transactions", Definition: "用户工资钱包转给币商时，币商 COIN_SELLER_COIN 入账的金币数量；国家/区域按发起转账用户归属。"}, + {Key: "avg_mic_online_ms", Label: "平均麦上时间", Status: "available", Source: "user_mic_daily_stats", Definition: "麦上总时长 / 当日麦上用户数，只统计 mic_online_ms > 0 的用户。"}, + {Key: "mic_online_ms", Label: "麦上总时长", Status: "available", Source: "user_mic_daily_stats.mic_online_ms", Definition: "用户每日麦上时长按国家/区域聚合后的毫秒总和。"}, + } +} + func (r *Repository) querySelfGameEvents(ctx context.Context, app, statTZ, startDay, endDay, gameID string, countryID int64, regionID int64) ([]SelfGameEventStat, error) { filter, args := selfGameFilter(app, statTZ, startDay, endDay, gameID, countryID, regionID) rows, err := r.db.QueryContext(ctx, ` diff --git a/services/statistics-service/internal/storage/mysql/query_test.go b/services/statistics-service/internal/storage/mysql/query_test.go index 4157d89f..dfabd894 100644 --- a/services/statistics-service/internal/storage/mysql/query_test.go +++ b/services/statistics-service/internal/storage/mysql/query_test.go @@ -128,6 +128,78 @@ func TestConsumeUserRegisteredCountsOnlyNewRegistration(t *testing.T) { } } +func TestConsumeReportMetricsAggregatesWalletSalaryGrantAndMic(t *testing.T) { + ctx := context.Background() + _, file, _, _ := runtime.Caller(0) + statsSchema := mysqlschema.New(t, mysqlschema.Config{ + InitDBPath: mysqlschema.InitDBPath(t, file, "..", "..", "..", "deploy", "mysql", "initdb", "001_statistics_service.sql"), + DatabasePrefix: "hy_stats_report_metrics_test", + }) + repository, err := Open(ctx, statsSchema.DSN) + if err != nil { + t.Fatalf("open repository: %v", err) + } + t.Cleanup(func() { _ = repository.Close() }) + + occurredAt := time.Date(2026, 6, 22, 3, 4, 5, 0, time.UTC).UnixMilli() + for _, event := range []UserRegisteredEvent{ + {AppCode: "lalu", EventID: "user_registered:report:1", UserID: 7001, CountryID: 15, RegionID: 10, OccurredAtMS: occurredAt}, + {AppCode: "lalu", EventID: "user_registered:report:2", UserID: 7002, CountryID: 15, RegionID: 10, OccurredAtMS: occurredAt}, + } { + if err := repository.ConsumeUserRegistered(ctx, event); err != nil { + t.Fatalf("consume user dimension: %v", err) + } + } + for _, event := range []WalletBalanceChangedEvent{ + {AppCode: "lalu", EventID: "balance:coin:recharge", UserID: 7001, AssetType: "COIN", BizType: "google_play_recharge", AvailableDelta: 1000, OccurredAtMS: occurredAt}, + {AppCode: "lalu", EventID: "balance:coin:gift", UserID: 7001, AssetType: "COIN", BizType: "gift_debit", AvailableDelta: -200, OccurredAtMS: occurredAt}, + {AppCode: "lalu", EventID: "balance:coin:refund", UserID: 7001, AssetType: "COIN", BizType: "game_refund", AvailableDelta: 50, OccurredAtMS: occurredAt}, + {AppCode: "lalu", EventID: "balance:coin:manual", UserID: 7001, AssetType: "COIN", BizType: "manual_credit", AvailableDelta: 70, OccurredAtMS: occurredAt}, + {AppCode: "lalu", EventID: "balance:salary:host", UserID: 7001, AssetType: "HOST_SALARY_USD", BizType: "host_salary_settlement", AvailableDelta: 123, OccurredAtMS: occurredAt}, + {AppCode: "lalu", EventID: "balance:salary:coin-seller", UserID: 9001, DimensionUserID: 7001, AssetType: "COIN_SELLER_COIN", BizType: "salary_transfer_to_coin_seller", AvailableDelta: 444, OccurredAtMS: occurredAt}, + } { + if err := repository.ConsumeWalletBalanceChanged(ctx, event); err != nil { + t.Fatalf("consume wallet balance changed %s: %v", event.EventID, err) + } + } + if err := repository.ConsumePlatformGrantCoin(ctx, PlatformGrantCoinEvent{ + AppCode: "lalu", EventID: "grant:task:1", EventType: "WalletTaskRewardCredited", UserID: 7001, Amount: 33, OccurredAtMS: occurredAt, + }); err != nil { + t.Fatalf("consume platform grant: %v", err) + } + for _, event := range []UserMicDailyStatsUpdatedEvent{ + {AppCode: "lalu", EventID: "mic:7001:first", UserID: 7001, StatDate: "2026-06-22", MicOnlineDeltaMS: 60_000, OccurredAtMS: occurredAt}, + {AppCode: "lalu", EventID: "mic:7001:second", UserID: 7001, StatDate: "2026-06-22", MicOnlineDeltaMS: 30_000, OccurredAtMS: occurredAt}, + {AppCode: "lalu", EventID: "mic:7002:first", UserID: 7002, StatDate: "2026-06-22", MicOnlineDeltaMS: 30_000, OccurredAtMS: occurredAt}, + } { + if err := repository.ConsumeUserMicDailyStatsUpdated(ctx, event); err != nil { + t.Fatalf("consume mic stats %s: %v", event.EventID, err) + } + } + + start := time.Date(2026, 6, 22, 0, 0, 0, 0, time.UTC).UnixMilli() + overview, err := repository.QueryOverview(ctx, OverviewQuery{AppCode: "lalu", StartMS: start, EndMS: start + int64(24*time.Hour/time.Millisecond)}) + if err != nil { + t.Fatalf("query overview: %v", err) + } + if overview.CoinTotal != 920 || overview.ConsumedCoin != 200 || overview.PlatformGrantCoin != 33 || overview.ManualGrantCoin != 70 || overview.SalaryUSDMinor != 123 || overview.SalaryTransferCoin != 444 { + t.Fatalf("report wallet metrics mismatch: %+v", overview) + } + if overview.MicOnlineMS != 120_000 || overview.MicOnlineUsers != 2 || overview.AvgMicOnlineMS != 60_000 { + t.Fatalf("mic metrics mismatch: %+v", overview) + } + if len(overview.CountryBreakdown) != 1 { + t.Fatalf("country breakdown missing: %+v", overview.CountryBreakdown) + } + country := overview.CountryBreakdown[0] + if country.CoinTotal != 920 || country.ConsumedCoin != 200 || country.SalaryTransferCoin != 444 || country.AvgMicOnlineMS != 60_000 { + t.Fatalf("country report metrics mismatch: %+v", country) + } + if len(overview.DailySeries) != 1 || overview.DailySeries[0].SalaryTransferCoin != 444 || overview.DailySeries[0].AvgMicOnlineMS != 60_000 { + t.Fatalf("daily report metrics mismatch: %+v", overview.DailySeries) + } +} + func TestConsumeRechargeWritesUTCAndChinaStatDays(t *testing.T) { ctx := context.Background() _, file, _, _ := runtime.Caller(0) @@ -327,11 +399,11 @@ func TestQueryOverviewReturnsCountryBreakdownAndIncludesCoinSellerRecharge(t *te if _, err := repository.db.ExecContext(ctx, ` INSERT INTO stat_app_day_country ( app_code, stat_day, country_id, region_id, active_users, paid_users, recharge_usd_minor, - coin_seller_recharge_usd_minor, google_recharge_usd_minor, updated_at_ms + coin_seller_recharge_usd_minor, coin_seller_stock_coin, google_recharge_usd_minor, updated_at_ms ) VALUES - ('lalu', '2026-06-06', 86, 210, 10, 2, 150, 400, 150, 1), - ('lalu', '2026-06-06', 66, 210, 5, 1, 100, 0, 100, 1), - ('lalu', '2026-06-06', 99, 220, 3, 1, 900, 0, 900, 1)`); err != nil { + ('lalu', '2026-06-06', 86, 210, 10, 2, 150, 400, 880000, 150, 1), + ('lalu', '2026-06-06', 66, 210, 5, 1, 100, 0, 0, 100, 1), + ('lalu', '2026-06-06', 99, 220, 3, 1, 900, 0, 0, 900, 1)`); err != nil { t.Fatalf("seed country statistics: %v", err) } @@ -340,17 +412,70 @@ func TestQueryOverviewReturnsCountryBreakdownAndIncludesCoinSellerRecharge(t *te if err != nil { t.Fatalf("query overview: %v", err) } - if overview.RechargeUSDMinor != 650 || overview.CoinSellerRechargeUSDMinor != 400 { + if overview.RechargeUSDMinor != 650 || overview.CoinSellerRechargeUSDMinor != 400 || overview.CoinSellerStockCoin != 880000 { t.Fatalf("overview recharge mismatch: %+v", overview) } if len(overview.CountryBreakdown) != 2 { t.Fatalf("country breakdown should come from aggregate query: %+v", overview.CountryBreakdown) } - if overview.CountryBreakdown[0].CountryID != 86 || overview.CountryBreakdown[0].RechargeUSDMinor != 550 || overview.CountryBreakdown[0].ARPPUUSDMinor != 275 { + if overview.CountryBreakdown[0].CountryID != 86 || overview.CountryBreakdown[0].RechargeUSDMinor != 550 || overview.CountryBreakdown[0].CoinSellerStockCoin != 880000 || overview.CountryBreakdown[0].ARPPUUSDMinor != 275 { t.Fatalf("first country breakdown mismatch: %+v", overview.CountryBreakdown[0]) } } +func TestQueryOverviewReturnsDailyCountryBreakdownAndSuperLuckyGift(t *testing.T) { + ctx := context.Background() + _, file, _, _ := runtime.Caller(0) + statsSchema := mysqlschema.New(t, mysqlschema.Config{ + InitDBPath: mysqlschema.InitDBPath(t, file, "..", "..", "..", "deploy", "mysql", "initdb", "001_statistics_service.sql"), + DatabasePrefix: "hy_stats_query_daily_country_test", + }) + repository, err := Open(ctx, statsSchema.DSN) + if err != nil { + t.Fatalf("open repository: %v", err) + } + t.Cleanup(func() { _ = repository.Close() }) + + if _, err := repository.db.ExecContext(ctx, ` + INSERT INTO stat_app_day_country ( + app_code, stat_day, country_id, region_id, active_users, paid_users, recharge_usd_minor, + coin_seller_stock_coin, lucky_gift_turnover, lucky_gift_payout, updated_at_ms + ) VALUES + ('lalu', '2026-06-05', 86, 210, 10, 2, 100, 7000, 900, 200, 1), + ('lalu', '2026-06-05', 66, 210, 5, 1, 80, 1000, 100, 20, 1), + ('lalu', '2026-06-06', 86, 210, 12, 3, 150, 9000, 1200, 300, 1)`); err != nil { + t.Fatalf("seed app country stats: %v", err) + } + if _, err := repository.db.ExecContext(ctx, ` + INSERT INTO stat_lucky_gift_pool_day_country ( + app_code, stat_day, country_id, region_id, pool_id, turnover_coin, payout_coin, updated_at_ms + ) VALUES + ('lalu', '2026-06-05', 86, 210, 'super_lucky', 700, 120, 1), + ('lalu', '2026-06-05', 66, 210, 'lucky', 100, 20, 1), + ('lalu', '2026-06-06', 86, 210, 'super_lucky_v2', 900, 180, 1)`); err != nil { + t.Fatalf("seed lucky gift pools: %v", err) + } + + start := time.Date(2026, 6, 5, 0, 0, 0, 0, time.UTC).UnixMilli() + overview, err := repository.QueryOverview(ctx, OverviewQuery{AppCode: "lalu", StartMS: start, EndMS: start + int64(48*time.Hour/time.Millisecond), RegionID: 210}) + if err != nil { + t.Fatalf("query overview: %v", err) + } + if overview.SuperLuckyGiftTurnover != 1600 || overview.SuperLuckyGiftPayout != 300 || overview.SuperLuckyGiftProfit != 1300 { + t.Fatalf("super lucky overview mismatch: %+v", overview) + } + if len(overview.DailyCountryBreakdown) != 3 { + t.Fatalf("daily country breakdown length mismatch: %+v", overview.DailyCountryBreakdown) + } + first := overview.DailyCountryBreakdown[0] + if first.StatDay != "2026-06-05" || first.CountryID != 86 || first.CoinSellerStockCoin != 7000 || first.SuperLuckyGiftTurnover != 700 || first.SuperLuckyGiftProfitRate <= 0 { + t.Fatalf("first daily country row mismatch: %+v", first) + } + if len(overview.ReportMetricSources) == 0 { + t.Fatalf("report metric sources should document available and missing report fields") + } +} + func TestQuerySelfGameOverviewUsesRealUserStakeForPlatformProfitRate(t *testing.T) { ctx := context.Background() _, file, _, _ := runtime.Caller(0) diff --git a/services/statistics-service/internal/storage/mysql/repository.go b/services/statistics-service/internal/storage/mysql/repository.go index 20ce10dc..6b7325b1 100644 --- a/services/statistics-service/internal/storage/mysql/repository.go +++ b/services/statistics-service/internal/storage/mysql/repository.go @@ -108,8 +108,13 @@ func (r *Repository) Migrate(ctx context.Context) error { region_id BIGINT NOT NULL DEFAULT 0, new_users BIGINT NOT NULL DEFAULT 0, active_users BIGINT NOT NULL DEFAULT 0, paid_users BIGINT NOT NULL DEFAULT 0, recharge_usd_minor BIGINT NOT NULL DEFAULT 0, new_user_recharge_usd_minor BIGINT NOT NULL DEFAULT 0, - coin_seller_recharge_usd_minor BIGINT NOT NULL DEFAULT 0, mifapay_recharge_usd_minor BIGINT NOT NULL DEFAULT 0, + coin_seller_recharge_usd_minor BIGINT NOT NULL DEFAULT 0, coin_seller_stock_coin BIGINT NOT NULL DEFAULT 0, + mifapay_recharge_usd_minor BIGINT NOT NULL DEFAULT 0, google_recharge_usd_minor BIGINT NOT NULL DEFAULT 0, coin_seller_transfer_coin BIGINT NOT NULL DEFAULT 0, + coin_total BIGINT NOT NULL DEFAULT 0, consumed_coin BIGINT NOT NULL DEFAULT 0, + platform_grant_coin BIGINT NOT NULL DEFAULT 0, manual_grant_coin BIGINT NOT NULL DEFAULT 0, + salary_usd_minor BIGINT NOT NULL DEFAULT 0, salary_transfer_coin BIGINT NOT NULL DEFAULT 0, + mic_online_ms BIGINT NOT NULL DEFAULT 0, mic_online_users BIGINT NOT NULL DEFAULT 0, gift_coin_spent BIGINT NOT NULL DEFAULT 0, lucky_gift_turnover BIGINT NOT NULL DEFAULT 0, lucky_gift_payout BIGINT NOT NULL DEFAULT 0, lucky_gift_payers BIGINT NOT NULL DEFAULT 0, game_turnover BIGINT NOT NULL DEFAULT 0, game_players BIGINT NOT NULL DEFAULT 0, @@ -131,6 +136,22 @@ func (r *Repository) Migrate(ctx context.Context) error { PRIMARY KEY (app_code, stat_tz, user_id), KEY idx_stat_user_registration_day (app_code, stat_tz, registered_day, country_id), KEY idx_stat_user_registration_region (app_code, stat_tz, registered_day, region_id) ) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_unicode_ci`, + `CREATE TABLE IF NOT EXISTS stat_user_dimension ( + app_code VARCHAR(32) NOT NULL, user_id BIGINT NOT NULL, + country_id BIGINT NOT NULL DEFAULT 0, region_id BIGINT NOT NULL DEFAULT 0, + updated_at_ms BIGINT NOT NULL, + PRIMARY KEY (app_code, user_id), + KEY idx_stat_user_dimension_country (app_code, country_id), + KEY idx_stat_user_dimension_region (app_code, region_id) + ) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_unicode_ci`, + `CREATE TABLE IF NOT EXISTS stat_user_day_mic ( + app_code VARCHAR(32) NOT NULL, stat_tz VARCHAR(64) NOT NULL DEFAULT 'UTC', stat_day DATE NOT NULL, + country_id BIGINT NOT NULL DEFAULT 0, region_id BIGINT NOT NULL DEFAULT 0, + user_id BIGINT NOT NULL, mic_online_ms BIGINT NOT NULL DEFAULT 0, updated_at_ms BIGINT NOT NULL, + PRIMARY KEY (app_code, stat_tz, stat_day, user_id), + KEY idx_stat_user_day_mic_country (app_code, stat_tz, stat_day, country_id), + KEY idx_stat_user_day_mic_region (app_code, stat_tz, stat_day, region_id) + ) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_unicode_ci`, `CREATE TABLE IF NOT EXISTS stat_recharge_day_payers ( app_code VARCHAR(32) NOT NULL, stat_tz VARCHAR(64) NOT NULL DEFAULT 'UTC', stat_day DATE NOT NULL, country_id BIGINT NOT NULL DEFAULT 0, region_id BIGINT NOT NULL DEFAULT 0, @@ -261,6 +282,7 @@ func (r *Repository) Migrate(ctx context.Context) error { "stat_app_day_country", "stat_user_day_activity", "stat_user_registration", + "stat_user_day_mic", "stat_recharge_day_payers", "stat_lucky_gift_day_payers", "stat_lucky_gift_pool_day_country", @@ -284,10 +306,20 @@ func (r *Repository) Migrate(ctx context.Context) error { `ALTER TABLE stat_app_day_country ADD COLUMN game_refund BIGINT NOT NULL DEFAULT 0 AFTER game_payout`, `ALTER TABLE stat_app_day_country ADD COLUMN lucky_gift_payout BIGINT NOT NULL DEFAULT 0 AFTER lucky_gift_turnover`, `ALTER TABLE stat_app_day_country ADD COLUMN coin_seller_transfer_coin BIGINT NOT NULL DEFAULT 0 AFTER google_recharge_usd_minor`, + `ALTER TABLE stat_app_day_country ADD COLUMN coin_seller_stock_coin BIGINT NOT NULL DEFAULT 0 AFTER coin_seller_recharge_usd_minor`, + `ALTER TABLE stat_app_day_country ADD COLUMN coin_total BIGINT NOT NULL DEFAULT 0 AFTER coin_seller_transfer_coin`, + `ALTER TABLE stat_app_day_country ADD COLUMN consumed_coin BIGINT NOT NULL DEFAULT 0 AFTER coin_total`, + `ALTER TABLE stat_app_day_country ADD COLUMN platform_grant_coin BIGINT NOT NULL DEFAULT 0 AFTER consumed_coin`, + `ALTER TABLE stat_app_day_country ADD COLUMN manual_grant_coin BIGINT NOT NULL DEFAULT 0 AFTER platform_grant_coin`, + `ALTER TABLE stat_app_day_country ADD COLUMN salary_usd_minor BIGINT NOT NULL DEFAULT 0 AFTER manual_grant_coin`, + `ALTER TABLE stat_app_day_country ADD COLUMN salary_transfer_coin BIGINT NOT NULL DEFAULT 0 AFTER salary_usd_minor`, + `ALTER TABLE stat_app_day_country ADD COLUMN mic_online_ms BIGINT NOT NULL DEFAULT 0 AFTER salary_transfer_coin`, + `ALTER TABLE stat_app_day_country ADD COLUMN mic_online_users BIGINT NOT NULL DEFAULT 0 AFTER mic_online_ms`, `ALTER TABLE stat_game_day_country ADD COLUMN refund_coin BIGINT NOT NULL DEFAULT 0 AFTER payout_coin`, `ALTER TABLE stat_app_day_country ADD COLUMN region_id BIGINT NOT NULL DEFAULT 0 AFTER country_id`, `ALTER TABLE stat_user_day_activity ADD COLUMN region_id BIGINT NOT NULL DEFAULT 0 AFTER country_id`, `ALTER TABLE stat_user_registration ADD COLUMN region_id BIGINT NOT NULL DEFAULT 0 AFTER country_id`, + `ALTER TABLE stat_user_day_mic ADD COLUMN region_id BIGINT NOT NULL DEFAULT 0 AFTER country_id`, `ALTER TABLE stat_recharge_day_payers ADD COLUMN region_id BIGINT NOT NULL DEFAULT 0 AFTER country_id`, `ALTER TABLE stat_lucky_gift_day_payers ADD COLUMN region_id BIGINT NOT NULL DEFAULT 0 AFTER country_id`, `ALTER TABLE stat_lucky_gift_pool_day_country ADD COLUMN region_id BIGINT NOT NULL DEFAULT 0 AFTER country_id`, @@ -296,6 +328,7 @@ func (r *Repository) Migrate(ctx context.Context) error { `ALTER TABLE stat_app_day_country ADD KEY idx_stat_app_day_region (app_code, stat_tz, stat_day, region_id)`, `ALTER TABLE stat_user_day_activity ADD KEY idx_stat_user_day_region (app_code, stat_tz, stat_day, region_id)`, `ALTER TABLE stat_user_registration ADD KEY idx_stat_user_registration_region (app_code, stat_tz, registered_day, region_id)`, + `ALTER TABLE stat_user_day_mic ADD KEY idx_stat_user_day_mic_region (app_code, stat_tz, stat_day, region_id)`, `ALTER TABLE stat_recharge_day_payers ADD KEY idx_stat_recharge_payer_region (app_code, stat_tz, stat_day, region_id)`, `ALTER TABLE stat_lucky_gift_day_payers ADD KEY idx_stat_lucky_payer_region (app_code, stat_tz, stat_day, region_id)`, `ALTER TABLE stat_lucky_gift_pool_day_country ADD KEY idx_stat_lucky_pool_region (app_code, stat_tz, stat_day, region_id)`, @@ -311,6 +344,8 @@ func (r *Repository) Migrate(ctx context.Context) error { "stat_app_day_country": {"app_code", "stat_tz", "stat_day", "country_id", "region_id"}, "stat_user_day_activity": {"app_code", "stat_tz", "stat_day", "user_id"}, "stat_user_registration": {"app_code", "stat_tz", "user_id"}, + "stat_user_dimension": {"app_code", "user_id"}, + "stat_user_day_mic": {"app_code", "stat_tz", "stat_day", "user_id"}, "stat_recharge_day_payers": {"app_code", "stat_tz", "stat_day", "user_id"}, "stat_lucky_gift_day_payers": {"app_code", "stat_tz", "stat_day", "user_id"}, "stat_lucky_gift_pool_day_country": {"app_code", "stat_tz", "stat_day", "country_id", "region_id", "pool_id"}, @@ -338,6 +373,10 @@ func (r *Repository) Migrate(ctx context.Context) error { {"stat_user_day_activity", "idx_stat_user_day_region", []string{"app_code", "stat_tz", "stat_day", "region_id"}}, {"stat_user_registration", "idx_stat_user_registration_day", []string{"app_code", "stat_tz", "registered_day", "country_id"}}, {"stat_user_registration", "idx_stat_user_registration_region", []string{"app_code", "stat_tz", "registered_day", "region_id"}}, + {"stat_user_dimension", "idx_stat_user_dimension_country", []string{"app_code", "country_id"}}, + {"stat_user_dimension", "idx_stat_user_dimension_region", []string{"app_code", "region_id"}}, + {"stat_user_day_mic", "idx_stat_user_day_mic_country", []string{"app_code", "stat_tz", "stat_day", "country_id"}}, + {"stat_user_day_mic", "idx_stat_user_day_mic_region", []string{"app_code", "stat_tz", "stat_day", "region_id"}}, {"stat_recharge_day_payers", "idx_stat_recharge_payer_country", []string{"app_code", "stat_tz", "stat_day", "country_id"}}, {"stat_recharge_day_payers", "idx_stat_recharge_payer_region", []string{"app_code", "stat_tz", "stat_day", "region_id"}}, {"stat_lucky_gift_day_payers", "idx_stat_lucky_payer_country", []string{"app_code", "stat_tz", "stat_day", "country_id"}}, @@ -451,6 +490,15 @@ type UserRegisteredEvent struct { OccurredAtMS int64 } +type UserRegionChangedEvent struct { + AppCode string + EventID string + UserID int64 + CountryID int64 + RegionID int64 + OccurredAtMS int64 +} + type RechargeEvent struct { AppCode string EventID string @@ -503,6 +551,41 @@ type LuckyGiftRewardEvent struct { OccurredAtMS int64 } +type PlatformGrantCoinEvent struct { + AppCode string + EventID string + EventType string + UserID int64 + CountryID int64 + RegionID int64 + Amount int64 + OccurredAtMS int64 +} + +type WalletBalanceChangedEvent struct { + AppCode string + EventID string + EventType string + UserID int64 + DimensionUserID int64 + AssetType string + BizType string + AvailableDelta int64 + FrozenDelta int64 + CountryID int64 + RegionID int64 + OccurredAtMS int64 +} + +type UserMicDailyStatsUpdatedEvent struct { + AppCode string + EventID string + UserID int64 + StatDate string + MicOnlineDeltaMS int64 + OccurredAtMS int64 +} + type UserActiveEvent struct { AppCode string EventID string @@ -604,6 +687,9 @@ type SelfGamePoolAdjustmentEvent struct { func (r *Repository) ConsumeUserRegistered(ctx context.Context, event UserRegisteredEvent) error { return r.withEvent(ctx, SourceUser, event.EventID, "UserRegistered", func(tx *sql.Tx, nowMS int64) error { countryID, regionID := normalizeDimension(event.CountryID, event.RegionID) + if err := upsertUserDimension(ctx, tx, event.AppCode, event.UserID, countryID, regionID, nowMS); err != nil { + return err + } for _, scope := range statDayScopesFromContext(ctx, event.OccurredAtMS) { if err := ensureAppDay(ctx, tx, event.AppCode, scope.tz, scope.day, countryID, regionID, nowMS); err != nil { return err @@ -637,6 +723,13 @@ func (r *Repository) ConsumeUserRegistered(ctx context.Context, event UserRegist }) } +func (r *Repository) ConsumeUserRegionChanged(ctx context.Context, event UserRegionChangedEvent) error { + return r.withEvent(ctx, SourceUser, event.EventID, "UserRegionChanged", func(tx *sql.Tx, nowMS int64) error { + countryID, regionID := normalizeDimension(event.CountryID, event.RegionID) + return upsertUserDimension(ctx, tx, event.AppCode, event.UserID, countryID, regionID, nowMS) + }) +} + func (r *Repository) ConsumeRecharge(ctx context.Context, event RechargeEvent) error { return r.withEvent(ctx, SourceWallet, event.EventID, event.EventType, func(tx *sql.Tx, nowMS int64) error { countryID, regionID := normalizeDimension(event.TargetCountryID, event.TargetRegionID) @@ -700,9 +793,10 @@ func (r *Repository) ConsumeCoinSellerStock(ctx context.Context, event CoinSelle UPDATE stat_app_day_country SET recharge_usd_minor = recharge_usd_minor + ?, coin_seller_recharge_usd_minor = coin_seller_recharge_usd_minor + ?, + coin_seller_stock_coin = coin_seller_stock_coin + ?, updated_at_ms = ? WHERE app_code = ? AND stat_tz = ? AND stat_day = ? AND country_id = ? AND region_id = ? - `, event.USDMinor, event.USDMinor, nowMS, appcode.Normalize(event.AppCode), scope.tz, scope.day, countryID, regionID); err != nil { + `, event.USDMinor, event.USDMinor, event.CoinAmount, nowMS, appcode.Normalize(event.AppCode), scope.tz, scope.day, countryID, regionID); err != nil { return err } } @@ -823,9 +917,146 @@ func (r *Repository) ConsumeLuckyGiftReward(ctx context.Context, event LuckyGift } if _, err := tx.ExecContext(ctx, ` UPDATE stat_app_day_country - SET lucky_gift_payout = lucky_gift_payout + ?, updated_at_ms = ? + SET lucky_gift_payout = lucky_gift_payout + ?, + platform_grant_coin = platform_grant_coin + ?, + updated_at_ms = ? WHERE app_code = ? AND stat_tz = ? AND stat_day = ? AND country_id = ? AND region_id = ? - `, event.Amount, nowMS, appcode.Normalize(event.AppCode), scope.tz, scope.day, countryID, regionID); err != nil { + `, event.Amount, event.Amount, nowMS, appcode.Normalize(event.AppCode), scope.tz, scope.day, countryID, regionID); err != nil { + return err + } + } + return nil + }) +} + +func (r *Repository) ConsumePlatformGrantCoin(ctx context.Context, event PlatformGrantCoinEvent) error { + eventType := strings.TrimSpace(event.EventType) + if eventType == "" { + eventType = "WalletPlatformGrantCoinCredited" + } + return r.withEvent(ctx, SourceWallet, event.EventID, eventType, func(tx *sql.Tx, nowMS int64) error { + countryID, regionID, err := r.dimensionForUser(ctx, tx, event.AppCode, event.UserID, event.CountryID, event.RegionID) + if err != nil { + return err + } + amount := normalizeID(event.Amount) + if amount == 0 { + return nil + } + for _, scope := range statDayScopesFromContext(ctx, event.OccurredAtMS) { + if err := ensureAppDay(ctx, tx, event.AppCode, scope.tz, scope.day, countryID, regionID, nowMS); err != nil { + return err + } + if _, err := tx.ExecContext(ctx, ` + UPDATE stat_app_day_country + SET platform_grant_coin = platform_grant_coin + ?, updated_at_ms = ? + WHERE app_code = ? AND stat_tz = ? AND stat_day = ? AND country_id = ? AND region_id = ? + `, amount, nowMS, appcode.Normalize(event.AppCode), scope.tz, scope.day, countryID, regionID); err != nil { + return err + } + } + return nil + }) +} + +func (r *Repository) ConsumeWalletBalanceChanged(ctx context.Context, event WalletBalanceChangedEvent) error { + return r.withEvent(ctx, SourceWallet, event.EventID, "WalletBalanceChanged", func(tx *sql.Tx, nowMS int64) error { + dimensionUserID := event.UserID + if event.DimensionUserID > 0 { + dimensionUserID = event.DimensionUserID + } + countryID, regionID, err := r.dimensionForUser(ctx, tx, event.AppCode, dimensionUserID, event.CountryID, event.RegionID) + if err != nil { + return err + } + totalDelta := event.AvailableDelta + event.FrozenDelta + coinTotal, consumed, manualGrant, salaryUSD, salaryTransferCoin := int64(0), int64(0), int64(0), int64(0), int64(0) + assetType := strings.ToUpper(strings.TrimSpace(event.AssetType)) + bizType := strings.ToLower(strings.TrimSpace(event.BizType)) + if assetType == "COIN" { + coinTotal = totalDelta + if totalDelta < 0 && isConsumedCoinBizType(bizType) { + consumed = -totalDelta + } + if totalDelta > 0 && bizType == "manual_credit" { + manualGrant = totalDelta + } + } + if totalDelta > 0 && bizType == "host_salary_settlement" && (assetType == "HOST_SALARY_USD" || assetType == "AGENCY_SALARY_USD") { + salaryUSD = totalDelta + } + if totalDelta > 0 && bizType == "salary_transfer_to_coin_seller" && assetType == "COIN_SELLER_COIN" { + salaryTransferCoin = totalDelta + } + if coinTotal == 0 && consumed == 0 && manualGrant == 0 && salaryUSD == 0 && salaryTransferCoin == 0 { + return nil + } + for _, scope := range statDayScopesFromContext(ctx, event.OccurredAtMS) { + if err := ensureAppDay(ctx, tx, event.AppCode, scope.tz, scope.day, countryID, regionID, nowMS); err != nil { + return err + } + if _, err := tx.ExecContext(ctx, ` + UPDATE stat_app_day_country + SET coin_total = coin_total + ?, + consumed_coin = consumed_coin + ?, + manual_grant_coin = manual_grant_coin + ?, + salary_usd_minor = salary_usd_minor + ?, + salary_transfer_coin = salary_transfer_coin + ?, + updated_at_ms = ? + WHERE app_code = ? AND stat_tz = ? AND stat_day = ? AND country_id = ? AND region_id = ? + `, coinTotal, consumed, manualGrant, salaryUSD, salaryTransferCoin, nowMS, appcode.Normalize(event.AppCode), scope.tz, scope.day, countryID, regionID); err != nil { + return err + } + } + return nil + }) +} + +func (r *Repository) ConsumeUserMicDailyStatsUpdated(ctx context.Context, event UserMicDailyStatsUpdatedEvent) error { + return r.withEvent(ctx, SourceUser, event.EventID, "UserMicDailyStatsUpdated", func(tx *sql.Tx, nowMS int64) error { + if event.MicOnlineDeltaMS == 0 || strings.TrimSpace(event.StatDate) == "" { + return nil + } + countryID, regionID, err := r.dimensionForUser(ctx, tx, event.AppCode, event.UserID, 0, 0) + if err != nil { + return err + } + for _, scope := range statDateScopesFromContext(ctx, event.StatDate) { + if err := ensureAppDay(ctx, tx, event.AppCode, scope.tz, scope.day, countryID, regionID, nowMS); err != nil { + return err + } + before, err := micOnlineBefore(ctx, tx, event.AppCode, scope.tz, scope.day, event.UserID) + if err != nil { + return err + } + after := before + event.MicOnlineDeltaMS + if after < 0 { + after = 0 + } + userDelta := int64(0) + if before <= 0 && after > 0 { + userDelta = 1 + } else if before > 0 && after <= 0 { + userDelta = -1 + } + if _, err := tx.ExecContext(ctx, ` + INSERT INTO stat_user_day_mic (app_code, stat_tz, stat_day, country_id, region_id, user_id, mic_online_ms, updated_at_ms) + VALUES (?, ?, ?, ?, ?, ?, ?, ?) + ON DUPLICATE KEY UPDATE + country_id = VALUES(country_id), + region_id = VALUES(region_id), + mic_online_ms = VALUES(mic_online_ms), + updated_at_ms = VALUES(updated_at_ms) + `, appcode.Normalize(event.AppCode), scope.tz, scope.day, countryID, regionID, event.UserID, after, nowMS); err != nil { + return err + } + if _, err := tx.ExecContext(ctx, ` + UPDATE stat_app_day_country + SET mic_online_ms = mic_online_ms + ?, + mic_online_users = GREATEST(mic_online_users + ?, 0), + updated_at_ms = ? + WHERE app_code = ? AND stat_tz = ? AND stat_day = ? AND country_id = ? AND region_id = ? + `, after-before, userDelta, nowMS, appcode.Normalize(event.AppCode), scope.tz, scope.day, countryID, regionID); err != nil { return err } } @@ -1276,6 +1507,79 @@ func applyActive(ctx context.Context, tx *sql.Tx, app string, statTZ string, day return err } +func upsertUserDimension(ctx context.Context, tx *sql.Tx, app string, userID int64, countryID int64, regionID int64, nowMS int64) error { + if userID <= 0 { + return nil + } + _, err := tx.ExecContext(ctx, ` + INSERT INTO stat_user_dimension (app_code, user_id, country_id, region_id, updated_at_ms) + VALUES (?, ?, ?, ?, ?) + ON DUPLICATE KEY UPDATE + country_id = CASE WHEN VALUES(country_id) > 0 THEN VALUES(country_id) ELSE country_id END, + region_id = CASE WHEN VALUES(region_id) > 0 THEN VALUES(region_id) ELSE region_id END, + updated_at_ms = VALUES(updated_at_ms) + `, appcode.Normalize(app), userID, normalizeID(countryID), normalizeID(regionID), nowMS) + return err +} + +func (r *Repository) dimensionForUser(ctx context.Context, tx *sql.Tx, app string, userID int64, countryID int64, regionID int64) (int64, int64, error) { + countryID, regionID = normalizeDimension(countryID, regionID) + if (countryID > 0 && regionID > 0) || userID <= 0 { + return countryID, regionID, nil + } + var storedCountryID, storedRegionID int64 + err := tx.QueryRowContext(ctx, ` + SELECT country_id, region_id + FROM stat_user_dimension + WHERE app_code = ? AND user_id = ? + `, appcode.Normalize(app), userID).Scan(&storedCountryID, &storedRegionID) + if err != nil && err != sql.ErrNoRows { + return 0, 0, err + } + if countryID == 0 { + countryID = storedCountryID + } + if regionID == 0 { + regionID = storedRegionID + } + countryID, regionID = normalizeDimension(countryID, regionID) + return countryID, regionID, nil +} + +func isConsumedCoinBizType(value string) bool { + switch strings.ToLower(strings.TrimSpace(value)) { + case "gift_debit", "direct_gift_debit", "vip_purchase", "resource_shop_purchase", + "red_packet_create", "game_debit", "wheel_draw_debit", "cp_breakup_fee": + return true + default: + return false + } +} + +func statDateScopesFromContext(ctx context.Context, statDate string) []statDayScope { + day := strings.TrimSpace(statDate) + if ctx != nil { + if statTZ, ok := ctx.Value(statTZScopeContextKey{}).(string); ok { + return []statDayScope{{tz: normalizeStatTZ(statTZ), day: day}} + } + } + return []statDayScope{{tz: StatTZUTC, day: day}, {tz: StatTZAsiaShanghai, day: day}} +} + +func micOnlineBefore(ctx context.Context, tx *sql.Tx, app string, statTZ string, day string, userID int64) (int64, error) { + var before int64 + err := tx.QueryRowContext(ctx, ` + SELECT mic_online_ms + FROM stat_user_day_mic + WHERE app_code = ? AND stat_tz = ? AND stat_day = ? AND user_id = ? + FOR UPDATE + `, appcode.Normalize(app), normalizeStatTZ(statTZ), day, userID).Scan(&before) + if err == sql.ErrNoRows { + return 0, nil + } + return before, err +} + func ensureAppDay(ctx context.Context, tx *sql.Tx, app string, statTZ string, day string, countryID int64, regionID int64, nowMS int64) error { _, err := tx.ExecContext(ctx, ` INSERT INTO stat_app_day_country (app_code, stat_tz, stat_day, country_id, region_id, updated_at_ms) diff --git a/services/user-service/configs/config.yaml b/services/user-service/configs/config.yaml index 68261c82..65dd0f6c 100644 --- a/services/user-service/configs/config.yaml +++ b/services/user-service/configs/config.yaml @@ -19,6 +19,7 @@ display_user_id: third_party: allowed_providers: - "firebase" + - "wechat" firebase: project_id: "lalu-party" allowed_sign_in_providers: diff --git a/services/wallet-service/internal/service/wallet/service_test.go b/services/wallet-service/internal/service/wallet/service_test.go index f99df4c0..3480fdb8 100644 --- a/services/wallet-service/internal/service/wallet/service_test.go +++ b/services/wallet-service/internal/service/wallet/service_test.go @@ -2250,6 +2250,9 @@ func TestTransferSalaryToCoinSellerUsesRegionRateTier(t *testing.T) { if got := repository.CountRows("wallet_recharge_records", "transaction_id = ?", first.TransactionID); got != 0 { t.Fatalf("salary transfer must not write player recharge record, got %d", got) } + if got := repository.CountRows("wallet_outbox", "transaction_id = ? AND event_type = ? AND JSON_UNQUOTE(JSON_EXTRACT(payload_json, '$.biz_type')) = ?", first.TransactionID, "WalletBalanceChanged", "salary_transfer_to_coin_seller"); got != 2 { + t.Fatalf("salary transfer balance events must expose biz_type for statistics, got %d", got) + } } // TestTransferHostSalaryToCoinSellerCreditsTargetSellerOnly 验证别人主播把工资转给币商时， diff --git a/services/wallet-service/internal/storage/mysql/admin_ledger.go b/services/wallet-service/internal/storage/mysql/admin_ledger.go index 4aea468a..c9ba7c5d 100644 --- a/services/wallet-service/internal/storage/mysql/admin_ledger.go +++ b/services/wallet-service/internal/storage/mysql/admin_ledger.go @@ -81,7 +81,7 @@ func (r *Repository) AdminCreditAsset(ctx context.Context, command ledger.AdminC return ledger.AssetBalance{}, "", err } if err := r.insertWalletOutbox(ctx, tx, []walletOutboxEvent{ - balanceChangedEvent(transactionID, command.CommandID, command.TargetUserID, command.AssetType, command.Amount, 0, balance.AvailableAmount, balance.FrozenAmount, balance.Version, metadata, nowMs), + balanceChangedEvent(transactionID, command.CommandID, command.TargetUserID, command.AssetType, command.Amount, 0, balance.AvailableAmount, balance.FrozenAmount, balance.Version, metadata, nowMs, bizTypeManualCredit), }); err != nil { return ledger.AssetBalance{}, "", err } diff --git a/services/wallet-service/internal/storage/mysql/coin_seller_ledger.go b/services/wallet-service/internal/storage/mysql/coin_seller_ledger.go index afb3b358..db887c82 100644 --- a/services/wallet-service/internal/storage/mysql/coin_seller_ledger.go +++ b/services/wallet-service/internal/storage/mysql/coin_seller_ledger.go @@ -481,8 +481,8 @@ func (r *Repository) TransferSalaryToCoinSeller(ctx context.Context, command led return ledger.SalaryTransferToCoinSellerReceipt{}, err } if err := r.insertWalletOutbox(ctx, tx, []walletOutboxEvent{ - balanceChangedEvent(transactionID, command.CommandID, command.SourceUserID, command.SalaryAssetType, -command.SalaryUSDMinor, 0, sourceAfter, source.FrozenAmount, source.Version+1, metadata, nowMs), - balanceChangedEvent(transactionID, command.CommandID, command.SellerUserID, ledger.AssetCoinSellerCoin, coinAmount, 0, sellerAfter, seller.FrozenAmount, seller.Version+1, metadata, nowMs), + balanceChangedEvent(transactionID, command.CommandID, command.SourceUserID, command.SalaryAssetType, -command.SalaryUSDMinor, 0, sourceAfter, source.FrozenAmount, source.Version+1, metadata, nowMs, bizTypeSalaryTransferToCoinSeller), + balanceChangedEvent(transactionID, command.CommandID, command.SellerUserID, ledger.AssetCoinSellerCoin, coinAmount, 0, sellerAfter, seller.FrozenAmount, seller.Version+1, metadata, nowMs, bizTypeSalaryTransferToCoinSeller), }); err != nil { return ledger.SalaryTransferToCoinSellerReceipt{}, err } diff --git a/services/wallet-service/internal/storage/mysql/game_ledger.go b/services/wallet-service/internal/storage/mysql/game_ledger.go index d2f56ac5..ada6d40f 100644 --- a/services/wallet-service/internal/storage/mysql/game_ledger.go +++ b/services/wallet-service/internal/storage/mysql/game_ledger.go @@ -84,7 +84,7 @@ func (r *Repository) ApplyGameCoinChange(ctx context.Context, command ledger.Gam return ledger.GameCoinChangeReceipt{}, err } if err := r.insertWalletOutbox(ctx, tx, []walletOutboxEvent{ - balanceChangedEvent(transactionID, command.CommandID, command.UserID, ledger.AssetCoin, delta, 0, balanceAfter, account.FrozenAmount, account.Version+1, metadata, nowMs), + balanceChangedEvent(transactionID, command.CommandID, command.UserID, ledger.AssetCoin, delta, 0, balanceAfter, account.FrozenAmount, account.Version+1, metadata, nowMs, bizType), }); err != nil { return ledger.GameCoinChangeReceipt{}, err } diff --git a/services/wallet-service/internal/storage/mysql/gift_batch_debit_repository.go b/services/wallet-service/internal/storage/mysql/gift_batch_debit_repository.go index 261c8002..1dbd77e1 100644 --- a/services/wallet-service/internal/storage/mysql/gift_batch_debit_repository.go +++ b/services/wallet-service/internal/storage/mysql/gift_batch_debit_repository.go @@ -320,13 +320,13 @@ func (r *Repository) BatchDebitGift(ctx context.Context, command ledger.BatchDeb if chargeSource != giftChargeSourceBag { if target.TargetUserID == command.SenderUserID && chargeAssetType == ledger.AssetCoin { - events = append(events, balanceChangedEvent(transactionID, target.CommandID, command.SenderUserID, ledger.AssetCoin, giftIncomeCoinAmount-chargeAmount, 0, metadata.BalanceAfter, senderState.account.FrozenAmount, senderState.account.Version, metadata, nowMs)) + events = append(events, balanceChangedEvent(transactionID, target.CommandID, command.SenderUserID, ledger.AssetCoin, giftIncomeCoinAmount-chargeAmount, 0, metadata.BalanceAfter, senderState.account.FrozenAmount, senderState.account.Version, metadata, nowMs, bizTypeGiftDebit)) } else { - events = append(events, balanceChangedEvent(transactionID, target.CommandID, command.SenderUserID, price.ChargeAssetType, -chargeAmount, 0, senderState.account.AvailableAmount, senderState.account.FrozenAmount, senderState.account.Version, metadata, nowMs)) + events = append(events, balanceChangedEvent(transactionID, target.CommandID, command.SenderUserID, price.ChargeAssetType, -chargeAmount, 0, senderState.account.AvailableAmount, senderState.account.FrozenAmount, senderState.account.Version, metadata, nowMs, bizTypeGiftDebit)) } } if giftIncomeCoinAmount > 0 && !(target.TargetUserID == command.SenderUserID && chargeAssetType == ledger.AssetCoin && chargeSource != giftChargeSourceBag) { - events = append(events, balanceChangedEvent(transactionID, target.CommandID, target.TargetUserID, ledger.AssetCoin, giftIncomeCoinAmount, 0, metadata.GiftIncomeBalanceAfter, targetCoinState.account.FrozenAmount, targetCoinState.account.Version, metadata, nowMs)) + events = append(events, balanceChangedEvent(transactionID, target.CommandID, target.TargetUserID, ledger.AssetCoin, giftIncomeCoinAmount, 0, metadata.GiftIncomeBalanceAfter, targetCoinState.account.FrozenAmount, targetCoinState.account.Version, metadata, nowMs, bizTypeGiftDebit)) } events = append(events, giftDebitedEvent(transactionID, target.CommandID, command.SenderUserID, chargeAssetType, -chargeAmount, 0, metadata, nowMs)) if giftIncomeCoinAmount > 0 { diff --git a/services/wallet-service/internal/storage/mysql/gift_debit_repository.go b/services/wallet-service/internal/storage/mysql/gift_debit_repository.go index f93d5555..9b983764 100644 --- a/services/wallet-service/internal/storage/mysql/gift_debit_repository.go +++ b/services/wallet-service/internal/storage/mysql/gift_debit_repository.go @@ -314,13 +314,13 @@ func (r *Repository) DebitGift(ctx context.Context, command ledger.DebitGiftComm if chargeSource != giftChargeSourceBag { if command.TargetUserID == command.SenderUserID && chargeAssetType == ledger.AssetCoin { - events = append(events, balanceChangedEvent(transactionID, command.CommandID, command.SenderUserID, ledger.AssetCoin, giftIncomeCoinAmount-chargeAmount, 0, metadata.BalanceAfter, senderState.account.FrozenAmount, senderState.account.Version, metadata, nowMs)) + events = append(events, balanceChangedEvent(transactionID, command.CommandID, command.SenderUserID, ledger.AssetCoin, giftIncomeCoinAmount-chargeAmount, 0, metadata.BalanceAfter, senderState.account.FrozenAmount, senderState.account.Version, metadata, nowMs, bizType)) } else { - events = append(events, balanceChangedEvent(transactionID, command.CommandID, command.SenderUserID, chargeAssetType, -chargeAmount, 0, senderState.account.AvailableAmount, senderState.account.FrozenAmount, senderState.account.Version, metadata, nowMs)) + events = append(events, balanceChangedEvent(transactionID, command.CommandID, command.SenderUserID, chargeAssetType, -chargeAmount, 0, senderState.account.AvailableAmount, senderState.account.FrozenAmount, senderState.account.Version, metadata, nowMs, bizType)) } } if giftIncomeCoinAmount > 0 && !(command.TargetUserID == command.SenderUserID && chargeAssetType == ledger.AssetCoin && chargeSource != giftChargeSourceBag) { - events = append(events, balanceChangedEvent(transactionID, command.CommandID, command.TargetUserID, ledger.AssetCoin, giftIncomeCoinAmount, 0, metadata.GiftIncomeBalanceAfter, targetCoinState.account.FrozenAmount, targetCoinState.account.Version, metadata, nowMs)) + events = append(events, balanceChangedEvent(transactionID, command.CommandID, command.TargetUserID, ledger.AssetCoin, giftIncomeCoinAmount, 0, metadata.GiftIncomeBalanceAfter, targetCoinState.account.FrozenAmount, targetCoinState.account.Version, metadata, nowMs, bizType)) } events = append(events, giftDebitedEvent(transactionID, command.CommandID, command.SenderUserID, chargeAssetType, -chargeAmount, 0, metadata, nowMs)) if giftIncomeCoinAmount > 0 { diff --git a/services/wallet-service/internal/storage/mysql/host_salary_settlement.go b/services/wallet-service/internal/storage/mysql/host_salary_settlement.go index 071955b3..0c05d8b4 100644 --- a/services/wallet-service/internal/storage/mysql/host_salary_settlement.go +++ b/services/wallet-service/internal/storage/mysql/host_salary_settlement.go @@ -511,7 +511,7 @@ func (r *Repository) applyHostSalaryCredits(ctx context.Context, tx *sql.Tx, tra return nil, err } metadata.HostUSDBalanceAfter = balanceAfter - events = append(events, balanceChangedEvent(transactionID, commandID, metadata.HostUserID, ledger.AssetHostSalaryUSD, hostUSDDelta, 0, balanceAfter, 0, version, metadata, nowMs)) + events = append(events, balanceChangedEvent(transactionID, commandID, metadata.HostUserID, ledger.AssetHostSalaryUSD, hostUSDDelta, 0, balanceAfter, 0, version, metadata, nowMs, bizTypeHostSalarySettlement)) } if hostCoinDelta > 0 { balanceAfter, version, err := r.creditSettlementAsset(ctx, tx, transactionID, metadata.HostUserID, ledger.AssetCoin, hostCoinDelta, 0, nowMs) @@ -519,7 +519,7 @@ func (r *Repository) applyHostSalaryCredits(ctx context.Context, tx *sql.Tx, tra return nil, err } metadata.HostCoinBalanceAfter = balanceAfter - events = append(events, balanceChangedEvent(transactionID, commandID, metadata.HostUserID, ledger.AssetCoin, hostCoinDelta, 0, balanceAfter, 0, version, metadata, nowMs)) + events = append(events, balanceChangedEvent(transactionID, commandID, metadata.HostUserID, ledger.AssetCoin, hostCoinDelta, 0, balanceAfter, 0, version, metadata, nowMs, bizTypeHostSalarySettlement)) } if agencyUSDDelta > 0 { // 代理工资进入代理工资钱包，同一用户若同时是主播也不会和主播工资钱包合并。 @@ -528,7 +528,7 @@ func (r *Repository) applyHostSalaryCredits(ctx context.Context, tx *sql.Tx, tra return nil, err } metadata.AgencyUSDBalanceAfter = balanceAfter - events = append(events, balanceChangedEvent(transactionID, commandID, metadata.AgencyOwnerUserID, ledger.AssetAgencySalaryUSD, agencyUSDDelta, 0, balanceAfter, 0, version, metadata, nowMs)) + events = append(events, balanceChangedEvent(transactionID, commandID, metadata.AgencyOwnerUserID, ledger.AssetAgencySalaryUSD, agencyUSDDelta, 0, balanceAfter, 0, version, metadata, nowMs, bizTypeHostSalarySettlement)) } return events, nil } diff --git a/services/wallet-service/internal/storage/mysql/outbox.go b/services/wallet-service/internal/storage/mysql/outbox.go index 7caaeb24..1318f3bb 100644 --- a/services/wallet-service/internal/storage/mysql/outbox.go +++ b/services/wallet-service/internal/storage/mysql/outbox.go @@ -345,7 +345,11 @@ func trimWalletOutboxError(value string) string { return value } -func balanceChangedEvent(transactionID string, commandID string, userID int64, assetType string, availableDelta int64, frozenDelta int64, availableAfter int64, frozenAfter int64, balanceVersion int64, payload any, nowMs int64) walletOutboxEvent { +func balanceChangedEvent(transactionID string, commandID string, userID int64, assetType string, availableDelta int64, frozenDelta int64, availableAfter int64, frozenAfter int64, balanceVersion int64, payload any, nowMs int64, bizTypes ...string) walletOutboxEvent { + bizType := "" + if len(bizTypes) > 0 { + bizType = strings.TrimSpace(bizTypes[0]) + } return walletOutboxEvent{ EventID: eventID(transactionID, "WalletBalanceChanged", userID, assetType), EventType: "WalletBalanceChanged", @@ -360,6 +364,7 @@ func balanceChangedEvent(transactionID string, commandID string, userID int64, a "command_id": commandID, "user_id": userID, "asset_type": assetType, + "biz_type": bizType, "available_delta": availableDelta, "frozen_delta": frozenDelta, "available_after": availableAfter, diff --git a/services/wallet-service/internal/storage/mysql/red_packet_create_repository.go b/services/wallet-service/internal/storage/mysql/red_packet_create_repository.go index f42c8197..0eeec102 100644 --- a/services/wallet-service/internal/storage/mysql/red_packet_create_repository.go +++ b/services/wallet-service/internal/storage/mysql/red_packet_create_repository.go @@ -127,7 +127,7 @@ func (r *Repository) CreateRedPacket(ctx context.Context, command ledger.RedPack return ledger.RedPacketCreateReceipt{}, err } if err := r.insertWalletOutbox(ctx, tx, []walletOutboxEvent{ - balanceChangedEvent(transactionID, command.CommandID, command.SenderUserID, ledger.AssetCoin, -command.TotalAmount, 0, balanceAfter, sender.FrozenAmount, sender.Version+1, metadata, nowMS), + balanceChangedEvent(transactionID, command.CommandID, command.SenderUserID, ledger.AssetCoin, -command.TotalAmount, 0, balanceAfter, sender.FrozenAmount, sender.Version+1, metadata, nowMS, bizTypeRedPacketCreate), redPacketCreatedEvent(transactionID, command.CommandID, metadata, nowMS), }); err != nil { return ledger.RedPacketCreateReceipt{}, err diff --git a/services/wallet-service/internal/storage/mysql/resource_shop_repository.go b/services/wallet-service/internal/storage/mysql/resource_shop_repository.go index 42706a65..46d8ac12 100644 --- a/services/wallet-service/internal/storage/mysql/resource_shop_repository.go +++ b/services/wallet-service/internal/storage/mysql/resource_shop_repository.go @@ -300,7 +300,7 @@ func (r *Repository) PurchaseResourceShopItem(ctx context.Context, command resou return resourcedomain.ResourceShopPurchaseReceipt{}, err } if err := r.insertWalletOutbox(ctx, tx, []walletOutboxEvent{ - balanceChangedEvent(transactionID, command.CommandID, command.UserID, ledger.AssetCoin, -item.CoinPrice, 0, coinBalanceAfter, account.FrozenAmount, account.Version+1, metadata, nowMs), + balanceChangedEvent(transactionID, command.CommandID, command.UserID, ledger.AssetCoin, -item.CoinPrice, 0, coinBalanceAfter, account.FrozenAmount, account.Version+1, metadata, nowMs, bizTypeResourceShopPurchase), resourceOutboxEvent("ResourceGranted", command.CommandID, command.UserID, item.ResourceID, map[string]any{"grant_id": grantID, "resource": item.Resource, "source": resourcedomain.GrantSourceResourceShop}, nowMs), { EventID: eventID(transactionID, "ResourceShopItemPurchased", command.UserID, resourceOutboxAsset), diff --git a/services/wallet-service/internal/storage/mysql/reward_ledger.go b/services/wallet-service/internal/storage/mysql/reward_ledger.go index 181a00a6..f109b133 100644 --- a/services/wallet-service/internal/storage/mysql/reward_ledger.go +++ b/services/wallet-service/internal/storage/mysql/reward_ledger.go @@ -88,7 +88,7 @@ func (r *Repository) DebitCPBreakupFee(ctx context.Context, command ledger.CPBre return ledger.CPBreakupFeeReceipt{}, err } if err := r.insertWalletOutbox(ctx, tx, []walletOutboxEvent{ - balanceChangedEvent(transactionID, command.CommandID, command.UserID, ledger.AssetCoin, -command.Amount, 0, balance.AvailableAmount, balance.FrozenAmount, balance.Version, metadata, nowMs), + balanceChangedEvent(transactionID, command.CommandID, command.UserID, ledger.AssetCoin, -command.Amount, 0, balance.AvailableAmount, balance.FrozenAmount, balance.Version, metadata, nowMs, bizTypeCPBreakupFee), }); err != nil { return ledger.CPBreakupFeeReceipt{}, err } @@ -180,7 +180,7 @@ func (r *Repository) DebitWheelDraw(ctx context.Context, command ledger.WheelDra return ledger.WheelDrawDebitReceipt{}, err } if err := r.insertWalletOutbox(ctx, tx, []walletOutboxEvent{ - balanceChangedEvent(transactionID, command.CommandID, command.UserID, ledger.AssetCoin, -command.Amount, 0, balance.AvailableAmount, balance.FrozenAmount, balance.Version, metadata, nowMs), + balanceChangedEvent(transactionID, command.CommandID, command.UserID, ledger.AssetCoin, -command.Amount, 0, balance.AvailableAmount, balance.FrozenAmount, balance.Version, metadata, nowMs, bizTypeWheelDrawDebit), }); err != nil { return ledger.WheelDrawDebitReceipt{}, err } diff --git a/services/wallet-service/internal/storage/mysql/vip_purchase_repository.go b/services/wallet-service/internal/storage/mysql/vip_purchase_repository.go index 591542f2..b158ca18 100644 --- a/services/wallet-service/internal/storage/mysql/vip_purchase_repository.go +++ b/services/wallet-service/internal/storage/mysql/vip_purchase_repository.go @@ -142,7 +142,7 @@ func (r *Repository) PurchaseVip(ctx context.Context, command ledger.PurchaseVip vipEventType = "VipUpgraded" } if err := r.insertWalletOutbox(ctx, tx, []walletOutboxEvent{ - balanceChangedEvent(transactionID, command.CommandID, command.UserID, ledger.AssetCoin, -level.PriceCoin, 0, coinBalanceAfter, account.FrozenAmount, account.Version+1, metadata, nowMs), + balanceChangedEvent(transactionID, command.CommandID, command.UserID, ledger.AssetCoin, -level.PriceCoin, 0, coinBalanceAfter, account.FrozenAmount, account.Version+1, metadata, nowMs, bizTypeVIPPurchase), { EventID: eventID(transactionID, vipEventType, command.UserID, "VIP"), EventType: vipEventType, diff --git a/tools/databi-real-flow/main.go b/tools/databi-real-flow/main.go new file mode 100644 index 00000000..aa1a4fb3 --- /dev/null +++ b/tools/databi-real-flow/main.go @@ -0,0 +1,1059 @@ +package main + +import ( + "context" + "database/sql" + "encoding/json" + "flag" + "fmt" + "log" + "os" + "strings" + "time" + + gamev1 "hyapp.local/api/proto/game/v1" + roomv1 "hyapp.local/api/proto/room/v1" + userv1 "hyapp.local/api/proto/user/v1" + walletv1 "hyapp.local/api/proto/wallet/v1" + + _ "github.com/go-sql-driver/mysql" + "google.golang.org/grpc" + "google.golang.org/grpc/credentials/insecure" +) + +const ( + defaultAppCode = "lalu" + countryCode = "US" + regionID = int64(3) + gameID = "dice" +) + +type config struct { + appCode string + runID string + userDSN string + walletDSN string + userAddr string + walletAddr string + roomAddr string + gameAddr string + countryID int64 + roomID string + admin account + sender account + host account + seller account + gamePlayer account + gameRival account + normalGift string + luckyGift string + superGift string + luckyPool string + superPool string + normalPrice int64 + luckyPrice int64 + superPrice int64 +} + +type account struct { + userID int64 + displayUserID string + username string +} + +type runner struct { + cfg config + userDB *sql.DB + walletDB *sql.DB + authClient userv1.AuthServiceClient + hostClient userv1.UserHostAdminServiceClient + walletClient walletv1.WalletServiceClient + cronClient walletv1.WalletCronServiceClient + roomClient roomv1.RoomCommandServiceClient + gameAdmin gamev1.GameAdminServiceClient + gameApp gamev1.GameAppServiceClient +} + +func main() { + var cfg config + flag.StringVar(&cfg.appCode, "app", envDefault("APP_CODE", defaultAppCode), "app code") + flag.StringVar(&cfg.runID, "run-id", envDefault("RUN_ID", ""), "stable run id") + flag.StringVar(&cfg.userDSN, "user-dsn", envDefault("USER_MYSQL_DSN", "hyapp:hyapp@tcp(127.0.0.1:23306)/hyapp_user?parseTime=true&charset=utf8mb4&loc=UTC&multiStatements=true"), "user mysql dsn") + flag.StringVar(&cfg.walletDSN, "wallet-dsn", envDefault("WALLET_MYSQL_DSN", "hyapp:hyapp@tcp(127.0.0.1:23306)/hyapp_wallet?parseTime=true&charset=utf8mb4&loc=UTC&multiStatements=true"), "wallet mysql dsn") + flag.StringVar(&cfg.userAddr, "user-addr", envDefault("USER_GRPC_ADDR", "127.0.0.1:13005"), "user-service grpc address") + flag.StringVar(&cfg.walletAddr, "wallet-addr", envDefault("WALLET_GRPC_ADDR", "127.0.0.1:13004"), "wallet-service grpc address") + flag.StringVar(&cfg.roomAddr, "room-addr", envDefault("ROOM_GRPC_ADDR", "127.0.0.1:13001"), "room-service grpc address") + flag.StringVar(&cfg.gameAddr, "game-addr", envDefault("GAME_GRPC_ADDR", "127.0.0.1:13008"), "game-service grpc address") + flag.Parse() + + if strings.TrimSpace(cfg.runID) == "" { + cfg.runID = fmt.Sprintf("%d", time.Now().UnixNano()) + } + cfg.runID = sanitizeID(cfg.runID) + cfg.roomID = shortID("dbroom", cfg.runID, 44) + cfg.normalGift = shortID("dbg_normal", cfg.runID, 60) + cfg.luckyGift = shortID("dbg_lucky", cfg.runID, 60) + cfg.superGift = shortID("dbg_super", cfg.runID, 60) + cfg.luckyPool = "lucky" + cfg.superPool = "super_lucky" + cfg.normalPrice = 200 + cfg.luckyPrice = 120 + cfg.superPrice = 160 + + ctx, cancel := context.WithTimeout(context.Background(), 8*time.Minute) + defer cancel() + + userDB, err := openDB(ctx, cfg.userDSN) + if err != nil { + log.Fatalf("open user db: %v", err) + } + defer userDB.Close() + walletDB, err := openDB(ctx, cfg.walletDSN) + if err != nil { + log.Fatalf("open wallet db: %v", err) + } + defer walletDB.Close() + + userConn := mustDial(ctx, cfg.userAddr) + defer userConn.Close() + walletConn := mustDial(ctx, cfg.walletAddr) + defer walletConn.Close() + roomConn := mustDial(ctx, cfg.roomAddr) + defer roomConn.Close() + gameConn := mustDial(ctx, cfg.gameAddr) + defer gameConn.Close() + + r := &runner{ + cfg: cfg, + userDB: userDB, + walletDB: walletDB, + authClient: userv1.NewAuthServiceClient(userConn), + hostClient: userv1.NewUserHostAdminServiceClient(userConn), + walletClient: walletv1.NewWalletServiceClient(walletConn), + cronClient: walletv1.NewWalletCronServiceClient(walletConn), + roomClient: roomv1.NewRoomCommandServiceClient(roomConn), + gameAdmin: gamev1.NewGameAdminServiceClient(gameConn), + gameApp: gamev1.NewGameAppServiceClient(gameConn), + } + if err := r.run(ctx); err != nil { + log.Fatalf("databi real flow failed: %v", err) + } +} + +func (r *runner) run(ctx context.Context) error { + if err := r.ensureUserMasterData(ctx); err != nil { + return err + } + if err := r.ensureSalaryConfig(ctx); err != nil { + return err + } + if err := r.ensureGameConfig(ctx); err != nil { + return err + } + if err := r.createAccounts(ctx); err != nil { + return err + } + if err := r.createBusinessRoles(ctx); err != nil { + return err + } + if err := r.prepareWalletAndGifts(ctx); err != nil { + return err + } + if err := r.simulateRechargeEvents(ctx); err != nil { + return err + } + if err := r.runRoomFlow(ctx); err != nil { + return err + } + if err := r.runSalaryFlow(ctx); err != nil { + return err + } + if err := r.runGameFlow(ctx); err != nil { + return err + } + if err := r.waitForAsyncSources(ctx); err != nil { + return err + } + fmt.Printf("databi real business flow completed run_id=%s app=%s country_id=%d room_id=%s\n", r.cfg.runID, r.cfg.appCode, r.cfg.countryID, r.cfg.roomID) + return nil +} + +func (r *runner) ensureUserMasterData(ctx context.Context) error { + now := time.Now().UnixMilli() + _, err := r.userDB.ExecContext(ctx, ` + INSERT INTO countries ( + app_code, country_name, country_code, iso_alpha3, iso_numeric, country_display_name, + phone_country_code, flag, enabled, sort_order, created_by_user_id, updated_by_user_id, created_at_ms, updated_at_ms + ) VALUES (?, 'United States', ?, 'USA', '840', 'United States', '+1', 'US', 1, 900, 0, 0, ?, ?) + ON DUPLICATE KEY UPDATE + country_name = VALUES(country_name), + country_display_name = VALUES(country_display_name), + enabled = 1, + updated_at_ms = VALUES(updated_at_ms)`, + r.cfg.appCode, countryCode, now, now) + if err != nil { + return fmt.Errorf("seed country: %w", err) + } + if err := r.userDB.QueryRowContext(ctx, + `SELECT country_id FROM countries WHERE app_code = ? AND country_code = ?`, + r.cfg.appCode, countryCode, + ).Scan(&r.cfg.countryID); err != nil { + return fmt.Errorf("read country id: %w", err) + } + _, err = r.userDB.ExecContext(ctx, ` + INSERT INTO region_countries ( + app_code, region_id, country_code, status, created_by_user_id, updated_by_user_id, created_at_ms, updated_at_ms + ) VALUES (?, ?, ?, 'active', 0, 0, ?, ?) + ON DUPLICATE KEY UPDATE + region_id = VALUES(region_id), + status = 'active', + updated_at_ms = VALUES(updated_at_ms)`, + r.cfg.appCode, regionID, countryCode, now, now) + if err != nil { + return fmt.Errorf("seed region country: %w", err) + } + fmt.Printf("seeded user master data country=%s country_id=%d region_id=%d\n", countryCode, r.cfg.countryID, regionID) + return nil +} + +func (r *runner) ensureSalaryConfig(ctx context.Context) error { + now := time.Now().UnixMilli() + _, err := r.walletDB.ExecContext(ctx, ` + INSERT INTO host_agency_salary_policies ( + app_code, policy_id, name, region_id, status, settlement_mode, settlement_trigger_mode, + gift_coin_to_diamond_ratio, residual_diamond_to_usd_rate, effective_from_ms, effective_to_ms, created_at_ms, updated_at_ms + ) VALUES (?, 990001, 'Databi Local Daily Policy', ?, 'active', 'daily', 'automatic', 1.000000, 0.000000000000, 0, 0, ?, ?) + ON DUPLICATE KEY UPDATE + region_id = VALUES(region_id), + status = 'active', + settlement_mode = 'daily', + settlement_trigger_mode = 'automatic', + gift_coin_to_diamond_ratio = VALUES(gift_coin_to_diamond_ratio), + updated_at_ms = VALUES(updated_at_ms)`, + r.cfg.appCode, regionID, now, now) + if err != nil { + return fmt.Errorf("seed salary policy: %w", err) + } + _, err = r.walletDB.ExecContext(ctx, ` + INSERT INTO host_agency_salary_policy_levels ( + app_code, policy_id, level_no, required_diamonds, host_salary_usd_minor, host_coin_reward, + agency_salary_usd_minor, status, sort_order, created_at_ms, updated_at_ms + ) VALUES (?, 990001, 1, 100, 150, 0, 50, 'active', 1, ?, ?) + ON DUPLICATE KEY UPDATE + required_diamonds = VALUES(required_diamonds), + host_salary_usd_minor = VALUES(host_salary_usd_minor), + agency_salary_usd_minor = VALUES(agency_salary_usd_minor), + status = 'active', + updated_at_ms = VALUES(updated_at_ms)`, + r.cfg.appCode, now, now) + if err != nil { + return fmt.Errorf("seed salary policy level: %w", err) + } + _, err = r.walletDB.ExecContext(ctx, + `DELETE FROM coin_seller_salary_exchange_rate_tiers WHERE app_code = ? AND region_id = ? AND sort_order = 990001`, + r.cfg.appCode, regionID) + if err != nil { + return fmt.Errorf("clear salary exchange tier: %w", err) + } + _, err = r.walletDB.ExecContext(ctx, ` + INSERT INTO coin_seller_salary_exchange_rate_tiers ( + app_code, region_id, min_usd_minor, max_usd_minor, coin_per_usd, status, + sort_order, created_by_user_id, updated_by_user_id, created_at_ms, updated_at_ms + ) VALUES (?, ?, 1, 99999999, 1000, 'active', 990001, 0, 0, ?, ?)`, + r.cfg.appCode, regionID, now, now) + if err != nil { + return fmt.Errorf("seed salary exchange tier: %w", err) + } + fmt.Println("seeded wallet salary policy and salary-to-seller exchange tier") + return nil +} + +func (r *runner) ensureGameConfig(ctx context.Context) error { + meta := r.gameMeta("game-admin") + if _, err := r.gameAdmin.UpsertPlatform(ctx, &gamev1.UpsertPlatformRequest{ + Meta: meta, + Platform: &gamev1.GamePlatform{ + AppCode: r.cfg.appCode, + PlatformCode: "dice", + PlatformName: "Databi Dice", + Status: "active", + ApiBaseUrl: "https://local.hyapp.invalid/dice", + SortOrder: 1, + AdapterType: "yomi_v4", + CallbackSecret: "local-databi-secret", + CallbackIpWhitelist: []string{}, + AdapterConfigJson: "{}", + }, + }); err != nil { + return fmt.Errorf("upsert game platform: %w", err) + } + if _, err := r.gameAdmin.UpsertCatalog(ctx, &gamev1.UpsertCatalogRequest{ + Meta: meta, + Game: &gamev1.GameCatalogItem{ + AppCode: r.cfg.appCode, + GameId: gameID, + PlatformCode: "dice", + ProviderGameId: "dice", + GameName: "Databi Dice", + Category: "self", + IconUrl: "https://local.hyapp.invalid/dice.png", + CoverUrl: "https://local.hyapp.invalid/dice-cover.png", + LaunchMode: "h5_popup", + Orientation: "portrait", + MinCoin: 100, + Status: "active", + SortOrder: 1, + Tags: []string{"self"}, + }, + }); err != nil { + return fmt.Errorf("upsert game catalog: %w", err) + } + if _, err := r.gameAdmin.UpdateDiceConfig(ctx, &gamev1.UpdateDiceConfigRequest{ + Meta: meta, + Config: &gamev1.DiceConfig{ + AppCode: r.cfg.appCode, + GameId: gameID, + Status: "active", + StakeOptions: []*gamev1.DiceStakeOption{{ + StakeCoin: 100, + Enabled: true, + SortOrder: 1, + }}, + FeeBps: 500, + PoolBps: 100, + MinPlayers: 2, + MaxPlayers: 2, + RobotEnabled: false, + RobotMatchWaitMs: 3000, + }, + }); err != nil { + return fmt.Errorf("upsert dice config: %w", err) + } + fmt.Println("seeded game platform/catalog/dice config through game-service RPC") + return nil +} + +func (r *runner) createAccounts(ctx context.Context) error { + admin, err := r.quickCreate(ctx, "admin") + if err != nil { + return err + } + r.cfg.admin = admin + fmt.Printf("created admin user_id=%d display=%s\n", admin.userID, admin.displayUserID) + + for _, spec := range []struct { + key string + target *account + }{ + {key: "sender", target: &r.cfg.sender}, + {key: "host", target: &r.cfg.host}, + {key: "seller", target: &r.cfg.seller}, + {key: "game_player", target: &r.cfg.gamePlayer}, + {key: "game_rival", target: &r.cfg.gameRival}, + } { + acc, err := r.registerAppAccount(ctx, spec.key) + if err != nil { + return err + } + *spec.target = acc + fmt.Printf("registered app user %-11s user_id=%d display=%s\n", spec.key, acc.userID, acc.displayUserID) + } + return nil +} + +func (r *runner) quickCreate(ctx context.Context, role string) (account, error) { + username := shortID("databi_"+role, r.cfg.runID, 28) + deviceID := shortID("device_"+role, r.cfg.runID, 60) + resp, err := r.authClient.QuickCreateAccount(ctx, &userv1.QuickCreateAccountRequest{ + Meta: &userv1.RequestMeta{ + RequestId: r.commandID("quick_create_" + role), + Caller: "databi-real-flow", + SentAtMs: time.Now().UnixMilli(), + DeviceId: deviceID, + ClientIp: "127.0.0.1", + UserAgent: "databi-real-flow", + CountryByIp: countryCode, + AppCode: r.cfg.appCode, + Platform: "ios", + Language: "en", + Timezone: "Asia/Shanghai", + }, + Password: "Databi@123456", + Username: username, + Avatar: "https://local.hyapp.invalid/avatar.png", + Gender: "female", + Country: countryCode, + DeviceId: deviceID, + Device: "local", + OsVersion: "17.0", + Birth: "1995-01-01", + AppVersion: "1.0.0", + BuildNumber: "1", + Source: "databi_real_flow", + InstallChannel: "local", + Platform: "ios", + Language: "en", + Timezone: "Asia/Shanghai", + }) + if err != nil { + return account{}, fmt.Errorf("quick create %s: %w", role, err) + } + token := resp.GetToken() + if token.GetUserId() <= 0 { + return account{}, fmt.Errorf("quick create %s returned empty user id", role) + } + return account{userID: token.GetUserId(), displayUserID: token.GetDisplayUserId(), username: username}, nil +} + +func (r *runner) registerAppAccount(ctx context.Context, role string) (account, error) { + username := shortID("databi_"+role, r.cfg.runID, 28) + deviceID := shortID("device_"+role, r.cfg.runID, 60) + resp, err := r.authClient.RegisterThirdParty(ctx, &userv1.RegisterThirdPartyRequest{ + Meta: &userv1.RequestMeta{ + RequestId: r.commandID("register_third_party_" + role), + Caller: "databi-real-flow", + SentAtMs: time.Now().UnixMilli(), + DeviceId: deviceID, + ClientIp: "127.0.0.1", + UserAgent: "databi-real-flow", + CountryByIp: countryCode, + AppCode: r.cfg.appCode, + Platform: "ios", + Language: "en", + Timezone: "Asia/Shanghai", + }, + Provider: "wechat", + Credential: "openid_" + role + "_" + r.cfg.runID, + DeviceId: deviceID, + Username: username, + Avatar: "https://local.hyapp.invalid/avatar.png", + Gender: "female", + Country: countryCode, + Device: "local", + OsVersion: "17.0", + Birth: "1995-01-01", + AppVersion: "1.0.0", + BuildNumber: "1", + Source: "databi_app_registration", + InstallChannel: "local", + Platform: "ios", + Language: "en", + Timezone: "Asia/Shanghai", + }) + if err != nil { + return account{}, fmt.Errorf("register app user %s: %w", role, err) + } + token := resp.GetToken() + if token.GetUserId() <= 0 { + return account{}, fmt.Errorf("register app user %s returned empty user id", role) + } + if !resp.GetIsNewUser() { + return account{}, fmt.Errorf("register app user %s reused an existing third-party credential", role) + } + return account{userID: token.GetUserId(), displayUserID: token.GetDisplayUserId(), username: username}, nil +} + +func (r *runner) createBusinessRoles(ctx context.Context) error { + if _, err := r.hostClient.CreateCoinSeller(ctx, &userv1.CreateCoinSellerRequest{ + Meta: r.userMeta("create_coin_seller"), + CommandId: r.commandID("create_coin_seller"), + AdminUserId: r.cfg.admin.userID, + TargetUserId: r.cfg.seller.userID, + Reason: "databi real flow", + }); err != nil { + return fmt.Errorf("create coin seller: %w", err) + } + if _, err := r.hostClient.CreateAgency(ctx, &userv1.CreateAgencyRequest{ + Meta: r.userMeta("create_agency"), + CommandId: r.commandID("create_agency"), + AdminUserId: r.cfg.admin.userID, + OwnerUserId: r.cfg.host.userID, + Name: shortID("Databi Agency", r.cfg.runID, 80), + JoinEnabled: true, + Reason: "databi real flow", + }); err != nil { + return fmt.Errorf("create agency/host: %w", err) + } + fmt.Printf("created roles seller_user_id=%d host_agency_owner_user_id=%d\n", r.cfg.seller.userID, r.cfg.host.userID) + return nil +} + +func (r *runner) prepareWalletAndGifts(ctx context.Context) error { + credits := []struct { + userID int64 + amount int64 + label string + }{ + {userID: r.cfg.sender.userID, amount: 1_000_000, label: "sender"}, + {userID: r.cfg.gamePlayer.userID, amount: 10_000, label: "game_player"}, + {userID: r.cfg.gameRival.userID, amount: 10_000, label: "game_rival"}, + } + for _, credit := range credits { + if _, err := r.walletClient.AdminCreditAsset(ctx, &walletv1.AdminCreditAssetRequest{ + CommandId: r.commandID("manual_credit_" + credit.label), + TargetUserId: credit.userID, + AssetType: "COIN", + Amount: credit.amount, + OperatorUserId: r.cfg.admin.userID, + Reason: "databi real flow manual grant", + EvidenceRef: "local-real-flow", + AppCode: r.cfg.appCode, + }); err != nil { + return fmt.Errorf("admin credit %s: %w", credit.label, err) + } + } + if _, err := r.walletClient.AdminCreditCoinSellerStock(ctx, &walletv1.AdminCreditCoinSellerStockRequest{ + CommandId: r.commandID("seller_stock"), + SellerUserId: r.cfg.seller.userID, + StockType: "usdt_purchase", + CoinAmount: 500_000, + PaidCurrencyCode: "USDT", + PaidAmountMicro: 10_000_000, + PaymentRef: r.commandID("seller_stock_payment"), + EvidenceRef: "local-real-flow", + OperatorUserId: r.cfg.admin.userID, + Reason: "databi real flow seller stock", + AppCode: r.cfg.appCode, + SellerCountryId: r.cfg.countryID, + SellerRegionId: regionID, + }); err != nil { + return fmt.Errorf("credit seller stock: %w", err) + } + if _, err := r.walletClient.TransferCoinFromSeller(ctx, &walletv1.TransferCoinFromSellerRequest{ + CommandId: r.commandID("seller_transfer"), + SellerUserId: r.cfg.seller.userID, + TargetUserId: r.cfg.sender.userID, + Amount: 50_000, + Reason: "databi real flow seller transfer", + AppCode: r.cfg.appCode, + SellerRegionId: regionID, + TargetRegionId: regionID, + TargetCountryId: r.cfg.countryID, + }); err != nil { + return fmt.Errorf("seller transfer coin: %w", err) + } + if _, err := r.walletClient.CreditTaskReward(ctx, &walletv1.CreditTaskRewardRequest{ + CommandId: r.commandID("task_reward"), + AppCode: r.cfg.appCode, + TargetUserId: r.cfg.sender.userID, + Amount: 1_000, + TaskType: "daily", + TaskId: "databi_daily", + CycleKey: time.Now().UTC().Format("2006-01-02"), + Reason: "databi real flow platform reward", + }); err != nil { + return fmt.Errorf("task reward: %w", err) + } + if _, err := r.walletClient.CreditLuckyGiftReward(ctx, &walletv1.CreditLuckyGiftRewardRequest{ + CommandId: r.commandID("super_lucky_reward"), + AppCode: r.cfg.appCode, + TargetUserId: r.cfg.sender.userID, + Amount: 2_000, + DrawId: r.commandID("super_lucky_draw"), + RoomId: r.cfg.roomID, + GiftId: r.cfg.superGift, + PoolId: r.cfg.superPool, + Reason: "databi real flow super lucky payout", + VisibleRegionId: regionID, + CountryId: r.cfg.countryID, + }); err != nil { + return fmt.Errorf("lucky gift reward: %w", err) + } + if err := r.createGift(ctx, r.cfg.normalGift, "normal", r.cfg.normalPrice); err != nil { + return err + } + if err := r.createGift(ctx, r.cfg.luckyGift, "lucky", r.cfg.luckyPrice); err != nil { + return err + } + if err := r.createGift(ctx, r.cfg.superGift, "super_lucky", r.cfg.superPrice); err != nil { + return err + } + fmt.Println("prepared wallet balances, seller stock/transfer, platform reward and gift configs") + return nil +} + +func (r *runner) createGift(ctx context.Context, giftID string, giftType string, price int64) error { + if err := r.ensureGiftType(ctx, giftType); err != nil { + return err + } + mgrGrant := true + resourceResp, err := r.walletClient.CreateResource(ctx, &walletv1.CreateResourceRequest{ + RequestId: r.commandID("resource_" + giftID), + AppCode: r.cfg.appCode, + ResourceCode: "res_" + giftID, + ResourceType: "gift", + Name: "Databi " + giftType, + Status: "active", + Grantable: true, + GrantStrategy: "increase_quantity", + UsageScopes: []string{"gift"}, + AssetUrl: "https://local.hyapp.invalid/" + giftID + ".png", + PreviewUrl: "https://local.hyapp.invalid/" + giftID + "-preview.png", + AnimationUrl: "https://local.hyapp.invalid/" + giftID + ".svga", + MetadataJson: "{}", + SortOrder: 900, + OperatorUserId: r.cfg.admin.userID, + ManagerGrantEnabled: &mgrGrant, + PriceType: "coin", + CoinPrice: price, + }) + if err != nil { + return fmt.Errorf("create resource %s: %w", giftID, err) + } + if _, err := r.walletClient.CreateGiftConfig(ctx, &walletv1.CreateGiftConfigRequest{ + RequestId: r.commandID("gift_config_" + giftID), + AppCode: r.cfg.appCode, + GiftId: giftID, + ResourceId: resourceResp.GetResource().GetResourceId(), + Status: "active", + Name: "Databi " + giftType, + SortOrder: 900, + PresentationJson: "{}", + PriceVersion: "v1", + CoinPrice: price, + HeatValue: price, + EffectiveAtMs: time.Now().UnixMilli(), + OperatorUserId: r.cfg.admin.userID, + RegionIds: []int64{regionID}, + GiftTypeCode: giftType, + ChargeAssetType: "COIN", + EffectiveFromMs: 0, + EffectiveToMs: 0, + }); err != nil { + return fmt.Errorf("create gift config %s: %w", giftID, err) + } + return nil +} + +func (r *runner) ensureGiftType(ctx context.Context, giftType string) error { + var count int64 + if err := r.walletDB.QueryRowContext(ctx, + `SELECT COUNT(*) FROM gift_type_configs WHERE app_code = ? AND type_code = ?`, + r.cfg.appCode, giftType, + ).Scan(&count); err != nil { + return fmt.Errorf("read gift type %s: %w", giftType, err) + } + if count > 0 { + return nil + } + if _, err := r.walletClient.UpsertGiftTypeConfig(ctx, &walletv1.UpsertGiftTypeConfigRequest{ + RequestId: r.commandID("gift_type_" + giftType), + AppCode: r.cfg.appCode, + TypeCode: giftType, + Name: giftType, + TabKey: shortID("databi_"+giftType, r.cfg.runID, 32), + Status: "active", + SortOrder: 900, + OperatorUserId: r.cfg.admin.userID, + }); err != nil { + return fmt.Errorf("upsert gift type %s: %w", giftType, err) + } + return nil +} + +func (r *runner) simulateRechargeEvents(ctx context.Context) error { + if err := r.insertSimulatedRecharge(ctx, "google", 199, 1_990_000); err != nil { + return err + } + if err := r.insertSimulatedRecharge(ctx, "mifapay", 299, 2_990_000); err != nil { + return err + } + fmt.Println("inserted simulated google/mifapay recharge outbox events") + return nil +} + +func (r *runner) insertSimulatedRecharge(ctx context.Context, channel string, usdMinor int64, amountMicro int64) error { + now := time.Now().UnixMilli() + eventID := r.commandID("sim_recharge_" + channel) + payload := map[string]any{ + "app_code": r.cfg.appCode, + "provider": channel, + "channel": channel, + "recharge_type": channel, + "target_user_id": r.cfg.sender.userID, + "target_country_id": r.cfg.countryID, + "country_id": r.cfg.countryID, + "target_region_id": regionID, + "region_id": regionID, + "amount_micro": amountMicro, + "recharge_usd_minor": usdMinor, + "recharge_currency_code": "USD", + "recharge_sequence": 1, + "created_at_ms": now, + } + raw, err := json.Marshal(payload) + if err != nil { + return err + } + _, err = r.walletDB.ExecContext(ctx, ` + INSERT INTO wallet_outbox ( + app_code, event_id, event_type, transaction_id, command_id, user_id, asset_type, + available_delta, frozen_delta, payload, status, worker_id, lock_until_ms, retry_count, + next_retry_at_ms, last_error, created_at_ms, updated_at_ms + ) VALUES (?, ?, 'WalletRechargeRecorded', ?, ?, ?, 'COIN', 0, 0, CAST(? AS JSON), 'pending', '', NULL, 0, NULL, NULL, ?, ?) + ON DUPLICATE KEY UPDATE payload = VALUES(payload), updated_at_ms = VALUES(updated_at_ms)`, + r.cfg.appCode, eventID, eventID, eventID, r.cfg.sender.userID, string(raw), now, now) + if err != nil { + return fmt.Errorf("insert simulated %s recharge: %w", channel, err) + } + return nil +} + +func (r *runner) runRoomFlow(ctx context.Context) error { + if _, err := r.roomClient.CreateRoom(ctx, &roomv1.CreateRoomRequest{ + Meta: r.roomMeta("create_room", r.cfg.sender.userID), + Mode: "voice", + VisibleRegionId: regionID, + RoomName: "Databi Real Flow", + RoomAvatar: "https://local.hyapp.invalid/room.png", + RoomDescription: "databi real flow", + RoomShortId: firstNonEmpty(r.cfg.sender.displayUserID, fmt.Sprintf("%d", r.cfg.sender.userID)), + OwnerCountryCode: countryCode, + }); err != nil { + return fmt.Errorf("create room: %w", err) + } + for _, acc := range []account{r.cfg.host, r.cfg.sender} { + if _, err := r.roomClient.JoinRoom(ctx, &roomv1.JoinRoomRequest{ + Meta: r.roomMeta("join_"+acc.username, acc.userID), + Role: "member", + ActorCountryId: r.cfg.countryID, + ActorRegionId: regionID, + ActorDisplayProfile: &roomv1.RoomUserDisplayProfile{ + UserId: acc.userID, + Nickname: acc.username, + Avatar: "https://local.hyapp.invalid/avatar.png", + }, + }); err != nil { + return fmt.Errorf("join room user %d: %w", acc.userID, err) + } + } + micResp, err := r.roomClient.MicUp(ctx, &roomv1.MicUpRequest{ + Meta: r.roomMeta("mic_up", r.cfg.host.userID), + SeatNo: 1, + TargetDisplayProfile: &roomv1.RoomUserDisplayProfile{ + UserId: r.cfg.host.userID, + Nickname: r.cfg.host.username, + Avatar: "https://local.hyapp.invalid/avatar.png", + }, + }) + if err != nil { + return fmt.Errorf("mic up: %w", err) + } + if _, err := r.roomClient.ConfirmMicPublishing(ctx, &roomv1.ConfirmMicPublishingRequest{ + Meta: r.roomMeta("mic_confirm", r.cfg.host.userID), + TargetUserId: r.cfg.host.userID, + MicSessionId: micResp.GetMicSessionId(), + RoomVersion: micResp.GetResult().GetRoomVersion(), + EventTimeMs: time.Now().UnixMilli(), + Source: "databi-real-flow", + }); err != nil { + return fmt.Errorf("confirm mic publishing: %w", err) + } + time.Sleep(2200 * time.Millisecond) + if _, err := r.roomClient.MicHeartbeat(ctx, &roomv1.MicHeartbeatRequest{ + Meta: r.roomMeta("mic_heartbeat", r.cfg.host.userID), + TargetUserId: r.cfg.host.userID, + MicSessionId: micResp.GetMicSessionId(), + }); err != nil { + return fmt.Errorf("mic heartbeat: %w", err) + } + if _, err := r.roomClient.MicDown(ctx, &roomv1.MicDownRequest{ + Meta: r.roomMeta("mic_down", r.cfg.host.userID), + TargetUserId: r.cfg.host.userID, + Reason: "databi-real-flow", + }); err != nil { + return fmt.Errorf("mic down: %w", err) + } + for _, gift := range []struct { + id string + count int32 + poolID string + }{ + {id: r.cfg.normalGift, count: 1}, + {id: r.cfg.luckyGift, count: 1, poolID: r.cfg.luckyPool}, + {id: r.cfg.superGift, count: 1, poolID: r.cfg.superPool}, + } { + if _, err := r.roomClient.SendGift(ctx, &roomv1.SendGiftRequest{ + Meta: r.roomMeta("send_gift_"+gift.id, r.cfg.sender.userID), + TargetUserId: r.cfg.host.userID, + GiftId: gift.id, + GiftCount: gift.count, + TargetType: "user", + PoolId: gift.poolID, + TargetIsHost: true, + TargetHostRegionId: regionID, + TargetAgencyOwnerUserId: r.cfg.host.userID, + SenderRegionId: regionID, + SenderCountryId: r.cfg.countryID, + SenderDisplayProfile: &roomv1.SendGiftDisplayProfile{ + UserId: r.cfg.sender.userID, + Username: r.cfg.sender.username, + Avatar: "https://local.hyapp.invalid/avatar.png", + }, + TargetDisplayProfiles: []*roomv1.SendGiftDisplayProfile{{ + UserId: r.cfg.host.userID, + Username: r.cfg.host.username, + Avatar: "https://local.hyapp.invalid/avatar.png", + }}, + }); err != nil { + return fmt.Errorf("send gift %s: %w", gift.id, err) + } + } + fmt.Println("completed room join, mic, normal/lucky/super-lucky gift flows") + return nil +} + +func (r *runner) runSalaryFlow(ctx context.Context) error { + cycleKey := time.Now().UTC().Format("2006-01") + resp, err := r.cronClient.ProcessHostSalaryDailySettlementBatch(ctx, &walletv1.CronBatchRequest{ + RequestId: r.commandID("salary_cron_request"), + AppCode: r.cfg.appCode, + RunId: r.commandID("salary_cron"), + WorkerId: "databi-real-flow", + BatchSize: 10, + LockTtlMs: 30_000, + TriggerMode: "automatic", + CycleKey: cycleKey, + SettlementRole: "", + }) + if err != nil { + return fmt.Errorf("salary daily settlement cron: %w", err) + } + fmt.Printf("salary settlement cron success=%d failure=%d\n", resp.GetSuccessCount(), resp.GetFailureCount()) + if _, err := r.walletClient.TransferSalaryToCoinSeller(ctx, &walletv1.TransferSalaryToCoinSellerRequest{ + CommandId: r.commandID("salary_to_seller"), + SourceUserId: r.cfg.host.userID, + SellerUserId: r.cfg.seller.userID, + SalaryAssetType: "HOST_SALARY_USD", + SalaryUsdMinor: 100, + RegionId: regionID, + Reason: "databi real flow salary to seller", + AppCode: r.cfg.appCode, + }); err != nil { + return fmt.Errorf("transfer salary to seller: %w", err) + } + fmt.Println("completed host/agency salary settlement and salary-to-seller transfer") + return nil +} + +func (r *runner) runGameFlow(ctx context.Context) error { + createResp, err := r.gameApp.CreateDiceMatch(ctx, &gamev1.CreateDiceMatchRequest{ + Meta: r.gameMeta("dice_create"), + UserId: r.cfg.gamePlayer.userID, + GameId: gameID, + RoomId: r.cfg.roomID, + RegionId: regionID, + StakeCoin: 100, + MinPlayers: 2, + MaxPlayers: 2, + CountryId: r.cfg.countryID, + }) + if err != nil { + return fmt.Errorf("create dice match: %w", err) + } + matchID := createResp.GetMatch().GetMatchId() + if _, err := r.gameApp.JoinDiceMatch(ctx, &gamev1.JoinDiceMatchRequest{ + Meta: r.gameMeta("dice_join"), + UserId: r.cfg.gameRival.userID, + MatchId: matchID, + GameId: gameID, + }); err != nil { + return fmt.Errorf("join dice match: %w", err) + } + for attempt := 1; attempt <= 5; attempt++ { + rollResp, err := r.gameApp.RollDiceMatch(ctx, &gamev1.RollDiceMatchRequest{ + Meta: r.gameMeta(fmt.Sprintf("dice_roll_%d", attempt)), + UserId: r.cfg.gamePlayer.userID, + MatchId: matchID, + GameId: gameID, + }) + if err != nil { + return fmt.Errorf("roll dice match attempt %d: %w", attempt, err) + } + status := rollResp.GetMatch().GetStatus() + fmt.Printf("dice roll attempt=%d status=%s result=%s\n", attempt, status, rollResp.GetMatch().GetResult()) + if status == "settled" { + fmt.Println("completed game dice settlement") + return nil + } + time.Sleep(2300 * time.Millisecond) + } + return fmt.Errorf("dice match %s did not settle after retries", matchID) +} + +func (r *runner) waitForAsyncSources(ctx context.Context) error { + checks := []struct { + name string + db *sql.DB + query string + args []any + }{ + { + name: "mic daily stats", + db: r.userDB, + query: `SELECT COUNT(*) FROM user_outbox WHERE app_code = ? AND event_type = 'UserMicDailyStatsUpdated' AND created_at_ms >= ?`, + args: []any{r.cfg.appCode, runStartMS(r.cfg.runID)}, + }, + { + name: "game settled outbox", + db: nil, + query: "", + }, + } + for _, check := range checks { + if check.db == nil { + continue + } + if err := waitForCount(ctx, check.db, check.query, check.args...); err != nil { + return fmt.Errorf("wait %s: %w", check.name, err) + } + } + time.Sleep(1500 * time.Millisecond) + return nil +} + +func (r *runner) userMeta(request string) *userv1.RequestMeta { + return &userv1.RequestMeta{ + RequestId: r.commandID(request), + Caller: "databi-real-flow", + SentAtMs: time.Now().UnixMilli(), + AppCode: r.cfg.appCode, + Platform: "ios", + Language: "en", + Timezone: "Asia/Shanghai", + } +} + +func (r *runner) roomMeta(action string, actorUserID int64) *roomv1.RequestMeta { + return &roomv1.RequestMeta{ + RequestId: r.commandID(action + "_request"), + CommandId: r.commandID(action), + ActorUserId: actorUserID, + RoomId: r.cfg.roomID, + GatewayNodeId: "databi-real-flow", + SentAtMs: time.Now().UnixMilli(), + AppCode: r.cfg.appCode, + } +} + +func (r *runner) gameMeta(request string) *gamev1.RequestMeta { + return &gamev1.RequestMeta{ + RequestId: r.commandID(request), + Caller: "databi-real-flow", + GatewayNodeId: "databi-real-flow", + ActorUserId: r.cfg.admin.userID, + SentAtMs: time.Now().UnixMilli(), + AppCode: r.cfg.appCode, + } +} + +func (r *runner) commandID(action string) string { + return shortID("databi_"+action, r.cfg.runID, 120) +} + +func openDB(ctx context.Context, dsn string) (*sql.DB, error) { + db, err := sql.Open("mysql", strings.TrimSpace(dsn)) + if err != nil { + return nil, err + } + db.SetMaxOpenConns(4) + db.SetMaxIdleConns(2) + if err := db.PingContext(ctx); err != nil { + _ = db.Close() + return nil, err + } + return db, nil +} + +func mustDial(ctx context.Context, addr string) *grpc.ClientConn { + conn, err := grpc.DialContext(ctx, addr, grpc.WithTransportCredentials(insecure.NewCredentials()), grpc.WithBlock()) + if err != nil { + log.Fatalf("dial %s: %v", addr, err) + } + return conn +} + +func waitForCount(ctx context.Context, db *sql.DB, query string, args ...any) error { + deadline := time.Now().Add(20 * time.Second) + for { + var count int64 + if err := db.QueryRowContext(ctx, query, args...).Scan(&count); err != nil { + return err + } + if count > 0 { + return nil + } + if time.Now().After(deadline) { + return fmt.Errorf("timed out waiting for count > 0") + } + select { + case <-ctx.Done(): + return ctx.Err() + case <-time.After(500 * time.Millisecond): + } + } +} + +func runStartMS(runID string) int64 { + if value, err := time.Parse("20060102150405", runID); err == nil { + return value.UnixMilli() + } + return time.Now().Add(-10 * time.Minute).UnixMilli() +} + +func shortID(prefix string, suffix string, limit int) string { + id := sanitizeID(prefix) + "_" + sanitizeID(suffix) + if len(id) <= limit { + return id + } + if limit <= 8 { + return id[:limit] + } + tailLen := limit - len(sanitizeID(prefix)) - 1 + if tailLen <= 0 { + return id[:limit] + } + if tailLen > len(sanitizeID(suffix)) { + tailLen = len(sanitizeID(suffix)) + } + return sanitizeID(prefix) + "_" + sanitizeID(suffix)[:tailLen] +} + +func sanitizeID(value string) string { + value = strings.TrimSpace(value) + if value == "" { + value = fmt.Sprintf("%d", time.Now().UnixNano()) + } + var b strings.Builder + for _, r := range value { + switch { + case r >= 'a' && r <= 'z': + b.WriteRune(r) + case r >= 'A' && r <= 'Z': + b.WriteRune(r) + case r >= '0' && r <= '9': + b.WriteRune(r) + case r == '_' || r == '-': + b.WriteRune(r) + default: + b.WriteByte('_') + } + } + return b.String() +} + +func firstNonEmpty(values ...string) string { + for _, value := range values { + if strings.TrimSpace(value) != "" { + return strings.TrimSpace(value) + } + } + return "" +} + +func envDefault(key string, fallback string) string { + if value := strings.TrimSpace(os.Getenv(key)); value != "" { + return value + } + return fallback +}