hy/hyapp-server
2
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

fix: optimize app tracking funnel query

Browse Source
This commit is contained in:
zhx 2026-07-09 23:16:17 +08:00
parent 07302f0aff
commit c09e7be659
4 changed files with 20 additions and 8 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

5
services/statistics-service/deploy/mysql/initdb/001_statistics_service.sql
View File

@ -27,6 +27,7 @@ CREATE TABLE IF NOT EXISTS app_tracking_events (
target_id VARCHAR(128) NOT NULL DEFAULT '' COMMENT '目标 ID', target_id VARCHAR(128) NOT NULL DEFAULT '' COMMENT '目标 ID',
user_id BIGINT NOT NULL DEFAULT 0 COMMENT '服务端识别的用户 ID，匿名为 0', user_id BIGINT NOT NULL DEFAULT 0 COMMENT '服务端识别的用户 ID，匿名为 0',
device_id VARCHAR(128) NOT NULL DEFAULT '' COMMENT '设备 ID', device_id VARCHAR(128) NOT NULL DEFAULT '' COMMENT '设备 ID',
identity_key VARCHAR(160) GENERATED ALWAYS AS (CASE WHEN user_id > 0 THEN CONCAT('u:', user_id) ELSE CONCAT('d:', device_id) END) VIRTUAL COMMENT '漏斗/留存去重身份，登录用户优先 user_id，匿名用户回退 device_id',
session_id VARCHAR(160) NOT NULL DEFAULT '' COMMENT '会话 ID', session_id VARCHAR(160) NOT NULL DEFAULT '' COMMENT '会话 ID',
platform VARCHAR(32) NOT NULL DEFAULT '' COMMENT '平台', platform VARCHAR(32) NOT NULL DEFAULT '' COMMENT '平台',
app_version VARCHAR(64) NOT NULL DEFAULT '' COMMENT 'App 版本', app_version VARCHAR(64) NOT NULL DEFAULT '' COMMENT 'App 版本',
@ -45,7 +46,9 @@ CREATE TABLE IF NOT EXISTS app_tracking_events (
KEY idx_app_tracking_event_name (app_code, stat_day, event_name), KEY idx_app_tracking_event_name (app_code, stat_day, event_name),
KEY idx_app_tracking_screen (app_code, stat_day, screen), KEY idx_app_tracking_screen (app_code, stat_day, screen),
KEY idx_app_tracking_user (app_code, user_id, occurred_at_ms), KEY idx_app_tracking_user (app_code, user_id, occurred_at_ms),
KEY idx_app_tracking_device (app_code, device_id, occurred_at_ms) KEY idx_app_tracking_device (app_code, device_id, occurred_at_ms),
KEY idx_app_tracking_event_window (app_code, event_name, occurred_at_ms, identity_key, device_id),
KEY idx_app_tracking_retention_identity (app_code, stat_day, identity_key)
) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_unicode_ci COMMENT='App 埋点原始明细表'; ) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_unicode_ci COMMENT='App 埋点原始明细表';
CREATE TABLE IF NOT EXISTS stat_app_day_country ( CREATE TABLE IF NOT EXISTS stat_app_day_country (

5
services/statistics-service/internal/storage/mysql/query.go
View File

@ -2438,7 +2438,10 @@ func appTrackingIdentitySQL(alias string) string {
if alias != "" { if alias != "" {
prefix = alias + "." prefix = alias + "."
} }
return "CASE WHEN " + prefix + "user_id > 0 THEN CONCAT('u:', " + prefix + "user_id) ELSE CONCAT('d:', " + prefix + "device_id) END" // identity_key is a generated column indexed by the statistics schema.
// Keeping identity derivation out of hot funnel SQL lets MySQL use window
// and retention indexes instead of rebuilding CONCAT keys for every row.
return prefix + "identity_key"
} }
func appTrackingEventNames() []string { func appTrackingEventNames() []string {

10
services/statistics-service/internal/storage/mysql/query_test.go
View File

@ -135,19 +135,19 @@ func TestConsumeAppTrackingEventsStoresRawEventsAndDeduplicates(t *testing.T) {
} }
var count int64 var count int64
var statDay, screen, propertiesJSON string var statDay, screen, identityKey, propertiesJSON string
var userID, countryID, regionID, durationMS int64 var userID, countryID, regionID, durationMS int64
var success bool var success bool
if err := repository.db.QueryRowContext(ctx, ` if err := repository.db.QueryRowContext(ctx, `
SELECT COUNT(*), MAX(CAST(stat_day AS CHAR)), MAX(screen), MAX(user_id), MAX(country_id), MAX(region_id), SELECT COUNT(*), MAX(CAST(stat_day AS CHAR)), MAX(screen), MAX(identity_key), MAX(user_id), MAX(country_id), MAX(region_id),
MAX(duration_ms), MAX(success), COALESCE(MAX(CAST(properties_json AS CHAR)), '{}') MAX(duration_ms), MAX(success), COALESCE(MAX(CAST(properties_json AS CHAR)), '{}')
FROM app_tracking_events FROM app_tracking_events
WHERE app_code = 'lalu' AND event_id = 'evt-banner-1' WHERE app_code = 'lalu' AND event_id = 'evt-banner-1'
`).Scan(&count, &statDay, &screen, &userID, &countryID, &regionID, &durationMS, &success, &propertiesJSON); err != nil { `).Scan(&count, &statDay, &screen, &identityKey, &userID, &countryID, &regionID, &durationMS, &success, &propertiesJSON); err != nil {
t.Fatalf("query app tracking event: %v", err) t.Fatalf("query app tracking event: %v", err)
} }
if count != 1 || statDay != "2026-07-02" || screen != "home" || userID != 42 || countryID != 86 || regionID != 210 || durationMS != 12 || !success { if count != 1 || statDay != "2026-07-02" || screen != "home" || identityKey != "u:42" || userID != 42 || countryID != 86 || regionID != 210 || durationMS != 12 || !success {
t.Fatalf("stored event mismatch: count=%d statDay=%s screen=%s userID=%d countryID=%d regionID=%d durationMS=%d success=%v", count, statDay, screen, userID, countryID, regionID, durationMS, success) t.Fatalf("stored event mismatch: count=%d statDay=%s screen=%s identityKey=%s userID=%d countryID=%d regionID=%d durationMS=%d success=%v", count, statDay, screen, identityKey, userID, countryID, regionID, durationMS, success)
} }
if DecodeJSON(propertiesJSON)["slot"] != "top" { if DecodeJSON(propertiesJSON)["slot"] != "top" {
t.Fatalf("properties_json mismatch: %s", propertiesJSON) t.Fatalf("properties_json mismatch: %s", propertiesJSON)

8
services/statistics-service/internal/storage/mysql/repository.go
View File

@ -125,6 +125,7 @@ func (r *Repository) Migrate(ctx context.Context) error {
event_type VARCHAR(64) NOT NULL DEFAULT '', screen VARCHAR(128) NOT NULL DEFAULT '', event_type VARCHAR(64) NOT NULL DEFAULT '', screen VARCHAR(128) NOT NULL DEFAULT '',
target_type VARCHAR(64) NOT NULL DEFAULT '', target_id VARCHAR(128) NOT NULL DEFAULT '', target_type VARCHAR(64) NOT NULL DEFAULT '', target_id VARCHAR(128) NOT NULL DEFAULT '',
user_id BIGINT NOT NULL DEFAULT 0, device_id VARCHAR(128) NOT NULL DEFAULT '', user_id BIGINT NOT NULL DEFAULT 0, device_id VARCHAR(128) NOT NULL DEFAULT '',
identity_key VARCHAR(160) GENERATED ALWAYS AS (CASE WHEN user_id > 0 THEN CONCAT('u:', user_id) ELSE CONCAT('d:', device_id) END) VIRTUAL,
session_id VARCHAR(160) NOT NULL DEFAULT '', platform VARCHAR(32) NOT NULL DEFAULT '', session_id VARCHAR(160) NOT NULL DEFAULT '', platform VARCHAR(32) NOT NULL DEFAULT '',
app_version VARCHAR(64) NOT NULL DEFAULT '', language VARCHAR(32) NOT NULL DEFAULT '', app_version VARCHAR(64) NOT NULL DEFAULT '', language VARCHAR(32) NOT NULL DEFAULT '',
timezone VARCHAR(64) NOT NULL DEFAULT '', country_id BIGINT NOT NULL DEFAULT 0, timezone VARCHAR(64) NOT NULL DEFAULT '', country_id BIGINT NOT NULL DEFAULT 0,
@ -136,7 +137,9 @@ func (r *Repository) Migrate(ctx context.Context) error {
KEY idx_app_tracking_event_name (app_code, stat_day, event_name), KEY idx_app_tracking_event_name (app_code, stat_day, event_name),
KEY idx_app_tracking_screen (app_code, stat_day, screen), KEY idx_app_tracking_screen (app_code, stat_day, screen),
KEY idx_app_tracking_user (app_code, user_id, occurred_at_ms), KEY idx_app_tracking_user (app_code, user_id, occurred_at_ms),
KEY idx_app_tracking_device (app_code, device_id, occurred_at_ms) KEY idx_app_tracking_device (app_code, device_id, occurred_at_ms),
KEY idx_app_tracking_event_window (app_code, event_name, occurred_at_ms, identity_key, device_id),
KEY idx_app_tracking_retention_identity (app_code, stat_day, identity_key)
) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_unicode_ci`, ) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_unicode_ci`,
`CREATE TABLE IF NOT EXISTS stat_app_day_country ( `CREATE TABLE IF NOT EXISTS stat_app_day_country (
app_code VARCHAR(32) NOT NULL, stat_tz VARCHAR(64) NOT NULL DEFAULT 'UTC', stat_day DATE NOT NULL, country_id BIGINT NOT NULL DEFAULT 0, app_code VARCHAR(32) NOT NULL, stat_tz VARCHAR(64) NOT NULL DEFAULT 'UTC', stat_day DATE NOT NULL, country_id BIGINT NOT NULL DEFAULT 0,
@ -377,6 +380,7 @@ func (r *Repository) Migrate(ctx context.Context) error {
} }
} }
alterStatements := []string{ alterStatements := []string{
`ALTER TABLE app_tracking_events ADD COLUMN identity_key VARCHAR(160) GENERATED ALWAYS AS (CASE WHEN user_id > 0 THEN CONCAT('u:', user_id) ELSE CONCAT('d:', device_id) END) VIRTUAL AFTER device_id`,
`ALTER TABLE stat_app_day_country ADD COLUMN paid_users BIGINT NOT NULL DEFAULT 0 AFTER active_users`, `ALTER TABLE stat_app_day_country ADD COLUMN paid_users BIGINT NOT NULL DEFAULT 0 AFTER active_users`,
`ALTER TABLE stat_app_day_country ADD COLUMN game_refund BIGINT NOT NULL DEFAULT 0 AFTER game_payout`, `ALTER TABLE stat_app_day_country ADD COLUMN game_refund BIGINT NOT NULL DEFAULT 0 AFTER game_payout`,
`ALTER TABLE stat_app_day_country ADD COLUMN lucky_gift_payout BIGINT NOT NULL DEFAULT 0 AFTER lucky_gift_turnover`, `ALTER TABLE stat_app_day_country ADD COLUMN lucky_gift_payout BIGINT NOT NULL DEFAULT 0 AFTER lucky_gift_turnover`,
@ -451,6 +455,8 @@ func (r *Repository) Migrate(ctx context.Context) error {
{"app_tracking_events", "idx_app_tracking_screen", []string{"app_code", "stat_day", "screen"}}, {"app_tracking_events", "idx_app_tracking_screen", []string{"app_code", "stat_day", "screen"}},
{"app_tracking_events", "idx_app_tracking_user", []string{"app_code", "user_id", "occurred_at_ms"}}, {"app_tracking_events", "idx_app_tracking_user", []string{"app_code", "user_id", "occurred_at_ms"}},
{"app_tracking_events", "idx_app_tracking_device", []string{"app_code", "device_id", "occurred_at_ms"}}, {"app_tracking_events", "idx_app_tracking_device", []string{"app_code", "device_id", "occurred_at_ms"}},
{"app_tracking_events", "idx_app_tracking_event_window", []string{"app_code", "event_name", "occurred_at_ms", "identity_key", "device_id"}},
{"app_tracking_events", "idx_app_tracking_retention_identity", []string{"app_code", "stat_day", "identity_key"}},
{"stat_app_day_country", "idx_stat_app_day_region", []string{"app_code", "stat_tz", "stat_day", "region_id"}}, {"stat_app_day_country", "idx_stat_app_day_region", []string{"app_code", "stat_tz", "stat_day", "region_id"}},
{"stat_user_day_activity", "idx_stat_user_day_country", []string{"app_code", "stat_tz", "stat_day", "country_id"}}, {"stat_user_day_activity", "idx_stat_user_day_country", []string{"app_code", "stat_tz", "stat_day", "country_id"}},
{"stat_user_day_activity", "idx_stat_user_day_region", []string{"app_code", "stat_tz", "stat_day", "region_id"}}, {"stat_user_day_activity", "idx_stat_user_day_region", []string{"app_code", "stat_tz", "stat_day", "region_id"}},