hy/hyapp-server
2
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

fix: optimize app tracking funnel query

Browse Source
This commit is contained in:
zhx 2026-07-09 23:16:17 +08:00
parent 07302f0aff
commit c09e7be659
4 changed files with 20 additions and 8 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

5
services/statistics-service/deploy/mysql/initdb/001_statistics_service.sql
View File

@ -27,6 +27,7 @@ CREATE TABLE IF NOT EXISTS app_tracking_events (
target_id VARCHAR(128) NOT NULL DEFAULT '' COMMENT '目标 ID',
user_id BIGINT NOT NULL DEFAULT 0 COMMENT '服务端识别的用户 ID，匿名为 0',
device_id VARCHAR(128) NOT NULL DEFAULT '' COMMENT '设备 ID',
identity_key VARCHAR(160) GENERATED ALWAYS AS (CASE WHEN user_id > 0 THEN CONCAT('u:', user_id) ELSE CONCAT('d:', device_id) END) VIRTUAL COMMENT '漏斗/留存去重身份，登录用户优先 user_id，匿名用户回退 device_id',
session_id VARCHAR(160) NOT NULL DEFAULT '' COMMENT '会话 ID',
platform VARCHAR(32) NOT NULL DEFAULT '' COMMENT '平台',
app_version VARCHAR(64) NOT NULL DEFAULT '' COMMENT 'App 版本',
@ -45,7 +46,9 @@ CREATE TABLE IF NOT EXISTS app_tracking_events (
KEY idx_app_tracking_event_name (app_code, stat_day, event_name),
KEY idx_app_tracking_screen (app_code, stat_day, screen),
KEY idx_app_tracking_user (app_code, user_id, occurred_at_ms),
KEY idx_app_tracking_device (app_code, device_id, occurred_at_ms)
KEY idx_app_tracking_device (app_code, device_id, occurred_at_ms),
KEY idx_app_tracking_event_window (app_code, event_name, occurred_at_ms, identity_key, device_id),
KEY idx_app_tracking_retention_identity (app_code, stat_day, identity_key)
) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_unicode_ci COMMENT='App 埋点原始明细表';
CREATE TABLE IF NOT EXISTS stat_app_day_country (

5
services/statistics-service/internal/storage/mysql/query.go
View File

@ -2438,7 +2438,10 @@ func appTrackingIdentitySQL(alias string) string {
if alias != "" {
prefix = alias + "."
}
return "CASE WHEN " + prefix + "user_id > 0 THEN CONCAT('u:', " + prefix + "user_id) ELSE CONCAT('d:', " + prefix + "device_id) END"
// identity_key is a generated column indexed by the statistics schema.
// Keeping identity derivation out of hot funnel SQL lets MySQL use window
// and retention indexes instead of rebuilding CONCAT keys for every row.
return prefix + "identity_key"
}
func appTrackingEventNames() []string {

10
services/statistics-service/internal/storage/mysql/query_test.go
View File

@ -135,19 +135,19 @@ func TestConsumeAppTrackingEventsStoresRawEventsAndDeduplicates(t *testing.T) {
}
var count int64
var statDay, screen, propertiesJSON string
var statDay, screen, identityKey, propertiesJSON string
var userID, countryID, regionID, durationMS int64
var success bool
if err := repository.db.QueryRowContext(ctx, `
SELECT COUNT(*), MAX(CAST(stat_day AS CHAR)), MAX(screen), MAX(user_id), MAX(country_id), MAX(region_id),
SELECT COUNT(*), MAX(CAST(stat_day AS CHAR)), MAX(screen), MAX(identity_key), MAX(user_id), MAX(country_id), MAX(region_id),
MAX(duration_ms), MAX(success), COALESCE(MAX(CAST(properties_json AS CHAR)), '{}')
FROM app_tracking_events
WHERE app_code = 'lalu' AND event_id = 'evt-banner-1'
`).Scan(&count, &statDay, &screen, &userID, &countryID, &regionID, &durationMS, &success, &propertiesJSON); err != nil {
`).Scan(&count, &statDay, &screen, &identityKey, &userID, &countryID, &regionID, &durationMS, &success, &propertiesJSON); err != nil {
t.Fatalf("query app tracking event: %v", err)
}
if count != 1 || statDay != "2026-07-02" || screen != "home" || userID != 42 || countryID != 86 || regionID != 210 || durationMS != 12 || !success {
t.Fatalf("stored event mismatch: count=%d statDay=%s screen=%s userID=%d countryID=%d regionID=%d durationMS=%d success=%v", count, statDay, screen, userID, countryID, regionID, durationMS, success)
if count != 1 || statDay != "2026-07-02" || screen != "home" || identityKey != "u:42" || userID != 42 || countryID != 86 || regionID != 210 || durationMS != 12 || !success {
t.Fatalf("stored event mismatch: count=%d statDay=%s screen=%s identityKey=%s userID=%d countryID=%d regionID=%d durationMS=%d success=%v", count, statDay, screen, identityKey, userID, countryID, regionID, durationMS, success)
}
if DecodeJSON(propertiesJSON)["slot"] != "top" {
t.Fatalf("properties_json mismatch: %s", propertiesJSON)

8
services/statistics-service/internal/storage/mysql/repository.go
View File

@ -125,6 +125,7 @@ func (r *Repository) Migrate(ctx context.Context) error {
event_type VARCHAR(64) NOT NULL DEFAULT '', screen VARCHAR(128) NOT NULL DEFAULT '',
target_type VARCHAR(64) NOT NULL DEFAULT '', target_id VARCHAR(128) NOT NULL DEFAULT '',
user_id BIGINT NOT NULL DEFAULT 0, device_id VARCHAR(128) NOT NULL DEFAULT '',
identity_key VARCHAR(160) GENERATED ALWAYS AS (CASE WHEN user_id > 0 THEN CONCAT('u:', user_id) ELSE CONCAT('d:', device_id) END) VIRTUAL,
session_id VARCHAR(160) NOT NULL DEFAULT '', platform VARCHAR(32) NOT NULL DEFAULT '',
app_version VARCHAR(64) NOT NULL DEFAULT '', language VARCHAR(32) NOT NULL DEFAULT '',
timezone VARCHAR(64) NOT NULL DEFAULT '', country_id BIGINT NOT NULL DEFAULT 0,
@ -136,7 +137,9 @@ func (r *Repository) Migrate(ctx context.Context) error {
KEY idx_app_tracking_event_name (app_code, stat_day, event_name),
KEY idx_app_tracking_screen (app_code, stat_day, screen),
KEY idx_app_tracking_user (app_code, user_id, occurred_at_ms),
KEY idx_app_tracking_device (app_code, device_id, occurred_at_ms)
KEY idx_app_tracking_device (app_code, device_id, occurred_at_ms),
KEY idx_app_tracking_event_window (app_code, event_name, occurred_at_ms, identity_key, device_id),
KEY idx_app_tracking_retention_identity (app_code, stat_day, identity_key)
) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_unicode_ci`,
`CREATE TABLE IF NOT EXISTS stat_app_day_country (
app_code VARCHAR(32) NOT NULL, stat_tz VARCHAR(64) NOT NULL DEFAULT 'UTC', stat_day DATE NOT NULL, country_id BIGINT NOT NULL DEFAULT 0,
@ -377,6 +380,7 @@ func (r *Repository) Migrate(ctx context.Context) error {
}
}
alterStatements := []string{
`ALTER TABLE app_tracking_events ADD COLUMN identity_key VARCHAR(160) GENERATED ALWAYS AS (CASE WHEN user_id > 0 THEN CONCAT('u:', user_id) ELSE CONCAT('d:', device_id) END) VIRTUAL AFTER device_id`,
`ALTER TABLE stat_app_day_country ADD COLUMN paid_users BIGINT NOT NULL DEFAULT 0 AFTER active_users`,
`ALTER TABLE stat_app_day_country ADD COLUMN game_refund BIGINT NOT NULL DEFAULT 0 AFTER game_payout`,
`ALTER TABLE stat_app_day_country ADD COLUMN lucky_gift_payout BIGINT NOT NULL DEFAULT 0 AFTER lucky_gift_turnover`,
@ -451,6 +455,8 @@ func (r *Repository) Migrate(ctx context.Context) error {
{"app_tracking_events", "idx_app_tracking_screen", []string{"app_code", "stat_day", "screen"}},
{"app_tracking_events", "idx_app_tracking_user", []string{"app_code", "user_id", "occurred_at_ms"}},
{"app_tracking_events", "idx_app_tracking_device", []string{"app_code", "device_id", "occurred_at_ms"}},
{"app_tracking_events", "idx_app_tracking_event_window", []string{"app_code", "event_name", "occurred_at_ms", "identity_key", "device_id"}},
{"app_tracking_events", "idx_app_tracking_retention_identity", []string{"app_code", "stat_day", "identity_key"}},
{"stat_app_day_country", "idx_stat_app_day_region", []string{"app_code", "stat_tz", "stat_day", "region_id"}},
{"stat_user_day_activity", "idx_stat_user_day_country", []string{"app_code", "stat_tz", "stat_day", "country_id"}},
{"stat_user_day_activity", "idx_stat_user_day_region", []string{"app_code", "stat_tz", "stat_day", "region_id"}},