fix: 修复用户画像版本分布查询
This commit is contained in:
zhx
parent
a2e6e5f84d
commit
902cf46516
@ -44,6 +44,9 @@ func (h *Handler) UserProfileOverview(c *gin.Context) {
|
||||
EndMS: parseInt64(c.Query("end_ms")),
|
||||
})
|
||||
if err != nil {
|
||||
// 对外继续返回稳定的通用文案，真实依赖/SQL 错误挂到 Gin request errors；access log 才能用
|
||||
// request_id 还原失败节点，同时避免把数据库结构或内部地址泄露给后台浏览器。
|
||||
_ = c.Error(err)
|
||||
response.ServerError(c, "获取用户画像总览失败")
|
||||
return
|
||||
}
|
||||
|
||||
@ -111,37 +111,58 @@ func (s *DashboardService) queryNaturalUserGenderDistribution(ctx context.Contex
|
||||
}
|
||||
|
||||
func (s *DashboardService) queryNaturalUserAppVersionDistribution(ctx context.Context, appCode string) ([]UserProfileDistributionItem, error) {
|
||||
// login_audit 的复合索引把 login_type 放在 created_at_ms 之前；若把三种登录类型写成 IN，
|
||||
// MySQL 必须为每个用户扫描并排序全部候选，线上百万级审计表会超时。这里按类型拆成三个等值索引
|
||||
// 范围，每个范围只取一条，再对最多三条候选按 (created_at_ms, id) 选最新记录，既保持原口径，
|
||||
// 也避免窗口函数物化整张审计表。natural 是 MySQL 保留字，用户别名必须使用 natural_user。
|
||||
rows, err := s.userDB.QueryContext(ctx, `
|
||||
WITH natural_users AS (
|
||||
SELECT user_id
|
||||
FROM users
|
||||
WHERE app_code = ?
|
||||
AND profile_completed = 1
|
||||
AND LOWER(TRIM(COALESCE(source, ''))) NOT IN ('game_robot', 'quick_account')
|
||||
), ranked_login AS (
|
||||
SELECT audit.user_id,
|
||||
TRIM(COALESCE(audit.app_version, '')) AS app_version,
|
||||
ROW_NUMBER() OVER (
|
||||
PARTITION BY audit.user_id
|
||||
ORDER BY audit.created_at_ms DESC, audit.id DESC
|
||||
) AS login_rank
|
||||
FROM login_audit AS audit
|
||||
INNER JOIN natural_users AS natural ON natural.user_id = audit.user_id
|
||||
WHERE audit.app_code = ?
|
||||
AND audit.result = 'success'
|
||||
AND audit.blocked = 0
|
||||
AND audit.login_type IN ('password', 'third_party', 'refresh')
|
||||
)
|
||||
SELECT CASE
|
||||
WHEN latest.app_version IS NULL OR latest.app_version = '' THEN 'unknown'
|
||||
ELSE latest.app_version
|
||||
WHEN latest.app_version IS NULL OR TRIM(latest.app_version) = '' THEN 'unknown'
|
||||
ELSE TRIM(latest.app_version)
|
||||
END AS version_key,
|
||||
COUNT(*) AS user_count
|
||||
FROM natural_users AS natural
|
||||
LEFT JOIN ranked_login AS latest
|
||||
ON latest.user_id = natural.user_id AND latest.login_rank = 1
|
||||
FROM users AS natural_user
|
||||
LEFT JOIN LATERAL (
|
||||
SELECT candidate.app_version
|
||||
FROM (
|
||||
(SELECT audit.app_version, audit.created_at_ms, audit.id
|
||||
FROM login_audit AS audit
|
||||
WHERE audit.app_code = natural_user.app_code
|
||||
AND audit.user_id = natural_user.user_id
|
||||
AND audit.result = 'success'
|
||||
AND audit.blocked = 0
|
||||
AND audit.login_type = 'password'
|
||||
ORDER BY audit.created_at_ms DESC, audit.id DESC
|
||||
LIMIT 1)
|
||||
UNION ALL
|
||||
(SELECT audit.app_version, audit.created_at_ms, audit.id
|
||||
FROM login_audit AS audit
|
||||
WHERE audit.app_code = natural_user.app_code
|
||||
AND audit.user_id = natural_user.user_id
|
||||
AND audit.result = 'success'
|
||||
AND audit.blocked = 0
|
||||
AND audit.login_type = 'third_party'
|
||||
ORDER BY audit.created_at_ms DESC, audit.id DESC
|
||||
LIMIT 1)
|
||||
UNION ALL
|
||||
(SELECT audit.app_version, audit.created_at_ms, audit.id
|
||||
FROM login_audit AS audit
|
||||
WHERE audit.app_code = natural_user.app_code
|
||||
AND audit.user_id = natural_user.user_id
|
||||
AND audit.result = 'success'
|
||||
AND audit.blocked = 0
|
||||
AND audit.login_type = 'refresh'
|
||||
ORDER BY audit.created_at_ms DESC, audit.id DESC
|
||||
LIMIT 1)
|
||||
) AS candidate
|
||||
ORDER BY candidate.created_at_ms DESC, candidate.id DESC
|
||||
LIMIT 1
|
||||
) AS latest ON TRUE
|
||||
WHERE natural_user.app_code = ?
|
||||
AND natural_user.profile_completed = 1
|
||||
AND LOWER(TRIM(COALESCE(natural_user.source, ''))) NOT IN ('game_robot', 'quick_account')
|
||||
GROUP BY version_key
|
||||
ORDER BY user_count DESC, version_key ASC`, appCode, appCode)
|
||||
ORDER BY user_count DESC, version_key ASC`, appCode)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
@ -0,0 +1,126 @@
|
||||
package dashboard
|
||||
|
||||
import (
|
||||
"context"
|
||||
"database/sql"
|
||||
"fmt"
|
||||
"os"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
mysqlDriver "github.com/go-sql-driver/mysql"
|
||||
)
|
||||
|
||||
func TestUserProfileVersionDistributionRealMySQL(t *testing.T) {
|
||||
baseDSN := strings.TrimSpace(os.Getenv("DASHBOARD_USER_PROFILE_MYSQL_TEST_DSN"))
|
||||
if baseDSN == "" {
|
||||
t.Skip("set DASHBOARD_USER_PROFILE_MYSQL_TEST_DSN to run the real MySQL user-profile query")
|
||||
}
|
||||
|
||||
// sqlmock only matches text and previously let a reserved-word alias reach production. This focused
|
||||
// integration test asks a real MySQL parser/optimizer to execute the LATERAL query and its tie-break rule.
|
||||
db := newUserProfileMySQLTestDB(t, baseDSN)
|
||||
execUserProfileSQL(t, db,
|
||||
`CREATE TABLE users (
|
||||
app_code VARCHAR(32) NOT NULL,
|
||||
user_id BIGINT NOT NULL,
|
||||
gender VARCHAR(32) NULL,
|
||||
profile_completed TINYINT(1) NOT NULL DEFAULT 0,
|
||||
source VARCHAR(64) NULL,
|
||||
PRIMARY KEY (app_code, user_id)
|
||||
) ENGINE=InnoDB`,
|
||||
`CREATE TABLE login_audit (
|
||||
app_code VARCHAR(32) NOT NULL,
|
||||
id BIGINT NOT NULL AUTO_INCREMENT PRIMARY KEY,
|
||||
user_id BIGINT NULL,
|
||||
login_type VARCHAR(32) NOT NULL,
|
||||
app_version VARCHAR(64) NOT NULL DEFAULT '',
|
||||
result VARCHAR(32) NOT NULL,
|
||||
blocked TINYINT(1) NOT NULL DEFAULT 0,
|
||||
created_at_ms BIGINT NOT NULL,
|
||||
KEY idx_login_audit_latest_success (app_code, user_id, result, blocked, login_type, created_at_ms, id)
|
||||
) ENGINE=InnoDB`,
|
||||
`INSERT INTO users (app_code, user_id, gender, profile_completed, source) VALUES
|
||||
('lalu', 1, 'male', 1, 'third_party'),
|
||||
('lalu', 2, 'female', 1, 'password'),
|
||||
('lalu', 3, 'male', 1, 'quick_account'),
|
||||
('lalu', 4, 'female', 1, 'third_party'),
|
||||
('lalu', 5, 'male', 1, 'password'),
|
||||
('lalu', 6, 'male', 1, 'game_robot'),
|
||||
('lalu', 7, 'female', 0, 'third_party'),
|
||||
('lalu', 9, 'female', 1, 'third_party'),
|
||||
('huwaa', 1, 'female', 1, 'third_party')`,
|
||||
`INSERT INTO login_audit (app_code, user_id, login_type, app_version, result, blocked, created_at_ms) VALUES
|
||||
('lalu', 1, 'password', '1.0.0', 'success', 0, 100),
|
||||
('lalu', 1, 'third_party', '2.0.0', 'success', 0, 200),
|
||||
('lalu', 1, 'refresh', '3.0.0', 'success', 0, 200),
|
||||
('lalu', 3, 'refresh', 'excluded', 'success', 0, 500),
|
||||
('lalu', 4, 'password', '', 'success', 0, 100),
|
||||
('lalu', 4, 'third_party', 'blocked', 'success', 1, 300),
|
||||
('lalu', 4, 'refresh', 'failed', 'failed', 0, 400),
|
||||
('lalu', 4, 'logout', 'unsupported', 'success', 0, 500),
|
||||
('lalu', 5, 'password', 'time-wins', 'success', 0, 500),
|
||||
('lalu', 5, 'password', 'larger-id-loses', 'success', 0, 100),
|
||||
('lalu', 6, 'password', 'excluded', 'success', 0, 500),
|
||||
('lalu', 7, 'password', 'excluded', 'success', 0, 500),
|
||||
('lalu', 9, 'password', ' ', 'success', 0, 500),
|
||||
('huwaa', 1, 'password', 'other-app', 'success', 0, 500)`,
|
||||
)
|
||||
|
||||
ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second)
|
||||
defer cancel()
|
||||
items, err := (&DashboardService{userDB: db}).queryNaturalUserAppVersionDistribution(ctx, "lalu")
|
||||
if err != nil {
|
||||
t.Fatalf("query real MySQL version distribution: %v", err)
|
||||
}
|
||||
if len(items) != 3 ||
|
||||
items[0] != (UserProfileDistributionItem{Key: "unknown", Label: "未知版本", Count: 3}) ||
|
||||
items[1] != (UserProfileDistributionItem{Key: "3.0.0", Label: "3.0.0", Count: 1}) ||
|
||||
items[2] != (UserProfileDistributionItem{Key: "time-wins", Label: "time-wins", Count: 1}) {
|
||||
t.Fatalf("version distribution mismatch: %+v", items)
|
||||
}
|
||||
}
|
||||
|
||||
func newUserProfileMySQLTestDB(t *testing.T, baseDSN string) *sql.DB {
|
||||
t.Helper()
|
||||
baseConfig, err := mysqlDriver.ParseDSN(baseDSN)
|
||||
if err != nil {
|
||||
t.Fatalf("parse MySQL test DSN: %v", err)
|
||||
}
|
||||
databaseName := fmt.Sprintf("hy_admin_user_profile_%d", time.Now().UnixNano())
|
||||
adminConfig := baseConfig.Clone()
|
||||
adminConfig.DBName = ""
|
||||
adminDB, err := sql.Open("mysql", adminConfig.FormatDSN())
|
||||
if err != nil {
|
||||
t.Fatalf("open MySQL admin connection: %v", err)
|
||||
}
|
||||
if _, err := adminDB.Exec("CREATE DATABASE `" + databaseName + "` DEFAULT CHARACTER SET utf8mb4 COLLATE utf8mb4_unicode_ci"); err != nil {
|
||||
_ = adminDB.Close()
|
||||
t.Fatalf("create MySQL test database: %v", err)
|
||||
}
|
||||
|
||||
testConfig := baseConfig.Clone()
|
||||
testConfig.DBName = databaseName
|
||||
db, err := sql.Open("mysql", testConfig.FormatDSN())
|
||||
if err != nil {
|
||||
_, _ = adminDB.Exec("DROP DATABASE IF EXISTS `" + databaseName + "`")
|
||||
_ = adminDB.Close()
|
||||
t.Fatalf("open MySQL test database: %v", err)
|
||||
}
|
||||
t.Cleanup(func() {
|
||||
_ = db.Close()
|
||||
_, _ = adminDB.Exec("DROP DATABASE IF EXISTS `" + databaseName + "`")
|
||||
_ = adminDB.Close()
|
||||
})
|
||||
return db
|
||||
}
|
||||
|
||||
func execUserProfileSQL(t *testing.T, db *sql.DB, statements ...string) {
|
||||
t.Helper()
|
||||
for _, statement := range statements {
|
||||
if _, err := db.Exec(statement); err != nil {
|
||||
t.Fatalf("execute user-profile fixture SQL: %v", err)
|
||||
}
|
||||
}
|
||||
}
|
||||
@ -2,8 +2,10 @@ package dashboard
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
@ -27,8 +29,8 @@ func TestUserProfileOverviewAggregatesNaturalUsersAndDailyStatistics(t *testing.
|
||||
AddRow("male", int64(4)).
|
||||
AddRow("female", int64(2)).
|
||||
AddRow("unknown", int64(1)))
|
||||
sqlMock.ExpectQuery(`(?s)WITH natural_users AS.*FROM users.*app_code = \?.*profile_completed = 1.*game_robot.*quick_account.*ranked_login AS.*ROW_NUMBER\(\) OVER.*PARTITION BY audit.user_id.*ORDER BY audit.created_at_ms DESC, audit.id DESC.*audit.app_code = \?.*audit.result = 'success'.*audit.blocked = 0.*audit.login_type IN \('password', 'third_party', 'refresh'\).*LEFT JOIN ranked_login.*latest.login_rank = 1.*GROUP BY version_key`).
|
||||
WithArgs("huwaa", "huwaa").
|
||||
sqlMock.ExpectQuery(`(?s)FROM users AS natural_user.*LEFT JOIN LATERAL.*login_type = 'password'.*ORDER BY audit.created_at_ms DESC, audit.id DESC.*UNION ALL.*login_type = 'third_party'.*UNION ALL.*login_type = 'refresh'.*ORDER BY candidate.created_at_ms DESC, candidate.id DESC.*WHERE natural_user.app_code = \?.*profile_completed = 1.*game_robot.*quick_account.*GROUP BY version_key`).
|
||||
WithArgs("huwaa").
|
||||
WillReturnRows(sqlmock.NewRows([]string{"version_key", "user_count"}).
|
||||
AddRow("2.4.0", int64(4)).
|
||||
AddRow("unknown", int64(3)))
|
||||
@ -122,6 +124,45 @@ func TestUserProfileOverviewRouteRequiresOverviewPermission(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
func TestUserProfileOverviewRecordsInternalErrorAgainstRequestID(t *testing.T) {
|
||||
userDB, sqlMock, err := sqlmock.New()
|
||||
if err != nil {
|
||||
t.Fatalf("create user sqlmock: %v", err)
|
||||
}
|
||||
defer userDB.Close()
|
||||
|
||||
sqlMock.ExpectQuery(`(?s)SELECT CASE.*FROM users.*profile_completed = 1`).
|
||||
WithArgs("lalu").
|
||||
WillReturnError(errors.New("synthetic mysql failure"))
|
||||
|
||||
service := NewService(nil, config.Config{}, nil, WithUserDB(userDB))
|
||||
gin.SetMode(gin.TestMode)
|
||||
router := gin.New()
|
||||
var recordedErrors string
|
||||
router.Use(func(c *gin.Context) {
|
||||
c.Request = c.Request.WithContext(appctx.WithContext(c.Request.Context(), "lalu"))
|
||||
c.Set(middleware.ContextPermissions, []string{"overview:view"})
|
||||
c.Next()
|
||||
recordedErrors = c.Errors.String()
|
||||
})
|
||||
protected := router.Group("/api/v1")
|
||||
RegisterRoutes(protected, NewWithService(service))
|
||||
request := httptest.NewRequest(http.MethodGet, "/api/v1/dashboard/user-profile-overview", nil)
|
||||
recorder := httptest.NewRecorder()
|
||||
|
||||
router.ServeHTTP(recorder, request)
|
||||
|
||||
if recorder.Code != http.StatusInternalServerError {
|
||||
t.Fatalf("status mismatch: got=%d body=%s", recorder.Code, recorder.Body.String())
|
||||
}
|
||||
if !strings.Contains(recordedErrors, "query dashboard user profile: synthetic mysql failure") {
|
||||
t.Fatalf("request error was not recorded: %q", recordedErrors)
|
||||
}
|
||||
if err := sqlMock.ExpectationsWereMet(); err != nil {
|
||||
t.Fatalf("sql expectations mismatch: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
func newDashboardUserProfileTestRouter(handler *Handler, permissions []string) *gin.Engine {
|
||||
gin.SetMode(gin.TestMode)
|
||||
router := gin.New()
|
||||
|
||||
Loading…
x
Reference in New Issue
Block a user