679 lines
22 KiB
Go
679 lines
22 KiB
Go
package storage
|
|
|
|
import (
|
|
"context"
|
|
"database/sql"
|
|
"log/slog"
|
|
"strconv"
|
|
"strings"
|
|
"time"
|
|
|
|
"go.mongodb.org/mongo-driver/v2/bson"
|
|
"go.mongodb.org/mongo-driver/v2/mongo"
|
|
"go.mongodb.org/mongo-driver/v2/mongo/options"
|
|
|
|
"dashboard-cdc-worker/internal/dashboard"
|
|
)
|
|
|
|
const activeLogCollection = "user_daily_active_log"
|
|
|
|
type activeLogRow struct {
|
|
UserID int64 `bson:"userId"`
|
|
ActiveDay string `bson:"activeDate"`
|
|
SysOrigin string `bson:"sysOrigin"`
|
|
RegisterCountryCode string `bson:"registerCountryCode"`
|
|
}
|
|
|
|
type activeLogEvent struct {
|
|
UserID int64
|
|
SysOrigin string
|
|
Country dashboard.Country
|
|
}
|
|
|
|
type activeDayStats struct {
|
|
active map[string]map[int64]struct{}
|
|
}
|
|
|
|
type activeRetentionValues struct {
|
|
countryName string
|
|
activeUsers int64
|
|
d1User int64
|
|
d1Base int64
|
|
d7User int64
|
|
d7Base int64
|
|
d30User int64
|
|
d30Base int64
|
|
}
|
|
|
|
type countryCount struct {
|
|
name string
|
|
count int64
|
|
}
|
|
|
|
type retentionCohort struct {
|
|
userID int64
|
|
sysOrigin string
|
|
country dashboard.Country
|
|
registerDate string
|
|
}
|
|
|
|
func (r *MySQLRepository) StartActiveReconciler(ctx context.Context, db *mongo.Database, logger *slog.Logger) {
|
|
if !r.cfg.Active.Enabled || r.cfg.Active.Interval <= 0 {
|
|
return
|
|
}
|
|
run := func() {
|
|
if err := r.RebuildRecentActiveRetention(ctx, db); err != nil {
|
|
if r.metrics != nil {
|
|
r.metrics.RecordError(err)
|
|
}
|
|
logger.Error("dashboard active reconcile failed", "error", err)
|
|
return
|
|
}
|
|
logger.Info("dashboard active reconcile applied", "lookbackDays", r.cfg.Active.LookbackDays)
|
|
}
|
|
go func() {
|
|
run()
|
|
ticker := time.NewTicker(r.cfg.Active.Interval)
|
|
defer ticker.Stop()
|
|
for {
|
|
select {
|
|
case <-ctx.Done():
|
|
return
|
|
case <-ticker.C:
|
|
run()
|
|
}
|
|
}
|
|
}()
|
|
}
|
|
|
|
func (r *MySQLRepository) RebuildRecentActiveRetention(ctx context.Context, db *mongo.Database) error {
|
|
location, err := time.LoadLocation(r.cfg.Dashboard.StorageTimezone)
|
|
if err != nil {
|
|
location = time.UTC
|
|
}
|
|
today := time.Now().In(location)
|
|
end := time.Date(today.Year(), today.Month(), today.Day(), 0, 0, 0, 0, location).AddDate(0, 0, 1)
|
|
start := end.AddDate(0, 0, -r.cfg.Active.LookbackDays)
|
|
return r.RebuildActiveRetentionRange(ctx, db, start, end)
|
|
}
|
|
|
|
func (r *MySQLRepository) RebuildActiveRetentionRange(ctx context.Context, db *mongo.Database, start, end time.Time) error {
|
|
events, err := r.loadActiveLogs(ctx, db, start, end)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
return r.WithDashboardLock(ctx, r.cfg.CDC.BackfillLockWait, func() error {
|
|
return r.WithTx(ctx, func(tx *sql.Tx) error {
|
|
for day := start; day.Before(end); day = day.AddDate(0, 0, 1) {
|
|
if err := r.replaceActiveRetentionDay(ctx, tx, db, day, events); err != nil {
|
|
return err
|
|
}
|
|
}
|
|
for _, statTimezone := range r.cfg.Dashboard.StatTimezones {
|
|
if err := r.rebuildAllPeriodFromDays(ctx, tx, statTimezone); err != nil {
|
|
return err
|
|
}
|
|
}
|
|
return nil
|
|
})
|
|
})
|
|
}
|
|
|
|
func (r *MySQLRepository) loadActiveLogs(ctx context.Context, db *mongo.Database, start, end time.Time) (map[string]map[int64]activeLogEvent, error) {
|
|
filter := bson.D{
|
|
{Key: "activeDate", Value: bson.D{{Key: "$gte", Value: start.Format("2006-01-02")}, {Key: "$lt", Value: end.Format("2006-01-02")}}},
|
|
{Key: "sysOrigin", Value: bson.D{{Key: "$in", Value: r.cfg.Dashboard.SysOrigins}}},
|
|
}
|
|
findOptions := options.Find().SetProjection(bson.D{
|
|
{Key: "userId", Value: 1},
|
|
{Key: "activeDate", Value: 1},
|
|
{Key: "sysOrigin", Value: 1},
|
|
{Key: "registerCountryCode", Value: 1},
|
|
})
|
|
cursor, err := db.Collection(activeLogCollection).Find(ctx, filter, findOptions)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
defer cursor.Close(ctx)
|
|
|
|
events := make(map[string]map[int64]activeLogEvent)
|
|
for cursor.Next(ctx) {
|
|
var row activeLogRow
|
|
if err := cursor.Decode(&row); err != nil {
|
|
return nil, err
|
|
}
|
|
if row.UserID == 0 || row.ActiveDay == "" || !r.allowedOrigin(row.SysOrigin) {
|
|
continue
|
|
}
|
|
users := events[row.ActiveDay]
|
|
if users == nil {
|
|
users = make(map[int64]activeLogEvent)
|
|
events[row.ActiveDay] = users
|
|
}
|
|
users[row.UserID] = activeLogEvent{
|
|
UserID: row.UserID,
|
|
SysOrigin: strings.TrimSpace(row.SysOrigin),
|
|
Country: r.activeLogCountry(row.RegisterCountryCode),
|
|
}
|
|
}
|
|
return events, cursor.Err()
|
|
}
|
|
|
|
func (r *MySQLRepository) activeLogCountry(registerCountryCode string) dashboard.Country {
|
|
code := strings.TrimSpace(registerCountryCode)
|
|
if code == "" {
|
|
code = r.cfg.Dashboard.UnknownCountryCode
|
|
}
|
|
return dashboard.Country{Code: code, Name: code}
|
|
}
|
|
|
|
func collectActiveUserIDs(events map[string]map[int64]activeLogEvent) []int64 {
|
|
seen := make(map[int64]struct{})
|
|
for _, users := range events {
|
|
for userID := range users {
|
|
seen[userID] = struct{}{}
|
|
}
|
|
}
|
|
out := make([]int64, 0, len(seen))
|
|
for userID := range seen {
|
|
out = append(out, userID)
|
|
}
|
|
return out
|
|
}
|
|
|
|
func activeUserSet(events map[int64]activeLogEvent) map[int64]struct{} {
|
|
out := make(map[int64]struct{}, len(events))
|
|
for userID := range events {
|
|
out[userID] = struct{}{}
|
|
}
|
|
return out
|
|
}
|
|
|
|
func stringSetToSlice(values map[string]struct{}) []string {
|
|
out := make([]string, 0, len(values))
|
|
for value := range values {
|
|
out = append(out, value)
|
|
}
|
|
return out
|
|
}
|
|
|
|
func retentionTargetDate(registerDate string, retentionDays int) string {
|
|
parsed, err := time.Parse("2006-01-02", strings.TrimSpace(registerDate))
|
|
if err != nil {
|
|
return ""
|
|
}
|
|
return parsed.AddDate(0, 0, retentionDays).Format("2006-01-02")
|
|
}
|
|
|
|
func activeUserDateKey(userID int64, activeDate string) string {
|
|
return strconv.FormatInt(userID, 10) + "|" + activeDate
|
|
}
|
|
|
|
func (r *MySQLRepository) loadUserCountriesByID(ctx context.Context, ids []int64) (map[int64]dashboard.UserCountry, error) {
|
|
result := make(map[int64]dashboard.UserCountry)
|
|
for start := 0; start < len(ids); start += bulkInsertChunkSize {
|
|
end := min(start+bulkInsertChunkSize, len(ids))
|
|
args := make([]interface{}, 0, end-start)
|
|
for _, id := range ids[start:end] {
|
|
args = append(args, id)
|
|
}
|
|
rows, err := r.db.QueryContext(ctx, `
|
|
SELECT user_id, sys_origin, country_code, country_name, is_deleted, user_created_at, user_updated_at
|
|
FROM dashboard_cdc_user_country
|
|
WHERE user_id IN `+inPlaceholders(end-start), args...)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
for rows.Next() {
|
|
var user dashboard.UserCountry
|
|
var createdAt, updatedAt sql.NullTime
|
|
if err := rows.Scan(&user.UserID, &user.SysOrigin, &user.Country.Code, &user.Country.Name,
|
|
&user.IsDeleted, &createdAt, &updatedAt); err != nil {
|
|
_ = rows.Close()
|
|
return nil, err
|
|
}
|
|
if createdAt.Valid {
|
|
user.CreatedAt = createdAt.Time
|
|
}
|
|
if updatedAt.Valid {
|
|
user.UpdatedAt = updatedAt.Time
|
|
}
|
|
result[user.UserID] = user
|
|
}
|
|
if err := rows.Close(); err != nil {
|
|
return nil, err
|
|
}
|
|
}
|
|
return result, nil
|
|
}
|
|
|
|
func (r *MySQLRepository) replaceActiveRetentionDay(ctx context.Context, tx *sql.Tx, db *mongo.Database,
|
|
day time.Time, events map[string]map[int64]activeLogEvent) error {
|
|
dayKey := day.Format("2006-01-02")
|
|
statsByOrigin := make(map[string]map[string]*activeDayStats)
|
|
for userID, event := range events[dayKey] {
|
|
sysOrigin := strings.TrimSpace(event.SysOrigin)
|
|
if !r.allowedOrigin(sysOrigin) {
|
|
continue
|
|
}
|
|
country := event.Country
|
|
if country.Code == "" {
|
|
country = dashboard.Country{Code: r.cfg.Dashboard.UnknownCountryCode, Name: r.cfg.Dashboard.UnknownCountryCode}
|
|
}
|
|
if country.Name == "" {
|
|
country.Name = country.Code
|
|
}
|
|
stats := statsByOrigin[sysOrigin]
|
|
if stats == nil {
|
|
stats = make(map[string]*activeDayStats)
|
|
statsByOrigin[sysOrigin] = stats
|
|
}
|
|
stat := stats[country.Code]
|
|
if stat == nil {
|
|
stat = &activeDayStats{active: make(map[string]map[int64]struct{})}
|
|
stats[country.Code] = stat
|
|
}
|
|
active := stat.active[country.Name]
|
|
if active == nil {
|
|
active = make(map[int64]struct{})
|
|
stat.active[country.Name] = active
|
|
}
|
|
active[userID] = struct{}{}
|
|
}
|
|
retentionByTimezone, err := r.retentionStatsForDay(ctx, db, day)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
for _, statTimezone := range r.cfg.Dashboard.StatTimezones {
|
|
for _, sysOrigin := range r.cfg.Dashboard.SysOrigins {
|
|
if _, err := tx.ExecContext(ctx, `
|
|
UPDATE country_dashboard_period_metric
|
|
SET daily_active_user = 0,
|
|
d1_retention_user = 0, d1_retention_base_user = 0,
|
|
d7_retention_user = 0, d7_retention_base_user = 0,
|
|
d30_retention_user = 0, d30_retention_base_user = 0,
|
|
refreshed_at = NOW()
|
|
WHERE period_type = 'DAY' AND period_key = ? AND stat_timezone = ? AND sys_origin = ?`, dayKey, statTimezone, sysOrigin); err != nil {
|
|
return err
|
|
}
|
|
if err := r.upsertActiveRetentionRows(ctx, tx, day, statTimezone, sysOrigin,
|
|
statsByOrigin[sysOrigin], retentionByTimezone[statTimezone][sysOrigin]); err != nil {
|
|
return err
|
|
}
|
|
}
|
|
}
|
|
return nil
|
|
}
|
|
|
|
func (r *MySQLRepository) retentionStatsForDay(ctx context.Context, db *mongo.Database,
|
|
day time.Time) (map[string]map[string]map[string]activeRetentionValues, error) {
|
|
out := make(map[string]map[string]map[string]activeRetentionValues)
|
|
for _, statTimezone := range r.cfg.Dashboard.StatTimezones {
|
|
location, err := time.LoadLocation(statTimezone)
|
|
if err != nil {
|
|
location = time.UTC
|
|
}
|
|
statDay := day.In(location)
|
|
dayStart := time.Date(statDay.Year(), statDay.Month(), statDay.Day(), 0, 0, 0, 0, location)
|
|
today := time.Now().In(location).Format("2006-01-02")
|
|
for _, days := range []int{1, 7, 30} {
|
|
cohortStart := dayStart.AddDate(0, 0, -days)
|
|
cohortEnd := cohortStart.AddDate(0, 0, 1)
|
|
cohorts, err := r.loadRetentionCohorts(ctx, cohortStart, cohortEnd, statTimezone)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
if len(cohorts) == 0 {
|
|
continue
|
|
}
|
|
userIDsByOrigin := make(map[string]map[int64]struct{})
|
|
activeDatesByOrigin := make(map[string]map[string]struct{})
|
|
targetByUser := make(map[int64]string, len(cohorts))
|
|
for _, cohort := range cohorts {
|
|
targetDate := retentionTargetDate(cohort.registerDate, days)
|
|
if targetDate == "" || targetDate > today {
|
|
continue
|
|
}
|
|
targetByUser[cohort.userID] = targetDate
|
|
if userIDsByOrigin[cohort.sysOrigin] == nil {
|
|
userIDsByOrigin[cohort.sysOrigin] = make(map[int64]struct{})
|
|
}
|
|
if activeDatesByOrigin[cohort.sysOrigin] == nil {
|
|
activeDatesByOrigin[cohort.sysOrigin] = make(map[string]struct{})
|
|
}
|
|
userIDsByOrigin[cohort.sysOrigin][cohort.userID] = struct{}{}
|
|
activeDatesByOrigin[cohort.sysOrigin][targetDate] = struct{}{}
|
|
}
|
|
activeByOrigin := make(map[string]map[string]struct{})
|
|
for sysOrigin, ids := range userIDsByOrigin {
|
|
activeDates, err := r.listActiveUserDateSet(ctx, db, int64SetToSlice(ids), stringSetToSlice(activeDatesByOrigin[sysOrigin]), sysOrigin)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
activeByOrigin[sysOrigin] = activeDates
|
|
}
|
|
for _, cohort := range cohorts {
|
|
targetDate := targetByUser[cohort.userID]
|
|
if targetDate == "" {
|
|
continue
|
|
}
|
|
timezoneOut := out[statTimezone]
|
|
if timezoneOut == nil {
|
|
timezoneOut = make(map[string]map[string]activeRetentionValues)
|
|
out[statTimezone] = timezoneOut
|
|
}
|
|
originOut := timezoneOut[cohort.sysOrigin]
|
|
if originOut == nil {
|
|
originOut = make(map[string]activeRetentionValues)
|
|
timezoneOut[cohort.sysOrigin] = originOut
|
|
}
|
|
country := cohort.country.Code
|
|
if country == "" {
|
|
country = r.cfg.Dashboard.UnknownCountryCode
|
|
}
|
|
stats := originOut[country]
|
|
if stats.countryName == "" {
|
|
stats.countryName = cohort.country.Name
|
|
if stats.countryName == "" {
|
|
stats.countryName = country
|
|
}
|
|
}
|
|
retained := false
|
|
if activeByOrigin[cohort.sysOrigin] != nil {
|
|
_, retained = activeByOrigin[cohort.sysOrigin][activeUserDateKey(cohort.userID, targetDate)]
|
|
}
|
|
switch days {
|
|
case 1:
|
|
stats.d1Base++
|
|
if retained {
|
|
stats.d1User++
|
|
}
|
|
case 7:
|
|
stats.d7Base++
|
|
if retained {
|
|
stats.d7User++
|
|
}
|
|
case 30:
|
|
stats.d30Base++
|
|
if retained {
|
|
stats.d30User++
|
|
}
|
|
}
|
|
originOut[country] = stats
|
|
}
|
|
}
|
|
}
|
|
return out, nil
|
|
}
|
|
|
|
func (r *MySQLRepository) loadRetentionBase(ctx context.Context, cohortDay time.Time) (map[string]map[string]countryCount, error) {
|
|
args := stringArgs(r.cfg.Dashboard.SysOrigins)
|
|
args = append(args, cohortDay, cohortDay.AddDate(0, 0, 1))
|
|
rows, err := r.db.QueryContext(ctx, `
|
|
SELECT sys_origin, country_code, MAX(country_name), COUNT(1)
|
|
FROM dashboard_cdc_user_country
|
|
WHERE is_deleted = 0
|
|
AND sys_origin IN `+inStringPlaceholders(len(r.cfg.Dashboard.SysOrigins))+`
|
|
AND user_created_at >= ?
|
|
AND user_created_at < ?
|
|
GROUP BY sys_origin, country_code`, args...)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
defer rows.Close()
|
|
result := make(map[string]map[string]countryCount)
|
|
for rows.Next() {
|
|
var origin, country, name string
|
|
var count int64
|
|
if err := rows.Scan(&origin, &country, &name, &count); err != nil {
|
|
return nil, err
|
|
}
|
|
countries := result[origin]
|
|
if countries == nil {
|
|
countries = make(map[string]countryCount)
|
|
result[origin] = countries
|
|
}
|
|
countries[country] = countryCount{name: name, count: count}
|
|
}
|
|
return result, rows.Err()
|
|
}
|
|
|
|
func (r *MySQLRepository) loadRetentionCohorts(ctx context.Context, start, end time.Time,
|
|
statTimezone string) ([]retentionCohort, error) {
|
|
args := []interface{}{timezoneOffset(statTimezone)}
|
|
args = append(args, stringArgs(r.cfg.Dashboard.SysOrigins)...)
|
|
args = append(args, start, end)
|
|
rows, err := r.db.QueryContext(ctx, `
|
|
SELECT u.id,
|
|
u.origin_sys,
|
|
NULLIF(u.country_code, ''),
|
|
NULLIF(u.country_name, ''),
|
|
DATE_FORMAT(CONVERT_TZ(u.create_time, '+03:00', ?), '%Y-%m-%d') AS register_date
|
|
FROM `+quoteIdent(r.cfg.MySQL.Schema)+`.user_base_info u
|
|
WHERE (u.is_del = 0 OR u.is_del IS NULL)
|
|
AND u.origin_sys IN `+inStringPlaceholders(len(r.cfg.Dashboard.SysOrigins))+`
|
|
AND u.create_time >= ?
|
|
AND u.create_time < ?`, args...)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
defer rows.Close()
|
|
var result []retentionCohort
|
|
for rows.Next() {
|
|
var item retentionCohort
|
|
var countryCode, countryName sql.NullString
|
|
if err := rows.Scan(&item.userID, &item.sysOrigin, &countryCode, &countryName, &item.registerDate); err != nil {
|
|
return nil, err
|
|
}
|
|
item.sysOrigin = strings.TrimSpace(item.sysOrigin)
|
|
if !r.allowedOrigin(item.sysOrigin) || item.userID == 0 || item.registerDate == "" {
|
|
continue
|
|
}
|
|
if countryCode.Valid && strings.TrimSpace(countryCode.String) != "" {
|
|
item.country.Code = strings.TrimSpace(countryCode.String)
|
|
} else {
|
|
item.country.Code = r.cfg.Dashboard.UnknownCountryCode
|
|
}
|
|
if countryName.Valid && strings.TrimSpace(countryName.String) != "" {
|
|
item.country.Name = strings.TrimSpace(countryName.String)
|
|
} else {
|
|
item.country.Name = item.country.Code
|
|
}
|
|
result = append(result, item)
|
|
}
|
|
return result, rows.Err()
|
|
}
|
|
|
|
func (r *MySQLRepository) listActiveUserDateSet(ctx context.Context, db *mongo.Database,
|
|
userIDs []int64, activeDates []string, sysOrigin string) (map[string]struct{}, error) {
|
|
result := make(map[string]struct{})
|
|
if len(userIDs) == 0 || len(activeDates) == 0 {
|
|
return result, nil
|
|
}
|
|
for start := 0; start < len(userIDs); start += bulkInsertChunkSize {
|
|
end := min(start+bulkInsertChunkSize, len(userIDs))
|
|
filter := bson.D{
|
|
{Key: "userId", Value: bson.D{{Key: "$in", Value: userIDs[start:end]}}},
|
|
{Key: "activeDate", Value: bson.D{{Key: "$in", Value: activeDates}}},
|
|
{Key: "sysOrigin", Value: sysOrigin},
|
|
}
|
|
cursor, err := db.Collection(activeLogCollection).Find(ctx, filter, options.Find().SetProjection(bson.D{
|
|
{Key: "userId", Value: 1},
|
|
{Key: "activeDate", Value: 1},
|
|
}))
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
for cursor.Next(ctx) {
|
|
var row activeLogRow
|
|
if err := cursor.Decode(&row); err != nil {
|
|
_ = cursor.Close(ctx)
|
|
return nil, err
|
|
}
|
|
if row.UserID != 0 && row.ActiveDay != "" {
|
|
result[activeUserDateKey(row.UserID, row.ActiveDay)] = struct{}{}
|
|
}
|
|
}
|
|
if err := cursor.Close(ctx); err != nil {
|
|
return nil, err
|
|
}
|
|
}
|
|
return result, nil
|
|
}
|
|
|
|
func (r *MySQLRepository) upsertActiveRetentionRows(ctx context.Context, tx *sql.Tx, day time.Time,
|
|
statTimezone string, sysOrigin string, active map[string]*activeDayStats, retention map[string]activeRetentionValues) error {
|
|
values := make(map[string]activeRetentionValues)
|
|
for country, stat := range active {
|
|
item := values[country]
|
|
for name, users := range stat.active {
|
|
item.countryName = name
|
|
item.activeUsers += int64(len(users))
|
|
}
|
|
values[country] = item
|
|
}
|
|
for country, item := range retention {
|
|
existing := values[country]
|
|
if existing.countryName == "" {
|
|
existing.countryName = item.countryName
|
|
}
|
|
existing.d1User = item.d1User
|
|
existing.d1Base = item.d1Base
|
|
existing.d7User = item.d7User
|
|
existing.d7Base = item.d7Base
|
|
existing.d30User = item.d30User
|
|
existing.d30Base = item.d30Base
|
|
values[country] = existing
|
|
}
|
|
if len(values) == 0 {
|
|
return nil
|
|
}
|
|
dayKey := day.Format("2006-01-02")
|
|
args := make([]interface{}, 0, len(values)*16)
|
|
for country, item := range values {
|
|
name := item.countryName
|
|
if name == "" {
|
|
name = country
|
|
}
|
|
args = append(args, dashboard.PeriodDay, dayKey, statTimezone, dayKey, day,
|
|
day.AddDate(0, 0, 1), sysOrigin, country, name, item.activeUsers,
|
|
item.d1User, item.d1Base, item.d7User, item.d7Base, item.d30User, item.d30Base)
|
|
}
|
|
_, err := tx.ExecContext(ctx, `
|
|
INSERT INTO country_dashboard_period_metric (
|
|
period_type, period_key, stat_timezone, period_name, period_start_date, period_end_date,
|
|
sys_origin, country_code, country_name, daily_active_user,
|
|
d1_retention_user, d1_retention_base_user,
|
|
d7_retention_user, d7_retention_base_user,
|
|
d30_retention_user, d30_retention_base_user
|
|
) VALUES `+placeholders(len(values), 16)+`
|
|
ON DUPLICATE KEY UPDATE
|
|
period_name = VALUES(period_name),
|
|
period_start_date = VALUES(period_start_date),
|
|
period_end_date = VALUES(period_end_date),
|
|
country_name = VALUES(country_name),
|
|
daily_active_user = VALUES(daily_active_user),
|
|
d1_retention_user = VALUES(d1_retention_user),
|
|
d1_retention_base_user = VALUES(d1_retention_base_user),
|
|
d7_retention_user = VALUES(d7_retention_user),
|
|
d7_retention_base_user = VALUES(d7_retention_base_user),
|
|
d30_retention_user = VALUES(d30_retention_user),
|
|
d30_retention_base_user = VALUES(d30_retention_base_user),
|
|
refreshed_at = NOW()`, args...)
|
|
return err
|
|
}
|
|
|
|
func (r *MySQLRepository) rebuildAllPeriodFromDays(ctx context.Context, tx *sql.Tx, statTimezone string) error {
|
|
for _, sysOrigin := range r.cfg.Dashboard.SysOrigins {
|
|
if _, err := tx.ExecContext(ctx, `
|
|
DELETE FROM country_dashboard_period_user_metric
|
|
WHERE period_type = 'ALL' AND period_key = 'ALL' AND stat_timezone = ? AND sys_origin = ?`, statTimezone, sysOrigin); err != nil {
|
|
return err
|
|
}
|
|
if _, err := tx.ExecContext(ctx, `
|
|
INSERT IGNORE INTO country_dashboard_period_user_metric (
|
|
period_type, period_key, stat_timezone, period_name, period_start_date, period_end_date,
|
|
sys_origin, country_code, country_name, user_id, metric_type
|
|
)
|
|
SELECT DISTINCT 'ALL', 'ALL', ?, '全部', NULL, NULL,
|
|
sys_origin, country_code, country_name, user_id, metric_type
|
|
FROM country_dashboard_period_user_metric
|
|
WHERE period_type = 'DAY' AND stat_timezone = ? AND sys_origin = ?`, statTimezone, statTimezone, sysOrigin); err != nil {
|
|
return err
|
|
}
|
|
if _, err := tx.ExecContext(ctx, `
|
|
DELETE FROM country_dashboard_period_metric
|
|
WHERE period_type = 'ALL' AND period_key = 'ALL' AND stat_timezone = ? AND sys_origin = ?`, statTimezone, sysOrigin); err != nil {
|
|
return err
|
|
}
|
|
if _, err := tx.ExecContext(ctx, `
|
|
INSERT INTO country_dashboard_period_metric (
|
|
period_type, period_key, stat_timezone, period_name, period_start_date, period_end_date,
|
|
sys_origin, country_code, country_name,
|
|
country_new_user, daily_active_user, new_user_recharge, official_recharge, mifapay_recharge,
|
|
google_recharge, dealer_recharge, salary_exchange, salary_transfer, gift_consume,
|
|
lucky_gift_total_flow, lucky_gift_user, lucky_gift_payout, lucky_gift_anchor_share,
|
|
game_total_flow, game_user, game_payout,
|
|
d1_retention_user, d1_retention_base_user, d7_retention_user, d7_retention_base_user,
|
|
d30_retention_user, d30_retention_base_user, refreshed_at
|
|
)
|
|
SELECT 'ALL', 'ALL', ?, '全部', NULL, NULL,
|
|
d.sys_origin, d.country_code, MAX(d.country_name),
|
|
IFNULL(MAX(u.country_new_user), 0),
|
|
IFNULL(SUM(d.daily_active_user), 0),
|
|
IFNULL(SUM(d.new_user_recharge), 0),
|
|
IFNULL(SUM(d.official_recharge), 0),
|
|
IFNULL(SUM(d.mifapay_recharge), 0),
|
|
IFNULL(SUM(d.google_recharge), 0),
|
|
IFNULL(SUM(d.dealer_recharge), 0),
|
|
IFNULL(SUM(d.salary_exchange), 0),
|
|
IFNULL(SUM(d.salary_transfer), 0),
|
|
IFNULL(SUM(d.gift_consume), 0),
|
|
IFNULL(SUM(d.lucky_gift_total_flow), 0),
|
|
IFNULL(MAX(u.lucky_gift_user), 0),
|
|
IFNULL(SUM(d.lucky_gift_payout), 0),
|
|
IFNULL(SUM(d.lucky_gift_anchor_share), 0),
|
|
IFNULL(SUM(d.game_total_flow), 0),
|
|
IFNULL(MAX(u.game_user), 0),
|
|
IFNULL(SUM(d.game_payout), 0),
|
|
IFNULL(SUM(d.d1_retention_user), 0),
|
|
IFNULL(SUM(d.d1_retention_base_user), 0),
|
|
IFNULL(SUM(d.d7_retention_user), 0),
|
|
IFNULL(SUM(d.d7_retention_base_user), 0),
|
|
IFNULL(SUM(d.d30_retention_user), 0),
|
|
IFNULL(SUM(d.d30_retention_base_user), 0),
|
|
NOW()
|
|
FROM country_dashboard_period_metric d
|
|
LEFT JOIN (
|
|
SELECT country_code,
|
|
SUM(metric_type = 'COUNTRY_NEW_USER') AS country_new_user,
|
|
SUM(metric_type = 'LUCKY_GIFT_USER') AS lucky_gift_user,
|
|
SUM(metric_type = 'GAME_USER') AS game_user
|
|
FROM country_dashboard_period_user_metric
|
|
WHERE period_type = 'ALL' AND period_key = 'ALL' AND stat_timezone = ? AND sys_origin = ?
|
|
GROUP BY country_code
|
|
) u ON u.country_code = d.country_code
|
|
WHERE d.period_type = 'DAY' AND d.stat_timezone = ? AND d.sys_origin = ?
|
|
GROUP BY d.sys_origin, d.country_code`, statTimezone, statTimezone, sysOrigin, statTimezone, sysOrigin); err != nil {
|
|
return err
|
|
}
|
|
}
|
|
return nil
|
|
}
|
|
|
|
func inPlaceholders(count int) string {
|
|
return "(" + placeholders(count, 1) + ")"
|
|
}
|
|
|
|
func inStringPlaceholders(count int) string {
|
|
return "(" + placeholders(count, 1) + ")"
|
|
}
|
|
|
|
func stringArgs(values []string) []interface{} {
|
|
args := make([]interface{}, 0, len(values))
|
|
for _, value := range values {
|
|
args = append(args, value)
|
|
}
|
|
return args
|
|
}
|