hy/yumi-java
2
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

feat(console): precompute country dashboard metrics

Browse Source
This commit is contained in:
hy001 2026-05-07 15:00:37 +08:00
parent ef3b8c90e5
commit ab515c122c
11 changed files with 1718 additions and 34 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

44
rc-service/rc-service-console/console-adapter/src/main/java/com/red/circle/console/adapter/app/user/DatavRestController.java
View File

@ -1,25 +1,29 @@
package com.red.circle.console.adapter.app.user;
import com.red.circle.common.business.core.enums.SysOriginPlatformEnum;
import com.red.circle.console.app.dto.clientobject.datav.CountryDashboardCO;
import com.red.circle.console.app.dto.clientobject.datav.CountryDashboardRechargeDetailCO;
import com.red.circle.console.app.dto.cmd.datav.CountryDashboardQryCmd;
import com.red.circle.console.app.dto.cmd.datav.CountryDashboardRechargeDetailQryCmd;
import com.red.circle.console.app.service.app.datav.CountryDashboardDailyMetricService;
import com.red.circle.console.app.service.app.datav.CountryDashboardService;
import com.red.circle.console.app.service.app.user.DatavService;
import com.red.circle.framework.dto.PageResult;
import com.red.circle.framework.web.controller.BaseController;
import com.red.circle.other.inner.model.cmd.user.OnlineCountQryCmd;
import com.red.circle.other.inner.model.dto.user.ActiveUserCountryCodeDTO;
import java.time.LocalDateTime;
import java.util.List;
import java.util.Objects;
import lombok.RequiredArgsConstructor;
import org.springframework.http.MediaType;
import org.springframework.web.bind.annotation.GetMapping;
import org.springframework.web.bind.annotation.RequestMapping;
import org.springframework.web.bind.annotation.RequestParam;
import org.springframework.web.bind.annotation.RestController;
import com.red.circle.framework.web.controller.BaseController;
import com.red.circle.other.inner.model.cmd.user.OnlineCountQryCmd;
import com.red.circle.other.inner.model.dto.user.ActiveUserCountryCodeDTO;
import java.time.LocalDate;
import java.time.LocalDateTime;
import java.util.List;
import java.util.Map;
import java.util.Objects;
import lombok.RequiredArgsConstructor;
import org.springframework.format.annotation.DateTimeFormat;
import org.springframework.http.MediaType;
import org.springframework.web.bind.annotation.GetMapping;
import org.springframework.web.bind.annotation.PostMapping;
import org.springframework.web.bind.annotation.RequestMapping;
import org.springframework.web.bind.annotation.RequestParam;
import org.springframework.web.bind.annotation.RestController;
/**
* @author pengliang on 2020/10/16
@ -32,6 +36,7 @@ public class DatavRestController extends BaseController {
private final DatavService userExpandService;
private final CountryDashboardService countryDashboardService;
private final CountryDashboardDailyMetricService countryDashboardDailyMetricService;
@GetMapping("/active/user-country-code")
public List<ActiveUserCountryCodeDTO> latestActiveUserCountryCode() {
@ -59,4 +64,17 @@ public class DatavRestController extends BaseController {
return countryDashboardService.pageRechargeDetails(cmd);
}
@PostMapping("/country-dashboard/backfill")
public Map<String, Object> backfillCountryDashboard(
@RequestParam @DateTimeFormat(iso = DateTimeFormat.ISO.DATE) LocalDate startDate,
@RequestParam @DateTimeFormat(iso = DateTimeFormat.ISO.DATE) LocalDate endDate,
@RequestParam(value = "sysOrigin", defaultValue = "LIKEI") String sysOrigin) {
countryDashboardDailyMetricService.refreshRange(startDate, endDate, sysOrigin);
return Map.of(
"status", "success",
"startDate", startDate.toString(),
"endDate", endDate.toString(),
"sysOrigin", sysOrigin);
}
}

46
rc-service/rc-service-console/console-application/src/main/java/com/red/circle/console/app/scheduler/CountryDashboardDailyMetricTask.java Normal file
View File

@ -0,0 +1,46 @@
package com.red.circle.console.app.scheduler;
import com.red.circle.component.redis.annotation.TaskCacheLock;
import com.red.circle.console.app.service.app.datav.CountryDashboardDailyMetricService;
import lombok.RequiredArgsConstructor;
import lombok.extern.slf4j.Slf4j;
import org.springframework.beans.factory.annotation.Value;
import org.springframework.scheduling.annotation.Scheduled;
import org.springframework.stereotype.Component;
/**
* 国家数据大屏日预聚合任务.
*/
@Slf4j
@Component
@RequiredArgsConstructor
public class CountryDashboardDailyMetricTask {
private final CountryDashboardDailyMetricService countryDashboardDailyMetricService;
@Value("${red-circle.country-dashboard.refresh-today-days:1}")
private int refreshTodayDays;
@Value("${red-circle.country-dashboard.refresh-recent-days:8}")
private int refreshRecentDays;
@Scheduled(cron = "${red-circle.country-dashboard.refresh-today-cron:0 */10 * * * ?}",
zone = "Asia/Shanghai")
@TaskCacheLock(key = "COUNTRY_DASHBOARD_DAILY_METRIC_REFRESH_TODAY", expireSecond = 60 * 8)
public void refreshToday() {
long start = System.currentTimeMillis();
countryDashboardDailyMetricService.refreshRecentDays(refreshTodayDays);
log.info("country dashboard refresh today task finished, days={}, costMs={}",
refreshTodayDays, System.currentTimeMillis() - start);
}
@Scheduled(cron = "${red-circle.country-dashboard.refresh-recent-cron:0 20 0 * * ?}",
zone = "Asia/Shanghai")
@TaskCacheLock(key = "COUNTRY_DASHBOARD_DAILY_METRIC_REFRESH_RECENT", expireSecond = 60 * 60)
public void refreshRecent() {
long start = System.currentTimeMillis();
countryDashboardDailyMetricService.refreshRecentDays(refreshRecentDays);
log.info("country dashboard refresh recent task finished, days={}, costMs={}",
refreshRecentDays, System.currentTimeMillis() - start);
}
}

438
rc-service/rc-service-console/console-application/src/main/java/com/red/circle/console/app/service/app/datav/CountryDashboardDailyMetricService.java Normal file
View File

@ -0,0 +1,438 @@
package com.red.circle.console.app.service.app.datav;
import com.red.circle.common.business.core.enums.SysOriginPlatformEnum;
import com.red.circle.console.infra.database.rds.dao.datav.CountryDashboardDAO;
import com.red.circle.console.infra.database.rds.dto.datav.CountryDashboardDailyMetricDTO;
import com.red.circle.console.infra.database.rds.dto.datav.CountryDashboardMetricDTO;
import com.red.circle.console.infra.database.rds.dto.datav.CountryDashboardPeriodMetricDTO;
import java.math.BigDecimal;
import java.sql.Timestamp;
import java.time.DayOfWeek;
import java.time.LocalDate;
import java.time.LocalDateTime;
import java.time.format.DateTimeFormatter;
import java.time.temporal.ChronoUnit;
import java.time.temporal.WeekFields;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.LinkedHashSet;
import java.util.LinkedHashMap;
import java.util.List;
import java.util.Locale;
import java.util.Map;
import java.util.Objects;
import java.util.Set;
import lombok.RequiredArgsConstructor;
import lombok.extern.slf4j.Slf4j;
import org.springframework.beans.factory.annotation.Value;
import org.springframework.stereotype.Service;
import org.springframework.transaction.support.TransactionTemplate;
/**
* 国家数据大屏日预聚合.
*/
@Slf4j
@Service
@RequiredArgsConstructor
public class CountryDashboardDailyMetricService {
private static final String PERIOD_DAY = "DAY";
private static final String PERIOD_WEEK = "WEEK";
private static final String PERIOD_MONTH = "MONTH";
private static final String PERIOD_ALL = "ALL";
private static final DateTimeFormatter DATE_NUMBER_FORMATTER = DateTimeFormatter.BASIC_ISO_DATE;
private static final DateTimeFormatter DATE_FORMATTER = DateTimeFormatter.ofPattern("yyyy-MM-dd");
private static final DateTimeFormatter MONTH_FORMATTER = DateTimeFormatter.ofPattern("yyyy-MM");
private static final WeekFields WEEK_FIELDS = WeekFields.ISO;
private final CountryDashboardDAO countryDashboardDAO;
private final TransactionTemplate transactionTemplate;
@Value("${red-circle.country-dashboard.refresh-sys-origins:LIKEI}")
private String refreshSysOrigins;
private volatile boolean metricSchemaEnsured;
public void refreshRecentDays(int days) {
ensureMetricSchema();
int safeDays = Math.max(days, 1);
LocalDate today = LocalDate.now();
List<LocalDate> statDates = new ArrayList<>();
for (int i = safeDays - 1; i >= 0; i--) {
statDates.add(today.minusDays(i));
}
List<String> sysOrigins = resolveSysOrigins();
for (String sysOrigin : sysOrigins) {
for (LocalDate statDate : statDates) {
try {
refreshDay(statDate, sysOrigin);
} catch (Exception ex) {
log.error("refresh country dashboard daily metric failed, statDate={}, sysOrigin={}",
statDate, sysOrigin, ex);
}
}
refreshAffectedPeriods(statDates, sysOrigin, false);
try {
refreshAll(sysOrigin);
} catch (Exception ex) {
log.error("refresh country dashboard all metric failed, sysOrigin={}", sysOrigin, ex);
}
}
}
public void refreshRange(LocalDate startDate, LocalDate endDate, String sysOrigin) {
ensureMetricSchema();
if (startDate == null || endDate == null) {
throw new IllegalArgumentException("startDate and endDate are required");
}
if (endDate.isBefore(startDate)) {
throw new IllegalArgumentException("endDate must be greater than or equal to startDate");
}
long dayCount = ChronoUnit.DAYS.between(startDate, endDate) + 1;
if (dayCount > 366) {
throw new IllegalArgumentException("backfill range can not exceed 366 days");
}
String safeSysOrigin = trimToNull(sysOrigin);
if (safeSysOrigin == null) {
safeSysOrigin = SysOriginPlatformEnum.LIKEI.name();
}
long start = System.currentTimeMillis();
List<LocalDate> statDates = new ArrayList<>();
for (long i = 0; i < dayCount; i++) {
statDates.add(startDate.plusDays(i));
}
log.info("refresh country dashboard range start, startDate={}, endDate={}, days={}, sysOrigin={}",
startDate, endDate, dayCount, safeSysOrigin);
for (LocalDate statDate : statDates) {
refreshDay(statDate, safeSysOrigin);
}
refreshAffectedPeriods(statDates, safeSysOrigin, true);
refreshAll(safeSysOrigin);
log.info("refresh country dashboard range success, startDate={}, endDate={}, days={}, sysOrigin={}, costMs={}",
startDate, endDate, dayCount, safeSysOrigin, System.currentTimeMillis() - start);
}
private void ensureMetricSchema() {
if (metricSchemaEnsured) {
return;
}
synchronized (this) {
if (metricSchemaEnsured) {
return;
}
countryDashboardDAO.createDailyMetricTable();
countryDashboardDAO.createPeriodMetricTable();
countryDashboardDAO.createPeriodUserMetricTable();
metricSchemaEnsured = true;
log.info("country dashboard metric tables ensured");
}
}
public void refreshDay(LocalDate statDate, String sysOrigin) {
long start = System.currentTimeMillis();
LocalDateTime startTime = statDate.atStartOfDay();
LocalDateTime endTime = statDate.plusDays(1).atStartOfDay();
Long startTimeMillis = Timestamp.valueOf(startTime).getTime();
Long endTimeMillis = Timestamp.valueOf(endTime).getTime();
Map<String, CountryDashboardDailyMetricDTO> rows = new LinkedHashMap<>();
mergeSql(rows, countryDashboardDAO.listNewUsers(
PERIOD_DAY, startTime, endTime, null, sysOrigin), statDate, sysOrigin);
mergeSql(rows, countryDashboardDAO.listNewUserRecharge(
PERIOD_DAY, startTime, endTime, null, sysOrigin), statDate, sysOrigin);
mergeSql(rows, countryDashboardDAO.listOfficialRecharge(
PERIOD_DAY, startTime, endTime, null, sysOrigin), statDate, sysOrigin);
mergeSql(rows, countryDashboardDAO.listMifapayRecharge(
PERIOD_DAY, startTime, endTime, null, sysOrigin), statDate, sysOrigin);
mergeSql(rows, countryDashboardDAO.listGoogleRecharge(
PERIOD_DAY, startTime, endTime, null, sysOrigin), statDate, sysOrigin);
mergeSql(rows, countryDashboardDAO.listDealerRecharge(
PERIOD_DAY, startTime, endTime, null, sysOrigin), statDate, sysOrigin);
mergeSql(rows, countryDashboardDAO.listSalaryExchange(
PERIOD_DAY, startTime, endTime, null, sysOrigin), statDate, sysOrigin);
mergeSql(rows, countryDashboardDAO.listLuckyGift(
PERIOD_DAY, startTime, endTime, null, sysOrigin), statDate, sysOrigin);
mergeSql(rows, countryDashboardDAO.listWalletGame(
PERIOD_DAY, startTime, endTime, startTimeMillis, endTimeMillis, null, sysOrigin),
statDate, sysOrigin);
mergeSql(rows, countryDashboardDAO.listLuckyBoxGame(
PERIOD_DAY, startTime, endTime, null, sysOrigin), statDate, sysOrigin);
mergeSql(rows, countryDashboardDAO.listGameUser(
PERIOD_DAY, startTime, endTime, startTimeMillis, endTimeMillis, null, sysOrigin),
statDate, sysOrigin);
List<CountryDashboardDailyMetricDTO> metrics = new ArrayList<>(rows.values());
RefreshWriteResult writeResult = transactionTemplate.execute(status -> {
int deleted = countryDashboardDAO.deleteDailyMetrics(statDate, sysOrigin);
int inserted = metrics.isEmpty() ? 0 : countryDashboardDAO.upsertDailyMetrics(metrics);
return new RefreshWriteResult(deleted, inserted, 0);
});
log.info("refresh country dashboard daily metric success, statDate={}, sysOrigin={}, countries={}, deleted={}, inserted={}, costMs={}",
statDate, sysOrigin, metrics.size(), writeResult.deleted(), writeResult.inserted(),
System.currentTimeMillis() - start);
refreshPeriod(toDayWindow(statDate), sysOrigin);
}
private void refreshAffectedPeriods(List<LocalDate> statDates, String sysOrigin,
boolean failFast) {
Set<PeriodWindow> windows = new LinkedHashSet<>();
for (LocalDate statDate : statDates) {
windows.add(toWeekWindow(statDate));
windows.add(toMonthWindow(statDate));
}
for (PeriodWindow window : windows) {
try {
refreshPeriod(window, sysOrigin);
} catch (Exception ex) {
if (failFast) {
throw ex;
}
log.error("refresh country dashboard period metric failed, periodType={}, periodKey={}, sysOrigin={}",
window.periodType(), window.periodKey(), sysOrigin, ex);
}
}
}
private void refreshPeriod(PeriodWindow window, String sysOrigin) {
long start = System.currentTimeMillis();
LocalDateTime startTime = window.periodStartDate().atStartOfDay();
LocalDateTime endTime = window.periodEndDate().atStartOfDay();
Long startTimeMillis = Timestamp.valueOf(startTime).getTime();
Long endTimeMillis = Timestamp.valueOf(endTime).getTime();
Map<String, CountryDashboardPeriodMetricDTO> rows = new LinkedHashMap<>();
mergePeriodAmounts(rows, countryDashboardDAO.listNewUserRecharge(
window.periodType(), startTime, endTime, null, sysOrigin), window, sysOrigin);
mergePeriodAmounts(rows, countryDashboardDAO.listOfficialRecharge(
window.periodType(), startTime, endTime, null, sysOrigin), window, sysOrigin);
mergePeriodAmounts(rows, countryDashboardDAO.listMifapayRecharge(
window.periodType(), startTime, endTime, null, sysOrigin), window, sysOrigin);
mergePeriodAmounts(rows, countryDashboardDAO.listGoogleRecharge(
window.periodType(), startTime, endTime, null, sysOrigin), window, sysOrigin);
mergePeriodAmounts(rows, countryDashboardDAO.listDealerRecharge(
window.periodType(), startTime, endTime, null, sysOrigin), window, sysOrigin);
mergePeriodAmounts(rows, countryDashboardDAO.listSalaryExchange(
window.periodType(), startTime, endTime, null, sysOrigin), window, sysOrigin);
mergePeriodAmounts(rows, countryDashboardDAO.listLuckyGift(
window.periodType(), startTime, endTime, null, sysOrigin), window, sysOrigin);
mergePeriodAmounts(rows, countryDashboardDAO.listWalletGame(
window.periodType(), startTime, endTime, startTimeMillis, endTimeMillis, null, sysOrigin),
window, sysOrigin);
mergePeriodAmounts(rows, countryDashboardDAO.listLuckyBoxGame(
window.periodType(), startTime, endTime, null, sysOrigin), window, sysOrigin);
RefreshWriteResult writeResult = transactionTemplate.execute(status -> {
countryDashboardDAO.deletePeriodUserMetrics(window.periodType(), window.periodKey(), sysOrigin);
int newUsers = countryDashboardDAO.insertPeriodNewUserMetrics(
window.periodType(), window.periodKey(), window.periodName(),
window.periodStartDate(), window.periodEndDate(), startTime, endTime, sysOrigin);
int luckyGiftUsers = countryDashboardDAO.insertPeriodLuckyGiftUserMetrics(
window.periodType(), window.periodKey(), window.periodName(),
window.periodStartDate(), window.periodEndDate(), startTime, endTime, sysOrigin);
int gameUsers = countryDashboardDAO.insertPeriodGameUserMetrics(
window.periodType(), window.periodKey(), window.periodName(),
window.periodStartDate(), window.periodEndDate(), startTime, endTime,
startTimeMillis, endTimeMillis, sysOrigin);
mergePeriodUserCounts(rows, countryDashboardDAO.listPeriodUserMetricCounts(
window.periodType(), window.periodKey(), sysOrigin), window, sysOrigin);
List<CountryDashboardPeriodMetricDTO> metrics = new ArrayList<>(rows.values());
int deleted = countryDashboardDAO.deletePeriodMetrics(window.periodType(), window.periodKey(), sysOrigin);
int inserted = metrics.isEmpty() ? 0 : countryDashboardDAO.upsertPeriodMetrics(metrics);
return new RefreshWriteResult(deleted, inserted, newUsers + luckyGiftUsers + gameUsers);
});
log.info("refresh country dashboard period metric success, periodType={}, periodKey={}, sysOrigin={}, countries={}, deleted={}, inserted={}, userRows={}, costMs={}",
window.periodType(), window.periodKey(), sysOrigin, rows.size(), writeResult.deleted(),
writeResult.inserted(), writeResult.userRows(), System.currentTimeMillis() - start);
}
private void refreshAll(String sysOrigin) {
long start = System.currentTimeMillis();
PeriodWindow window = new PeriodWindow(PERIOD_ALL, PERIOD_ALL, "全部", null, null);
Map<String, CountryDashboardPeriodMetricDTO> rows = new LinkedHashMap<>();
mergePeriodAmounts(rows, countryDashboardDAO.listAllPeriodAmountsFromDaily(sysOrigin),
window, sysOrigin);
RefreshWriteResult writeResult = transactionTemplate.execute(status -> {
countryDashboardDAO.deletePeriodUserMetrics(PERIOD_ALL, PERIOD_ALL, sysOrigin);
int userRows = countryDashboardDAO.insertAllPeriodUserMetricsFromDaily(sysOrigin);
mergePeriodUserCounts(rows, countryDashboardDAO.listPeriodUserMetricCounts(
PERIOD_ALL, PERIOD_ALL, sysOrigin), window, sysOrigin);
List<CountryDashboardPeriodMetricDTO> metrics = new ArrayList<>(rows.values());
int deleted = countryDashboardDAO.deletePeriodMetrics(PERIOD_ALL, PERIOD_ALL, sysOrigin);
int inserted = metrics.isEmpty() ? 0 : countryDashboardDAO.upsertPeriodMetrics(metrics);
return new RefreshWriteResult(deleted, inserted, userRows);
});
log.info("refresh country dashboard all metric success, sysOrigin={}, countries={}, deleted={}, inserted={}, userRows={}, costMs={}",
sysOrigin, rows.size(), writeResult.deleted(), writeResult.inserted(),
writeResult.userRows(), System.currentTimeMillis() - start);
}
private void mergeSql(Map<String, CountryDashboardDailyMetricDTO> rows,
List<CountryDashboardMetricDTO> metrics, LocalDate statDate, String sysOrigin) {
if (metrics == null || metrics.isEmpty()) {
return;
}
for (CountryDashboardMetricDTO metric : metrics) {
CountryDashboardDailyMetricDTO row = getRow(rows, metric, statDate, sysOrigin);
row.setCountryNewUser(safeLong(row.getCountryNewUser()) + safeLong(metric.getCountryNewUser()));
row.setNewUserRecharge(add(row.getNewUserRecharge(), metric.getNewUserRecharge()));
row.setOfficialRecharge(add(row.getOfficialRecharge(), metric.getOfficialRecharge()));
row.setMifapayRecharge(add(row.getMifapayRecharge(), metric.getMifapayRecharge()));
row.setGoogleRecharge(add(row.getGoogleRecharge(), metric.getGoogleRecharge()));
row.setDealerRecharge(add(row.getDealerRecharge(), metric.getDealerRecharge()));
row.setSalaryExchange(add(row.getSalaryExchange(), metric.getSalaryExchange()));
row.setLuckyGiftTotalFlow(add(row.getLuckyGiftTotalFlow(), metric.getLuckyGiftTotalFlow()));
row.setLuckyGiftUser(safeLong(row.getLuckyGiftUser()) + safeLong(metric.getLuckyGiftUser()));
row.setLuckyGiftPayout(add(row.getLuckyGiftPayout(), metric.getLuckyGiftPayout()));
row.setGameTotalFlow(add(row.getGameTotalFlow(), metric.getGameTotalFlow()));
row.setGameUser(safeLong(row.getGameUser()) + safeLong(metric.getGameUser()));
row.setGamePayout(add(row.getGamePayout(), metric.getGamePayout()));
}
}
private void mergePeriodAmounts(Map<String, CountryDashboardPeriodMetricDTO> rows,
List<CountryDashboardMetricDTO> metrics, PeriodWindow window, String sysOrigin) {
if (metrics == null || metrics.isEmpty()) {
return;
}
for (CountryDashboardMetricDTO metric : metrics) {
CountryDashboardPeriodMetricDTO row = getPeriodRow(rows, metric, window, sysOrigin);
row.setNewUserRecharge(add(row.getNewUserRecharge(), metric.getNewUserRecharge()));
row.setOfficialRecharge(add(row.getOfficialRecharge(), metric.getOfficialRecharge()));
row.setMifapayRecharge(add(row.getMifapayRecharge(), metric.getMifapayRecharge()));
row.setGoogleRecharge(add(row.getGoogleRecharge(), metric.getGoogleRecharge()));
row.setDealerRecharge(add(row.getDealerRecharge(), metric.getDealerRecharge()));
row.setSalaryExchange(add(row.getSalaryExchange(), metric.getSalaryExchange()));
row.setLuckyGiftTotalFlow(add(row.getLuckyGiftTotalFlow(), metric.getLuckyGiftTotalFlow()));
row.setLuckyGiftPayout(add(row.getLuckyGiftPayout(), metric.getLuckyGiftPayout()));
row.setGameTotalFlow(add(row.getGameTotalFlow(), metric.getGameTotalFlow()));
row.setGamePayout(add(row.getGamePayout(), metric.getGamePayout()));
}
}
private void mergePeriodUserCounts(Map<String, CountryDashboardPeriodMetricDTO> rows,
List<CountryDashboardMetricDTO> metrics, PeriodWindow window, String sysOrigin) {
if (metrics == null || metrics.isEmpty()) {
return;
}
for (CountryDashboardMetricDTO metric : metrics) {
CountryDashboardPeriodMetricDTO row = getPeriodRow(rows, metric, window, sysOrigin);
row.setCountryNewUser(safeLong(metric.getCountryNewUser()));
row.setLuckyGiftUser(safeLong(metric.getLuckyGiftUser()));
row.setGameUser(safeLong(metric.getGameUser()));
}
}
private CountryDashboardDailyMetricDTO getRow(Map<String, CountryDashboardDailyMetricDTO> rows,
CountryDashboardMetricDTO metric, LocalDate statDate, String sysOrigin) {
String countryCode = hasText(metric.getCountryCode()) ? metric.getCountryCode() : "UNKNOWN";
String countryName = hasText(metric.getCountryName()) ? metric.getCountryName() : countryCode;
return rows.computeIfAbsent(countryCode,
key -> new CountryDashboardDailyMetricDTO()
.setStatDate(statDate)
.setDateNumber(Integer.valueOf(statDate.format(DATE_NUMBER_FORMATTER)))
.setSysOrigin(sysOrigin)
.setCountryCode(countryCode)
.setCountryName(countryName));
}
private CountryDashboardPeriodMetricDTO getPeriodRow(Map<String, CountryDashboardPeriodMetricDTO> rows,
CountryDashboardMetricDTO metric, PeriodWindow window, String sysOrigin) {
String countryCode = hasText(metric.getCountryCode()) ? metric.getCountryCode() : "UNKNOWN";
String countryName = hasText(metric.getCountryName()) ? metric.getCountryName() : countryCode;
return rows.computeIfAbsent(countryCode,
key -> new CountryDashboardPeriodMetricDTO()
.setPeriodType(window.periodType())
.setPeriodKey(window.periodKey())
.setPeriodName(window.periodName())
.setPeriodStartDate(window.periodStartDate())
.setPeriodEndDate(window.periodEndDate())
.setSysOrigin(sysOrigin)
.setCountryCode(countryCode)
.setCountryName(countryName));
}
private PeriodWindow toDayWindow(LocalDate statDate) {
String key = statDate.format(DATE_FORMATTER);
return new PeriodWindow(PERIOD_DAY, key, key, statDate, statDate.plusDays(1));
}
private PeriodWindow toWeekWindow(LocalDate statDate) {
LocalDate start = statDate.with(DayOfWeek.MONDAY);
LocalDate end = capCurrentPeriodEnd(start.plusDays(7));
int weekYear = statDate.get(WEEK_FIELDS.weekBasedYear());
int week = statDate.get(WEEK_FIELDS.weekOfWeekBasedYear());
String key = String.format(Locale.ROOT, "%d-W%02d", weekYear, week);
return new PeriodWindow(PERIOD_WEEK, key, key, start, end);
}
private PeriodWindow toMonthWindow(LocalDate statDate) {
LocalDate start = statDate.withDayOfMonth(1);
LocalDate end = capCurrentPeriodEnd(start.plusMonths(1));
String key = statDate.format(MONTH_FORMATTER);
return new PeriodWindow(PERIOD_MONTH, key, key, start, end);
}
private LocalDate capCurrentPeriodEnd(LocalDate naturalEndExclusive) {
LocalDate tomorrow = LocalDate.now().plusDays(1);
return naturalEndExclusive.isAfter(tomorrow) ? tomorrow : naturalEndExclusive;
}
private List<String> resolveSysOrigins() {
String config = trimToNull(refreshSysOrigins);
if (config == null) {
return List.of(SysOriginPlatformEnum.LIKEI.name());
}
if ("ALL".equalsIgnoreCase(config)) {
return SysOriginPlatformEnum.getVoiceSystems().stream()
.map(SysOriginPlatformEnum::name)
.filter(Objects::nonNull)
.distinct()
.toList();
}
List<String> sysOrigins = Arrays.stream(config.split(","))
.map(String::trim)
.filter(CountryDashboardDailyMetricService::hasText)
.distinct()
.toList();
return sysOrigins.isEmpty() ? List.of(SysOriginPlatformEnum.LIKEI.name()) : sysOrigins;
}
private static BigDecimal add(BigDecimal left, BigDecimal right) {
return safe(left).add(safe(right));
}
private static BigDecimal safe(BigDecimal value) {
return value == null ? BigDecimal.ZERO : value;
}
private static long safeLong(Long value) {
return value == null ? 0L : value;
}
private static boolean hasText(String value) {
return value != null && !value.trim().isEmpty();
}
private static String trimToNull(String value) {
if (value == null) {
return null;
}
String trimmed = value.trim();
return trimmed.isEmpty() ? null : trimmed;
}
private record PeriodWindow(String periodType, String periodKey, String periodName,
LocalDate periodStartDate, LocalDate periodEndDate) {
}
private record RefreshWriteResult(int deleted, int inserted, int userRows) {
}
}

165
rc-service/rc-service-console/console-application/src/main/java/com/red/circle/console/app/service/app/datav/CountryDashboardServiceImpl.java
View File

@ -33,6 +33,7 @@ import java.util.Objects;
import java.util.Set;
import java.util.function.BiConsumer;
import lombok.RequiredArgsConstructor;
import lombok.extern.slf4j.Slf4j;
import org.bson.Document;
import org.bson.types.Decimal128;
import org.springframework.data.mongodb.core.MongoTemplate;
@ -47,6 +48,7 @@ import org.springframework.stereotype.Service;
*/
@Service
@RequiredArgsConstructor
@Slf4j
public class CountryDashboardServiceImpl implements CountryDashboardService {
private static final String COLLECTION_GIFT_GIVE_RUNNING_WATER = "gift_give_running_water";
@ -64,6 +66,41 @@ public class CountryDashboardServiceImpl implements CountryDashboardService {
QueryCondition condition = QueryCondition.of(cmd);
Map<String, CountryDashboardMetricCO> rows = new LinkedHashMap<>();
if (shouldUsePrecomputedMetrics(condition)) {
mergeSql(rows, countryDashboardDAO.listPrecomputedPeriodMetrics(
condition.periodType, condition.startTime, condition.endTime,
condition.countryKeyword, condition.sysOrigin));
} else {
mergeLiveSql(rows, condition);
}
mergeMongo(rows, condition, listGiftConsume(condition),
(row, amount) -> row.setGiftConsume(add(row.getGiftConsume(), amount)));
mergeMongo(rows, condition, listLuckyGiftAnchorShare(condition),
(row, amount) -> row.setLuckyGiftAnchorShare(add(row.getLuckyGiftAnchorShare(), amount)));
mergeDailyActive(rows, condition, listDailyActive(condition));
mergeRetention(rows, condition);
List<CountryDashboardMetricCO> records = rows.values().stream()
.peek(this::calculateFormulaFields)
.sorted(Comparator.comparing(CountryDashboardMetricCO::getPeriodKey,
Comparator.nullsLast(Comparator.reverseOrder()))
.thenComparing(CountryDashboardMetricCO::getCountryCode,
Comparator.nullsLast(String::compareTo)))
.toList();
CountryDashboardMetricCO total = buildTotal(records, condition.periodType);
return new CountryDashboardCO()
.setPeriodType(condition.periodType)
.setStartDate(condition.startDate)
.setEndDate(condition.endDate)
.setComputedAt(LocalDateTime.now().format(DateTimeFormatter.ofPattern("yyyy-MM-dd HH:mm:ss")))
.setTotal(total)
.setRecords(records);
}
private void mergeLiveSql(Map<String, CountryDashboardMetricCO> rows, QueryCondition condition) {
mergeSql(rows, countryDashboardDAO.listNewUsers(
condition.periodType, condition.startTime, condition.endTime,
condition.countryKeyword, condition.sysOrigin));
@ -90,38 +127,116 @@ public class CountryDashboardServiceImpl implements CountryDashboardService {
condition.countryKeyword, condition.sysOrigin));
mergeSql(rows, countryDashboardDAO.listWalletGame(
condition.periodType, condition.startTime, condition.endTime,
condition.startTimeMillis(), condition.endTimeMillis(),
condition.countryKeyword, condition.sysOrigin));
mergeSql(rows, countryDashboardDAO.listLuckyBoxGame(
condition.periodType, condition.startTime, condition.endTime,
condition.countryKeyword, condition.sysOrigin));
mergeSql(rows, countryDashboardDAO.listGameUser(
condition.periodType, condition.startTime, condition.endTime,
condition.startTimeMillis(), condition.endTimeMillis(),
condition.countryKeyword, condition.sysOrigin));
}
mergeMongo(rows, condition, listGiftConsume(condition),
(row, amount) -> row.setGiftConsume(add(row.getGiftConsume(), amount)));
mergeMongo(rows, condition, listLuckyGiftAnchorShare(condition),
(row, amount) -> row.setLuckyGiftAnchorShare(add(row.getLuckyGiftAnchorShare(), amount)));
mergeDailyActive(rows, condition, listDailyActive(condition));
mergeRetention(rows, condition);
private boolean shouldUsePrecomputedMetrics(QueryCondition condition) {
if (!hasText(condition.sysOrigin)) {
return false;
}
if (PERIOD_ALL.equals(condition.periodType)) {
if (condition.startTime != null || condition.endTime != null) {
return false;
}
long count = safeLong(countryDashboardDAO.countAllPrecomputedPeriodMetrics(condition.sysOrigin));
if (count <= 0) {
log.warn("country dashboard all precomputed metric missing, skip live all-history scan, sysOrigin={}",
condition.sysOrigin);
}
return true;
}
List<CountryDashboardMetricCO> records = rows.values().stream()
.peek(this::calculateFormulaFields)
.sorted(Comparator.comparing(CountryDashboardMetricCO::getPeriodKey,
Comparator.nullsLast(Comparator.reverseOrder()))
.thenComparing(CountryDashboardMetricCO::getCountryCode,
Comparator.nullsLast(String::compareTo)))
.toList();
List<String> periodKeys = expectedPrecomputedPeriodKeys(condition);
if (periodKeys.isEmpty()) {
return false;
}
long actualPeriods = safeLong(countryDashboardDAO.countPrecomputedPeriods(
condition.periodType, periodKeys, condition.sysOrigin));
boolean complete = actualPeriods >= periodKeys.size();
if (!complete) {
log.info("country dashboard use live sql, period precomputed metric incomplete, periodType={}, sysOrigin={}, start={}, end={}, expectedPeriods={}, actualPeriods={}",
condition.periodType, condition.sysOrigin, condition.startDate, condition.endDate,
periodKeys.size(), actualPeriods);
}
return complete;
}
CountryDashboardMetricCO total = buildTotal(records, condition.periodType);
private List<String> expectedPrecomputedPeriodKeys(QueryCondition condition) {
if (condition.startTime == null || condition.endTime == null
|| !condition.startTime.isBefore(condition.endTime)) {
return List.of();
}
LocalDate start = condition.startTime.toLocalDate();
LocalDate endExclusive = condition.endTime.toLocalDate();
return switch (condition.periodType) {
case "DAY" -> expectedDayKeys(start, endExclusive);
case "WEEK", "MONTH" -> expectedAlignedPeriodKeys(condition.periodType, start, endExclusive);
default -> List.of();
};
}
return new CountryDashboardCO()
.setPeriodType(condition.periodType)
.setStartDate(condition.startDate)
.setEndDate(condition.endDate)
.setComputedAt(LocalDateTime.now().format(DateTimeFormatter.ofPattern("yyyy-MM-dd HH:mm:ss")))
.setTotal(total)
.setRecords(records);
private List<String> expectedDayKeys(LocalDate start, LocalDate endExclusive) {
List<String> keys = new ArrayList<>();
for (LocalDate cursor = start; cursor.isBefore(endExclusive); cursor = cursor.plusDays(1)) {
keys.add(cursor.format(DATE_FORMATTER));
}
return keys;
}
private List<String> expectedAlignedPeriodKeys(String periodType, LocalDate start,
LocalDate endExclusive) {
List<String> keys = new ArrayList<>();
LocalDate cursor = start;
LocalDate tomorrow = LocalDate.now().plusDays(1);
while (cursor.isBefore(endExclusive)) {
LocalDate periodStart = periodStart(periodType, cursor);
if (!cursor.equals(periodStart)) {
return List.of();
}
LocalDate naturalEnd = periodEnd(periodType, periodStart);
LocalDate currentEnd = naturalEnd.isAfter(tomorrow) ? tomorrow : naturalEnd;
LocalDate windowEnd = currentEnd.isAfter(endExclusive) ? endExclusive : currentEnd;
boolean fullPeriod = windowEnd.equals(naturalEnd);
boolean currentPartialPeriod = naturalEnd.isAfter(tomorrow) && windowEnd.equals(tomorrow);
if (!fullPeriod && !currentPartialPeriod) {
return List.of();
}
keys.add(periodKey(periodType, cursor));
cursor = windowEnd;
}
return cursor.equals(endExclusive) ? keys : List.of();
}
private LocalDate periodStart(String periodType, LocalDate date) {
if ("WEEK".equals(periodType)) {
return date.with(DayOfWeek.MONDAY);
}
return date.withDayOfMonth(1);
}
private LocalDate periodEnd(String periodType, LocalDate periodStart) {
if ("WEEK".equals(periodType)) {
return periodStart.plusDays(7);
}
return periodStart.plusMonths(1);
}
private String periodKey(String periodType, LocalDate date) {
if ("WEEK".equals(periodType)) {
WeekFields weekFields = WeekFields.ISO;
int weekYear = date.get(weekFields.weekBasedYear());
int week = date.get(weekFields.weekOfWeekBasedYear());
return String.format(Locale.ROOT, "%d-W%02d", weekYear, week);
}
return date.format(MONTH_FORMATTER);
}
@Override
@ -716,6 +831,14 @@ public class CountryDashboardServiceImpl implements CountryDashboardService {
return new PeriodBucket(day, day);
}
private Long startTimeMillis() {
return startTime == null ? null : Timestamp.valueOf(startTime).getTime();
}
private Long endTimeMillis() {
return endTime == null ? null : Timestamp.valueOf(endTime).getTime();
}
private static String normalizePeriodType(String periodType) {
String value = trimToNull(periodType);
if (value == null) {

98
rc-service/rc-service-console/console-infrastructure/src/main/java/com/red/circle/console/infra/database/rds/dao/datav/CountryDashboardDAO.java
View File

@ -1,11 +1,14 @@
package com.red.circle.console.infra.database.rds.dao.datav;
import com.red.circle.console.infra.database.rds.dto.datav.CountryDashboardDailyMetricDTO;
import com.red.circle.console.infra.database.rds.dto.datav.CountryDashboardMetricDTO;
import com.red.circle.console.infra.database.rds.dto.datav.CountryDashboardPeriodMetricDTO;
import com.red.circle.console.infra.database.rds.dto.datav.CountryDashboardRechargeDetailDTO;
import com.red.circle.console.infra.database.rds.dto.datav.CountryDashboardRetentionUserDTO;
import com.red.circle.console.infra.database.rds.dto.datav.CountryDashboardUserCountryDTO;
import com.red.circle.console.infra.database.rds.entity.app.IUserBaseInfo;
import com.red.circle.framework.mybatis.dao.BaseDAO;
import java.time.LocalDate;
import java.time.LocalDateTime;
import java.util.Collection;
import java.util.List;
@ -16,6 +19,12 @@ import org.apache.ibatis.annotations.Param;
*/
public interface CountryDashboardDAO extends BaseDAO<IUserBaseInfo> {
int createDailyMetricTable();
int createPeriodMetricTable();
int createPeriodUserMetricTable();
List<CountryDashboardMetricDTO> listNewUsers(
@Param("periodType") String periodType,
@Param("startTime") LocalDateTime startTime,
@ -76,6 +85,8 @@ public interface CountryDashboardDAO extends BaseDAO<IUserBaseInfo> {
@Param("periodType") String periodType,
@Param("startTime") LocalDateTime startTime,
@Param("endTime") LocalDateTime endTime,
@Param("startTimeMillis") Long startTimeMillis,
@Param("endTimeMillis") Long endTimeMillis,
@Param("countryKeyword") String countryKeyword,
@Param("sysOrigin") String sysOrigin);
@ -90,9 +101,96 @@ public interface CountryDashboardDAO extends BaseDAO<IUserBaseInfo> {
@Param("periodType") String periodType,
@Param("startTime") LocalDateTime startTime,
@Param("endTime") LocalDateTime endTime,
@Param("startTimeMillis") Long startTimeMillis,
@Param("endTimeMillis") Long endTimeMillis,
@Param("countryKeyword") String countryKeyword,
@Param("sysOrigin") String sysOrigin);
List<CountryDashboardMetricDTO> listPrecomputedMetrics(
@Param("periodType") String periodType,
@Param("startTime") LocalDateTime startTime,
@Param("endTime") LocalDateTime endTime,
@Param("countryKeyword") String countryKeyword,
@Param("sysOrigin") String sysOrigin);
List<CountryDashboardMetricDTO> listPrecomputedPeriodMetrics(
@Param("periodType") String periodType,
@Param("startTime") LocalDateTime startTime,
@Param("endTime") LocalDateTime endTime,
@Param("countryKeyword") String countryKeyword,
@Param("sysOrigin") String sysOrigin);
Long countPrecomputedMetricDates(
@Param("startTime") LocalDateTime startTime,
@Param("endTime") LocalDateTime endTime,
@Param("sysOrigin") String sysOrigin);
Long countPrecomputedPeriods(
@Param("periodType") String periodType,
@Param("periodKeys") Collection<String> periodKeys,
@Param("sysOrigin") String sysOrigin);
Long countAllPrecomputedPeriodMetrics(@Param("sysOrigin") String sysOrigin);
int deleteDailyMetrics(
@Param("statDate") LocalDate statDate,
@Param("sysOrigin") String sysOrigin);
int upsertDailyMetrics(@Param("metrics") List<CountryDashboardDailyMetricDTO> metrics);
int deletePeriodMetrics(
@Param("periodType") String periodType,
@Param("periodKey") String periodKey,
@Param("sysOrigin") String sysOrigin);
int upsertPeriodMetrics(@Param("metrics") List<CountryDashboardPeriodMetricDTO> metrics);
int deletePeriodUserMetrics(
@Param("periodType") String periodType,
@Param("periodKey") String periodKey,
@Param("sysOrigin") String sysOrigin);
int insertPeriodNewUserMetrics(
@Param("periodType") String periodType,
@Param("periodKey") String periodKey,
@Param("periodName") String periodName,
@Param("periodStartDate") LocalDate periodStartDate,
@Param("periodEndDate") LocalDate periodEndDate,
@Param("startTime") LocalDateTime startTime,
@Param("endTime") LocalDateTime endTime,
@Param("sysOrigin") String sysOrigin);
int insertPeriodLuckyGiftUserMetrics(
@Param("periodType") String periodType,
@Param("periodKey") String periodKey,
@Param("periodName") String periodName,
@Param("periodStartDate") LocalDate periodStartDate,
@Param("periodEndDate") LocalDate periodEndDate,
@Param("startTime") LocalDateTime startTime,
@Param("endTime") LocalDateTime endTime,
@Param("sysOrigin") String sysOrigin);
int insertPeriodGameUserMetrics(
@Param("periodType") String periodType,
@Param("periodKey") String periodKey,
@Param("periodName") String periodName,
@Param("periodStartDate") LocalDate periodStartDate,
@Param("periodEndDate") LocalDate periodEndDate,
@Param("startTime") LocalDateTime startTime,
@Param("endTime") LocalDateTime endTime,
@Param("startTimeMillis") Long startTimeMillis,
@Param("endTimeMillis") Long endTimeMillis,
@Param("sysOrigin") String sysOrigin);
int insertAllPeriodUserMetricsFromDaily(@Param("sysOrigin") String sysOrigin);
List<CountryDashboardMetricDTO> listPeriodUserMetricCounts(
@Param("periodType") String periodType,
@Param("periodKey") String periodKey,
@Param("sysOrigin") String sysOrigin);
List<CountryDashboardMetricDTO> listAllPeriodAmountsFromDaily(@Param("sysOrigin") String sysOrigin);
List<CountryDashboardRetentionUserDTO> listNewUserCohorts(
@Param("periodType") String periodType,
@Param("startTime") LocalDateTime startTime,

50
rc-service/rc-service-console/console-infrastructure/src/main/java/com/red/circle/console/infra/database/rds/dto/datav/CountryDashboardDailyMetricDTO.java Normal file
View File

@ -0,0 +1,50 @@
package com.red.circle.console.infra.database.rds.dto.datav;
import java.math.BigDecimal;
import java.time.LocalDate;
import lombok.Data;
import lombok.experimental.Accessors;
/**
* 国家数据大屏日预聚合结果.
*/
@Data
@Accessors(chain = true)
public class CountryDashboardDailyMetricDTO {
private LocalDate statDate;
private Integer dateNumber;
private String sysOrigin;
private String countryCode;
private String countryName;
private Long countryNewUser;
private BigDecimal newUserRecharge;
private BigDecimal officialRecharge;
private BigDecimal mifapayRecharge;
private BigDecimal googleRecharge;
private BigDecimal dealerRecharge;
private BigDecimal salaryExchange;
private BigDecimal luckyGiftTotalFlow;
private Long luckyGiftUser;
private BigDecimal luckyGiftPayout;
private BigDecimal gameTotalFlow;
private Long gameUser;
private BigDecimal gamePayout;
}

56
rc-service/rc-service-console/console-infrastructure/src/main/java/com/red/circle/console/infra/database/rds/dto/datav/CountryDashboardPeriodMetricDTO.java Normal file
View File

@ -0,0 +1,56 @@
package com.red.circle.console.infra.database.rds.dto.datav;
import java.math.BigDecimal;
import java.time.LocalDate;
import lombok.Data;
import lombok.experimental.Accessors;
/**
* 国家数据大屏周期预聚合结果.
*/
@Data
@Accessors(chain = true)
public class CountryDashboardPeriodMetricDTO {
private String periodType;
private String periodKey;
private String periodName;
private LocalDate periodStartDate;
private LocalDate periodEndDate;
private String sysOrigin;
private String countryCode;
private String countryName;
private Long countryNewUser;
private BigDecimal newUserRecharge;
private BigDecimal officialRecharge;
private BigDecimal mifapayRecharge;
private BigDecimal googleRecharge;
private BigDecimal dealerRecharge;
private BigDecimal salaryExchange;
private BigDecimal luckyGiftTotalFlow;
private Long luckyGiftUser;
private BigDecimal luckyGiftPayout;
private BigDecimal gameTotalFlow;
private Long gameUser;
private BigDecimal gamePayout;
}

599
rc-service/rc-service-console/console-infrastructure/src/main/resources/dao/datav/CountryDashboardDAO.xml
View File

@ -3,6 +3,92 @@
"http://mybatis.org/dtd/mybatis-3-mapper.dtd">
<mapper namespace="com.red.circle.console.infra.database.rds.dao.datav.CountryDashboardDAO">
<update id="createDailyMetricTable">
CREATE TABLE IF NOT EXISTS `country_dashboard_daily_metric` (
`id` bigint NOT NULL AUTO_INCREMENT COMMENT '主键',
`stat_date` date NOT NULL COMMENT '统计日期',
`date_number` int NOT NULL COMMENT '统计日期yyyyMMdd',
`sys_origin` varchar(32) CHARACTER SET utf8mb4 COLLATE utf8mb4_unicode_ci NOT NULL DEFAULT '' COMMENT '系统来源',
`country_code` varchar(64) CHARACTER SET utf8mb4 COLLATE utf8mb4_unicode_ci NOT NULL DEFAULT 'UNKNOWN' COMMENT '国家码',
`country_name` varchar(128) CHARACTER SET utf8mb4 COLLATE utf8mb4_unicode_ci NOT NULL DEFAULT 'UNKNOWN' COMMENT '国家名称',
`country_new_user` bigint NOT NULL DEFAULT 0 COMMENT '新增用户数',
`new_user_recharge` decimal(20,2) NOT NULL DEFAULT 0.00 COMMENT '新增用户充值',
`official_recharge` decimal(20,2) NOT NULL DEFAULT 0.00 COMMENT '官方充值',
`mifapay_recharge` decimal(20,2) NOT NULL DEFAULT 0.00 COMMENT 'MifaPay充值',
`google_recharge` decimal(20,2) NOT NULL DEFAULT 0.00 COMMENT 'Google充值',
`dealer_recharge` decimal(20,2) NOT NULL DEFAULT 0.00 COMMENT '代理充值',
`salary_exchange` decimal(20,2) NOT NULL DEFAULT 0.00 COMMENT '工资兑换',
`lucky_gift_total_flow` decimal(20,2) NOT NULL DEFAULT 0.00 COMMENT '幸运礼物流水',
`lucky_gift_user` bigint NOT NULL DEFAULT 0 COMMENT '幸运礼物用户数',
`lucky_gift_payout` decimal(20,2) NOT NULL DEFAULT 0.00 COMMENT '幸运礼物返奖',
`game_total_flow` decimal(20,2) NOT NULL DEFAULT 0.00 COMMENT '游戏流水',
`game_user` bigint NOT NULL DEFAULT 0 COMMENT '游戏用户数',
`game_payout` decimal(20,2) NOT NULL DEFAULT 0.00 COMMENT '游戏返奖',
`refreshed_at` datetime NOT NULL DEFAULT CURRENT_TIMESTAMP COMMENT '最近刷新时间',
`create_time` datetime NOT NULL DEFAULT CURRENT_TIMESTAMP COMMENT '创建时间',
`update_time` datetime NOT NULL DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP COMMENT '更新时间',
PRIMARY KEY (`id`),
UNIQUE KEY `uk_date_origin_country` (`stat_date`, `sys_origin`, `country_code`),
KEY `idx_origin_date` (`sys_origin`, `stat_date`),
KEY `idx_country_date` (`country_code`, `stat_date`)
) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_unicode_ci COMMENT='国家数据大屏日预聚合'
</update>
<update id="createPeriodMetricTable">
CREATE TABLE IF NOT EXISTS `country_dashboard_period_metric` (
`id` bigint NOT NULL AUTO_INCREMENT COMMENT '主键',
`period_type` varchar(16) CHARACTER SET utf8mb4 COLLATE utf8mb4_unicode_ci NOT NULL COMMENT '周期类型:DAY/WEEK/MONTH/ALL',
`period_key` varchar(32) CHARACTER SET utf8mb4 COLLATE utf8mb4_unicode_ci NOT NULL COMMENT '周期键',
`period_name` varchar(64) CHARACTER SET utf8mb4 COLLATE utf8mb4_unicode_ci NOT NULL DEFAULT '' COMMENT '周期名称',
`period_start_date` date DEFAULT NULL COMMENT '周期开始日期',
`period_end_date` date DEFAULT NULL COMMENT '周期结束日期,开区间',
`sys_origin` varchar(32) CHARACTER SET utf8mb4 COLLATE utf8mb4_unicode_ci NOT NULL DEFAULT '' COMMENT '系统来源',
`country_code` varchar(64) CHARACTER SET utf8mb4 COLLATE utf8mb4_unicode_ci NOT NULL DEFAULT 'UNKNOWN' COMMENT '国家码',
`country_name` varchar(128) CHARACTER SET utf8mb4 COLLATE utf8mb4_unicode_ci NOT NULL DEFAULT 'UNKNOWN' COMMENT '国家名称',
`country_new_user` bigint NOT NULL DEFAULT 0 COMMENT '新增用户数',
`new_user_recharge` decimal(20,2) NOT NULL DEFAULT 0.00 COMMENT '新增用户充值',
`official_recharge` decimal(20,2) NOT NULL DEFAULT 0.00 COMMENT '官方充值',
`mifapay_recharge` decimal(20,2) NOT NULL DEFAULT 0.00 COMMENT 'MifaPay充值',
`google_recharge` decimal(20,2) NOT NULL DEFAULT 0.00 COMMENT 'Google充值',
`dealer_recharge` decimal(20,2) NOT NULL DEFAULT 0.00 COMMENT '代理充值',
`salary_exchange` decimal(20,2) NOT NULL DEFAULT 0.00 COMMENT '工资兑换',
`lucky_gift_total_flow` decimal(20,2) NOT NULL DEFAULT 0.00 COMMENT '幸运礼物流水',
`lucky_gift_user` bigint NOT NULL DEFAULT 0 COMMENT '幸运礼物用户数',
`lucky_gift_payout` decimal(20,2) NOT NULL DEFAULT 0.00 COMMENT '幸运礼物返奖',
`game_total_flow` decimal(20,2) NOT NULL DEFAULT 0.00 COMMENT '游戏流水',
`game_user` bigint NOT NULL DEFAULT 0 COMMENT '游戏用户数',
`game_payout` decimal(20,2) NOT NULL DEFAULT 0.00 COMMENT '游戏返奖',
`refreshed_at` datetime NOT NULL DEFAULT CURRENT_TIMESTAMP COMMENT '最近刷新时间',
`create_time` datetime NOT NULL DEFAULT CURRENT_TIMESTAMP COMMENT '创建时间',
`update_time` datetime NOT NULL DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP COMMENT '更新时间',
PRIMARY KEY (`id`),
UNIQUE KEY `uk_period_origin_country` (`period_type`, `period_key`, `sys_origin`, `country_code`),
KEY `idx_origin_period` (`sys_origin`, `period_type`, `period_key`),
KEY `idx_period_date` (`period_type`, `period_start_date`, `period_end_date`)
) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_unicode_ci COMMENT='国家数据大屏周期预聚合'
</update>
<update id="createPeriodUserMetricTable">
CREATE TABLE IF NOT EXISTS `country_dashboard_period_user_metric` (
`id` bigint NOT NULL AUTO_INCREMENT COMMENT '主键',
`period_type` varchar(16) CHARACTER SET utf8mb4 COLLATE utf8mb4_unicode_ci NOT NULL COMMENT '周期类型:DAY/WEEK/MONTH/ALL',
`period_key` varchar(32) CHARACTER SET utf8mb4 COLLATE utf8mb4_unicode_ci NOT NULL COMMENT '周期键',
`period_name` varchar(64) CHARACTER SET utf8mb4 COLLATE utf8mb4_unicode_ci NOT NULL DEFAULT '' COMMENT '周期名称',
`period_start_date` date DEFAULT NULL COMMENT '周期开始日期',
`period_end_date` date DEFAULT NULL COMMENT '周期结束日期,开区间',
`sys_origin` varchar(32) CHARACTER SET utf8mb4 COLLATE utf8mb4_unicode_ci NOT NULL DEFAULT '' COMMENT '系统来源',
`country_code` varchar(64) CHARACTER SET utf8mb4 COLLATE utf8mb4_unicode_ci NOT NULL DEFAULT 'UNKNOWN' COMMENT '国家码',
`country_name` varchar(128) CHARACTER SET utf8mb4 COLLATE utf8mb4_unicode_ci NOT NULL DEFAULT 'UNKNOWN' COMMENT '国家名称',
`user_id` bigint NOT NULL COMMENT '用户ID',
`metric_type` varchar(32) CHARACTER SET utf8mb4 COLLATE utf8mb4_unicode_ci NOT NULL COMMENT '人数指标类型',
`create_time` datetime NOT NULL DEFAULT CURRENT_TIMESTAMP COMMENT '创建时间',
PRIMARY KEY (`id`),
UNIQUE KEY `uk_period_user_metric` (`period_type`, `period_key`, `sys_origin`, `country_code`, `user_id`, `metric_type`),
KEY `idx_period_metric_country` (`period_type`, `period_key`, `sys_origin`, `metric_type`, `country_code`),
KEY `idx_origin_user_metric` (`sys_origin`, `user_id`, `metric_type`)
) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_unicode_ci COMMENT='国家数据大屏周期用户去重明细'
</update>
<sql id="CountryColumns">
NULLIF(u.country_code, '') AS countryCode,
NULLIF(u.country_name, '') AS countryName
@ -79,6 +165,23 @@
</if>
</sql>
<sql id="PrecomputedPeriodColumns">
<choose>
<when test="periodType == 'DAY'">DATE_FORMAT(t.stat_date, '%Y-%m-%d')</when>
<when test="periodType == 'WEEK'">DATE_FORMAT(t.stat_date, '%x-W%v')</when>
<when test="periodType == 'MONTH'">DATE_FORMAT(t.stat_date, '%Y-%m')</when>
<otherwise>'ALL'</otherwise>
</choose>
AS periodKey,
<choose>
<when test="periodType == 'DAY'">DATE_FORMAT(t.stat_date, '%Y-%m-%d')</when>
<when test="periodType == 'WEEK'">DATE_FORMAT(t.stat_date, '%x-W%v')</when>
<when test="periodType == 'MONTH'">DATE_FORMAT(t.stat_date, '%Y-%m')</when>
<otherwise>'全部'</otherwise>
</choose>
AS periodName
</sql>
<sql id="CountryCodesWhere">
<if test="countryCodes != null and countryCodes != ''">
AND FIND_IN_SET(u.country_code, #{countryCodes})
@ -138,42 +241,68 @@
<include refid="SourceSysOriginWhere"/>
</sql>
<sql id="WalletGoldAssetRecordShardWhere">
<where>
<if test="startTimeMillis != null">
create_time &gt;= #{startTimeMillis}
</if>
<if test="endTimeMillis != null">
AND create_time &lt; #{endTimeMillis}
</if>
<if test="sysOrigin != null and sysOrigin != ''">
AND sys_origin = #{sysOrigin}
</if>
</where>
</sql>
<sql id="WalletGoldAssetRecordSource">
SELECT user_id, sys_origin, type, penny_amount, event_type, FROM_UNIXTIME(create_time / 1000) AS create_time
FROM likei_wallet.wallet_gold_asset_record_1
<include refid="WalletGoldAssetRecordShardWhere"/>
UNION ALL
SELECT user_id, sys_origin, type, penny_amount, event_type, FROM_UNIXTIME(create_time / 1000) AS create_time
FROM likei_wallet.wallet_gold_asset_record_2
<include refid="WalletGoldAssetRecordShardWhere"/>
UNION ALL
SELECT user_id, sys_origin, type, penny_amount, event_type, FROM_UNIXTIME(create_time / 1000) AS create_time
FROM likei_wallet.wallet_gold_asset_record_3
<include refid="WalletGoldAssetRecordShardWhere"/>
UNION ALL
SELECT user_id, sys_origin, type, penny_amount, event_type, FROM_UNIXTIME(create_time / 1000) AS create_time
FROM likei_wallet.wallet_gold_asset_record_4
<include refid="WalletGoldAssetRecordShardWhere"/>
UNION ALL
SELECT user_id, sys_origin, type, penny_amount, event_type, FROM_UNIXTIME(create_time / 1000) AS create_time
FROM likei_wallet.wallet_gold_asset_record_5
<include refid="WalletGoldAssetRecordShardWhere"/>
UNION ALL
SELECT user_id, sys_origin, type, penny_amount, event_type, FROM_UNIXTIME(create_time / 1000) AS create_time
FROM likei_wallet.wallet_gold_asset_record_6
<include refid="WalletGoldAssetRecordShardWhere"/>
UNION ALL
SELECT user_id, sys_origin, type, penny_amount, event_type, FROM_UNIXTIME(create_time / 1000) AS create_time
FROM likei_wallet.wallet_gold_asset_record_7
<include refid="WalletGoldAssetRecordShardWhere"/>
UNION ALL
SELECT user_id, sys_origin, type, penny_amount, event_type, FROM_UNIXTIME(create_time / 1000) AS create_time
FROM likei_wallet.wallet_gold_asset_record_8
<include refid="WalletGoldAssetRecordShardWhere"/>
UNION ALL
SELECT user_id, sys_origin, type, penny_amount, event_type, FROM_UNIXTIME(create_time / 1000) AS create_time
FROM likei_wallet.wallet_gold_asset_record_9
<include refid="WalletGoldAssetRecordShardWhere"/>
UNION ALL
SELECT user_id, sys_origin, type, penny_amount, event_type, FROM_UNIXTIME(create_time / 1000) AS create_time
FROM likei_wallet.wallet_gold_asset_record_10
<include refid="WalletGoldAssetRecordShardWhere"/>
UNION ALL
SELECT user_id, sys_origin, type, penny_amount, event_type, FROM_UNIXTIME(create_time / 1000) AS create_time
FROM likei_wallet.wallet_gold_asset_record_11
<include refid="WalletGoldAssetRecordShardWhere"/>
UNION ALL
SELECT user_id, sys_origin, type, penny_amount, event_type, FROM_UNIXTIME(create_time / 1000) AS create_time
FROM likei_wallet.wallet_gold_asset_record_12
<include refid="WalletGoldAssetRecordShardWhere"/>
</sql>
<sql id="WalletGameEventWhere">
@ -213,6 +342,476 @@
AND t.event_type NOT LIKE 'ACCEPT_GIFT%'
</sql>
<select id="listPrecomputedMetrics"
resultType="com.red.circle.console.infra.database.rds.dto.datav.CountryDashboardMetricDTO">
SELECT
t.country_code AS countryCode,
MAX(t.country_name) AS countryName,
<include refid="PrecomputedPeriodColumns"/>,
IFNULL(SUM(t.country_new_user), 0) AS countryNewUser,
IFNULL(SUM(t.new_user_recharge), 0) AS newUserRecharge,
IFNULL(SUM(t.official_recharge), 0) AS officialRecharge,
IFNULL(SUM(t.mifapay_recharge), 0) AS mifapayRecharge,
IFNULL(SUM(t.google_recharge), 0) AS googleRecharge,
IFNULL(SUM(t.dealer_recharge), 0) AS dealerRecharge,
IFNULL(SUM(t.salary_exchange), 0) AS salaryExchange,
IFNULL(SUM(t.lucky_gift_total_flow), 0) AS luckyGiftTotalFlow,
IFNULL(SUM(t.lucky_gift_user), 0) AS luckyGiftUser,
IFNULL(SUM(t.lucky_gift_payout), 0) AS luckyGiftPayout,
IFNULL(SUM(t.game_total_flow), 0) AS gameTotalFlow,
IFNULL(SUM(t.game_user), 0) AS gameUser,
IFNULL(SUM(t.game_payout), 0) AS gamePayout
FROM country_dashboard_daily_metric t
WHERE 1 = 1
<if test="startTime != null">
AND t.stat_date &gt;= DATE(#{startTime})
</if>
<if test="endTime != null">
AND t.stat_date &lt; DATE(#{endTime})
</if>
<if test="countryKeyword != null and countryKeyword != ''">
AND (
t.country_code = #{countryKeyword}
OR t.country_name LIKE CONCAT('%', #{countryKeyword}, '%')
)
</if>
<if test="sysOrigin != null and sysOrigin != ''">
AND t.sys_origin = #{sysOrigin}
</if>
GROUP BY countryCode, periodKey, periodName
</select>
<select id="countPrecomputedMetricDates" resultType="java.lang.Long">
SELECT COUNT(DISTINCT t.stat_date)
FROM country_dashboard_daily_metric t
WHERE 1 = 1
<if test="startTime != null">
AND t.stat_date &gt;= DATE(#{startTime})
</if>
<if test="endTime != null">
AND t.stat_date &lt; DATE(#{endTime})
</if>
<if test="sysOrigin != null and sysOrigin != ''">
AND t.sys_origin = #{sysOrigin}
</if>
</select>
<select id="listPrecomputedPeriodMetrics"
resultType="com.red.circle.console.infra.database.rds.dto.datav.CountryDashboardMetricDTO">
SELECT
t.country_code AS countryCode,
MAX(t.country_name) AS countryName,
t.period_key AS periodKey,
MAX(t.period_name) AS periodName,
IFNULL(SUM(t.country_new_user), 0) AS countryNewUser,
IFNULL(SUM(t.new_user_recharge), 0) AS newUserRecharge,
IFNULL(SUM(t.official_recharge), 0) AS officialRecharge,
IFNULL(SUM(t.mifapay_recharge), 0) AS mifapayRecharge,
IFNULL(SUM(t.google_recharge), 0) AS googleRecharge,
IFNULL(SUM(t.dealer_recharge), 0) AS dealerRecharge,
IFNULL(SUM(t.salary_exchange), 0) AS salaryExchange,
IFNULL(SUM(t.lucky_gift_total_flow), 0) AS luckyGiftTotalFlow,
IFNULL(SUM(t.lucky_gift_user), 0) AS luckyGiftUser,
IFNULL(SUM(t.lucky_gift_payout), 0) AS luckyGiftPayout,
IFNULL(SUM(t.game_total_flow), 0) AS gameTotalFlow,
IFNULL(SUM(t.game_user), 0) AS gameUser,
IFNULL(SUM(t.game_payout), 0) AS gamePayout
FROM country_dashboard_period_metric t
WHERE t.period_type = #{periodType}
<if test="periodType != 'ALL' and startTime != null">
AND t.period_start_date &gt;= DATE(#{startTime})
</if>
<if test="periodType != 'ALL' and endTime != null">
AND t.period_start_date &lt; DATE(#{endTime})
</if>
<if test="countryKeyword != null and countryKeyword != ''">
AND (
t.country_code = #{countryKeyword}
OR t.country_name LIKE CONCAT('%', #{countryKeyword}, '%')
)
</if>
<if test="sysOrigin != null and sysOrigin != ''">
AND t.sys_origin = #{sysOrigin}
</if>
GROUP BY countryCode, periodKey
</select>
<select id="countPrecomputedPeriods" resultType="java.lang.Long">
SELECT COUNT(DISTINCT t.period_key)
FROM country_dashboard_period_metric t
WHERE t.period_type = #{periodType}
<if test="periodKeys != null and periodKeys.size() &gt; 0">
AND t.period_key IN
<foreach collection="periodKeys" item="periodKey" open="(" separator="," close=")">
#{periodKey}
</foreach>
</if>
<if test="sysOrigin != null and sysOrigin != ''">
AND t.sys_origin = #{sysOrigin}
</if>
</select>
<select id="countAllPrecomputedPeriodMetrics" resultType="java.lang.Long">
SELECT COUNT(1)
FROM country_dashboard_period_metric t
WHERE t.period_type = 'ALL'
AND t.period_key = 'ALL'
<if test="sysOrigin != null and sysOrigin != ''">
AND t.sys_origin = #{sysOrigin}
</if>
</select>
<delete id="deleteDailyMetrics">
DELETE FROM country_dashboard_daily_metric
WHERE stat_date = #{statDate}
<if test="sysOrigin != null and sysOrigin != ''">
AND sys_origin = #{sysOrigin}
</if>
</delete>
<insert id="upsertDailyMetrics">
INSERT INTO country_dashboard_daily_metric (
stat_date,
date_number,
sys_origin,
country_code,
country_name,
country_new_user,
new_user_recharge,
official_recharge,
mifapay_recharge,
google_recharge,
dealer_recharge,
salary_exchange,
lucky_gift_total_flow,
lucky_gift_user,
lucky_gift_payout,
game_total_flow,
game_user,
game_payout,
refreshed_at
)
VALUES
<foreach collection="metrics" item="item" separator=",">
(
#{item.statDate},
#{item.dateNumber},
#{item.sysOrigin},
#{item.countryCode},
#{item.countryName},
IFNULL(#{item.countryNewUser}, 0),
IFNULL(#{item.newUserRecharge}, 0),
IFNULL(#{item.officialRecharge}, 0),
IFNULL(#{item.mifapayRecharge}, 0),
IFNULL(#{item.googleRecharge}, 0),
IFNULL(#{item.dealerRecharge}, 0),
IFNULL(#{item.salaryExchange}, 0),
IFNULL(#{item.luckyGiftTotalFlow}, 0),
IFNULL(#{item.luckyGiftUser}, 0),
IFNULL(#{item.luckyGiftPayout}, 0),
IFNULL(#{item.gameTotalFlow}, 0),
IFNULL(#{item.gameUser}, 0),
IFNULL(#{item.gamePayout}, 0),
NOW()
)
</foreach>
ON DUPLICATE KEY UPDATE
country_name = VALUES(country_name),
country_new_user = VALUES(country_new_user),
new_user_recharge = VALUES(new_user_recharge),
official_recharge = VALUES(official_recharge),
mifapay_recharge = VALUES(mifapay_recharge),
google_recharge = VALUES(google_recharge),
dealer_recharge = VALUES(dealer_recharge),
salary_exchange = VALUES(salary_exchange),
lucky_gift_total_flow = VALUES(lucky_gift_total_flow),
lucky_gift_user = VALUES(lucky_gift_user),
lucky_gift_payout = VALUES(lucky_gift_payout),
game_total_flow = VALUES(game_total_flow),
game_user = VALUES(game_user),
game_payout = VALUES(game_payout),
refreshed_at = NOW()
</insert>
<delete id="deletePeriodMetrics">
DELETE FROM country_dashboard_period_metric
WHERE period_type = #{periodType}
AND period_key = #{periodKey}
<if test="sysOrigin != null and sysOrigin != ''">
AND sys_origin = #{sysOrigin}
</if>
</delete>
<insert id="upsertPeriodMetrics">
INSERT INTO country_dashboard_period_metric (
period_type,
period_key,
period_name,
period_start_date,
period_end_date,
sys_origin,
country_code,
country_name,
country_new_user,
new_user_recharge,
official_recharge,
mifapay_recharge,
google_recharge,
dealer_recharge,
salary_exchange,
lucky_gift_total_flow,
lucky_gift_user,
lucky_gift_payout,
game_total_flow,
game_user,
game_payout,
refreshed_at
)
VALUES
<foreach collection="metrics" item="item" separator=",">
(
#{item.periodType},
#{item.periodKey},
#{item.periodName},
#{item.periodStartDate},
#{item.periodEndDate},
#{item.sysOrigin},
#{item.countryCode},
#{item.countryName},
IFNULL(#{item.countryNewUser}, 0),
IFNULL(#{item.newUserRecharge}, 0),
IFNULL(#{item.officialRecharge}, 0),
IFNULL(#{item.mifapayRecharge}, 0),
IFNULL(#{item.googleRecharge}, 0),
IFNULL(#{item.dealerRecharge}, 0),
IFNULL(#{item.salaryExchange}, 0),
IFNULL(#{item.luckyGiftTotalFlow}, 0),
IFNULL(#{item.luckyGiftUser}, 0),
IFNULL(#{item.luckyGiftPayout}, 0),
IFNULL(#{item.gameTotalFlow}, 0),
IFNULL(#{item.gameUser}, 0),
IFNULL(#{item.gamePayout}, 0),
NOW()
)
</foreach>
ON DUPLICATE KEY UPDATE
period_name = VALUES(period_name),
period_start_date = VALUES(period_start_date),
period_end_date = VALUES(period_end_date),
country_name = VALUES(country_name),
country_new_user = VALUES(country_new_user),
new_user_recharge = VALUES(new_user_recharge),
official_recharge = VALUES(official_recharge),
mifapay_recharge = VALUES(mifapay_recharge),
google_recharge = VALUES(google_recharge),
dealer_recharge = VALUES(dealer_recharge),
salary_exchange = VALUES(salary_exchange),
lucky_gift_total_flow = VALUES(lucky_gift_total_flow),
lucky_gift_user = VALUES(lucky_gift_user),
lucky_gift_payout = VALUES(lucky_gift_payout),
game_total_flow = VALUES(game_total_flow),
game_user = VALUES(game_user),
game_payout = VALUES(game_payout),
refreshed_at = NOW()
</insert>
<delete id="deletePeriodUserMetrics">
DELETE FROM country_dashboard_period_user_metric
WHERE period_type = #{periodType}
AND period_key = #{periodKey}
<if test="sysOrigin != null and sysOrigin != ''">
AND sys_origin = #{sysOrigin}
</if>
</delete>
<insert id="insertPeriodNewUserMetrics">
INSERT IGNORE INTO country_dashboard_period_user_metric (
period_type,
period_key,
period_name,
period_start_date,
period_end_date,
sys_origin,
country_code,
country_name,
user_id,
metric_type
)
SELECT DISTINCT
#{periodType},
#{periodKey},
#{periodName},
#{periodStartDate},
#{periodEndDate},
#{sysOrigin},
IFNULL(NULLIF(u.country_code, ''), 'UNKNOWN') AS countryCode,
IFNULL(NULLIF(u.country_name, ''), IFNULL(NULLIF(u.country_code, ''), 'UNKNOWN')) AS countryName,
u.id AS userId,
'COUNTRY_NEW_USER'
FROM user_base_info u
WHERE 1 = 1
AND (u.is_del = 0 OR u.is_del IS NULL)
<if test="sysOrigin != null and sysOrigin != ''">
AND u.origin_sys = #{sysOrigin}
</if>
<include refid="UserTimeWhere"/>
</insert>
<insert id="insertPeriodLuckyGiftUserMetrics">
INSERT IGNORE INTO country_dashboard_period_user_metric (
period_type,
period_key,
period_name,
period_start_date,
period_end_date,
sys_origin,
country_code,
country_name,
user_id,
metric_type
)
SELECT DISTINCT
#{periodType},
#{periodKey},
#{periodName},
#{periodStartDate},
#{periodEndDate},
#{sysOrigin},
IFNULL(NULLIF(u.country_code, ''), 'UNKNOWN') AS countryCode,
IFNULL(NULLIF(u.country_name, ''), IFNULL(NULLIF(u.country_code, ''), 'UNKNOWN')) AS countryName,
t.user_id AS userId,
'LUCKY_GIFT_USER'
FROM game_lucky_gift_count t
INNER JOIN user_base_info u ON u.id = t.user_id
WHERE 1 = 1
AND (u.is_del = 0 OR u.is_del IS NULL)
<if test="sysOrigin != null and sysOrigin != ''">
AND u.origin_sys = #{sysOrigin}
</if>
<include refid="SourceTimeWhere"/>
<include refid="SourceSysOriginWhere"/>
</insert>
<insert id="insertPeriodGameUserMetrics">
INSERT IGNORE INTO country_dashboard_period_user_metric (
period_type,
period_key,
period_name,
period_start_date,
period_end_date,
sys_origin,
country_code,
country_name,
user_id,
metric_type
)
SELECT DISTINCT
#{periodType},
#{periodKey},
#{periodName},
#{periodStartDate},
#{periodEndDate},
#{sysOrigin},
IFNULL(NULLIF(u.country_code, ''), 'UNKNOWN') AS countryCode,
IFNULL(NULLIF(u.country_name, ''), IFNULL(NULLIF(u.country_code, ''), 'UNKNOWN')) AS countryName,
s.user_id AS userId,
'GAME_USER'
FROM (
SELECT t.user_id
FROM (
<include refid="WalletGoldAssetRecordSource"/>
) t
WHERE <include refid="WalletGameEventWhere"/>
UNION
SELECT t.user_id
FROM game_lucky_box_count t
WHERE 1 = 1
<include refid="SourceTimeWhere"/>
<include refid="SourceSysOriginWhere"/>
) s
INNER JOIN user_base_info u ON u.id = s.user_id
WHERE 1 = 1
AND (u.is_del = 0 OR u.is_del IS NULL)
<if test="sysOrigin != null and sysOrigin != ''">
AND u.origin_sys = #{sysOrigin}
</if>
</insert>
<insert id="insertAllPeriodUserMetricsFromDaily">
INSERT IGNORE INTO country_dashboard_period_user_metric (
period_type,
period_key,
period_name,
period_start_date,
period_end_date,
sys_origin,
country_code,
country_name,
user_id,
metric_type
)
SELECT DISTINCT
'ALL',
'ALL',
'全部',
NULL,
NULL,
t.sys_origin,
t.country_code,
t.country_name,
t.user_id,
t.metric_type
FROM country_dashboard_period_user_metric t
WHERE t.period_type = 'DAY'
<if test="sysOrigin != null and sysOrigin != ''">
AND t.sys_origin = #{sysOrigin}
</if>
</insert>
<select id="listPeriodUserMetricCounts"
resultType="com.red.circle.console.infra.database.rds.dto.datav.CountryDashboardMetricDTO">
SELECT
t.country_code AS countryCode,
MAX(t.country_name) AS countryName,
t.period_key AS periodKey,
MAX(t.period_name) AS periodName,
SUM(CASE WHEN t.metric_type = 'COUNTRY_NEW_USER' THEN 1 ELSE 0 END) AS countryNewUser,
SUM(CASE WHEN t.metric_type = 'LUCKY_GIFT_USER' THEN 1 ELSE 0 END) AS luckyGiftUser,
SUM(CASE WHEN t.metric_type = 'GAME_USER' THEN 1 ELSE 0 END) AS gameUser
FROM country_dashboard_period_user_metric t
WHERE t.period_type = #{periodType}
AND t.period_key = #{periodKey}
<if test="sysOrigin != null and sysOrigin != ''">
AND t.sys_origin = #{sysOrigin}
</if>
GROUP BY countryCode, periodKey
</select>
<select id="listAllPeriodAmountsFromDaily"
resultType="com.red.circle.console.infra.database.rds.dto.datav.CountryDashboardMetricDTO">
SELECT
t.country_code AS countryCode,
MAX(t.country_name) AS countryName,
'ALL' AS periodKey,
'全部' AS periodName,
IFNULL(SUM(t.new_user_recharge), 0) AS newUserRecharge,
IFNULL(SUM(t.official_recharge), 0) AS officialRecharge,
IFNULL(SUM(t.mifapay_recharge), 0) AS mifapayRecharge,
IFNULL(SUM(t.google_recharge), 0) AS googleRecharge,
IFNULL(SUM(t.dealer_recharge), 0) AS dealerRecharge,
IFNULL(SUM(t.salary_exchange), 0) AS salaryExchange,
IFNULL(SUM(t.lucky_gift_total_flow), 0) AS luckyGiftTotalFlow,
IFNULL(SUM(t.lucky_gift_payout), 0) AS luckyGiftPayout,
IFNULL(SUM(t.game_total_flow), 0) AS gameTotalFlow,
IFNULL(SUM(t.game_payout), 0) AS gamePayout
FROM country_dashboard_daily_metric t
WHERE 1 = 1
<if test="sysOrigin != null and sysOrigin != ''">
AND t.sys_origin = #{sysOrigin}
</if>
GROUP BY countryCode
</select>
<select id="listNewUsers"
resultType="com.red.circle.console.infra.database.rds.dto.datav.CountryDashboardMetricDTO">
SELECT

174
rc-service/rc-service-console/docs/country-dashboard-data-query.md Normal file
View File

@ -0,0 +1,174 @@
# 国家数据大屏数据查询技术文档
## 目标
国家数据大屏接口 `/console/datav/country-dashboard` 原来在页面请求时实时扫描订单、用户、游戏流水和 wallet 分表。线上慢查询主要集中在 `wallet_gold_asset_record_1..12` 的游戏流水、游戏人数统计。当前改为 SQL 指标预聚合，页面优先查询汇总表；汇总缺失或请求范围不符合预聚合口径时，才回退原实时 SQL。
## 查询入口
- Controller: `DatavRestController#countryDashboard`
- Service: `CountryDashboardServiceImpl#query`
- DAO: `CountryDashboardDAO`
- Mapper: `CountryDashboardDAO.xml`
查询参数：
- `periodType`: `DAY` / `WEEK` / `MONTH` / `ALL`
- `startDate`, `endDate`: 闭区间日期，后端转为 `[startDate 00:00:00, endDate + 1 day 00:00:00)`
- `countryKeyword`: 国家码或国家名模糊匹配
- `sysOrigin`: 系统来源，预聚合查询必须传
## 预聚合表
### `country_dashboard_daily_metric`
日级 SQL 指标表，按 `stat_date + sys_origin + country_code` 唯一。
用途：
- 保留日级汇总结果。
- 作为 `ALL` 周期金额汇总的数据来源。
- 作为历史回填和排查的基础日表。
### `country_dashboard_period_metric`
周期汇总表，按 `period_type + period_key + sys_origin + country_code` 唯一。
周期键示例：
- `DAY`: `2026-05-07`
- `WEEK`: `2026-W19`
- `MONTH`: `2026-05`
- `ALL`: `ALL`
页面查询优先读取这张表。
### `country_dashboard_period_user_metric`
周期用户去重明细表，按 `period_type + period_key + sys_origin + country_code + user_id + metric_type` 唯一。
当前维护的 `metric_type`：
- `COUNTRY_NEW_USER`: 新增用户人数
- `LUCKY_GIFT_USER`: 幸运礼物用户人数
- `GAME_USER`: 游戏用户人数
`WEEK/MONTH/ALL` 的人数类指标不再从日表简单相加，而是基于这张明细表聚合，避免同一个用户跨多天被重复计算。
## 查询路由
`CountryDashboardServiceImpl#shouldUsePrecomputedMetrics` 会先判断是否能走预聚合。
走预聚合的条件：
- `sysOrigin` 不为空。
- `DAY`: 查询范围内每一天都有 `country_dashboard_period_metric` 数据。
- `WEEK/MONTH`: 查询范围必须按周期边界对齐，当前未结束的周/月允许到今天。
- `ALL`: 无 `startDate/endDate` 时固定走 `period_type=ALL, period_key=ALL`。如果汇总表未回填，返回空汇总，不再回退全历史实时扫描。
`DAY/WEEK/MONTH` 不满足条件时回退原实时 SQL，保证数据不因预聚合缺失而返回错误。`ALL` 不回退实时 SQL，是为了避免一次后台点击触发全历史 wallet 分表扫描。
## 刷新任务
任务类：`CountryDashboardDailyMetricTask`
默认任务：
- 每 10 分钟刷新最近 1 天：`red-circle.country-dashboard.refresh-today-cron`
- 每天 00:20 刷新最近 8 天：`red-circle.country-dashboard.refresh-recent-cron`
默认平台：
```yaml
red-circle:
country-dashboard:
refresh-sys-origins: LIKEI
```
需要刷新多个平台时配置为逗号分隔，例如：
```yaml
red-circle:
country-dashboard:
refresh-sys-origins: LIKEI,TARAB
```
刷新链路：
1. 刷新日表 `country_dashboard_daily_metric`。
2. 刷新对应 `DAY` 周期表和用户去重明细。
3. 根据最近日期刷新受影响的 `WEEK`、`MONTH`。
4. 基于已存在的日表和日级用户明细刷新 `ALL`。
## 指标来源
SQL 预聚合指标：
- 新增用户：`user_base_info`
- 新用户充值、官方充值、Google 充值：`order_purchase_history`
- MifaPay 充值：`order_user_purchase_pay`
- 代理充值：`likei_wallet.user_freight_balance_running_water`
- 工资兑换：`user_salary_account_running_water`, `user_salary_diamond_running_water`
- 幸运礼物流水/返奖：`game_lucky_gift_count`
- 游戏流水/返奖：`wallet_gold_asset_record_1..12`, `game_lucky_box_count`
- 游戏人数：`country_dashboard_period_user_metric` 的 `GAME_USER`
仍实时从 Mongo 聚合的指标：
- 礼物消耗：`gift_give_running_water`
- 幸运礼物主播分成：`gift_give_running_water`
- 日活：`user_daily_active_log`
- 留存：`user_daily_active_log` + 注册 cohort
## 上线和回填
上线前可以先执行：
- `sql/20260507_country_dashboard_daily_metric.sql`
- `sql/20260507_country_dashboard_period_metric.sql`
为避免遗漏，console 的刷新和手动回填入口会先执行同等的 `CREATE TABLE IF NOT EXISTS` 检查，再写入预聚合数据。
首次上线后，预聚合表为空时接口会回退实时 SQL。为了尽快切到预聚合，需要先回填最近常用查询范围，例如最近 8 天、当前周、当前月。
`ALL` 的数据由已回填的日表和日级用户明细汇总而来。如果需要后台“全部”展示完整历史，必须先完成历史日数据回填；否则 `ALL` 只代表当前已经进入预聚合表的数据范围。
手动回填入口：
```bash
curl -X POST 'http://127.0.0.1:{console_port}/console/datav/country-dashboard/backfill?startDate=2026-04-08&endDate=2026-05-07&sysOrigin=LIKEI'
```
该入口会按天重算日表和 `DAY` 周期，再刷新受影响的 `WEEK`、`MONTH` 和 `ALL`。
## 验证 SQL
检查某天是否已刷新：
```sql
SELECT stat_date, sys_origin, COUNT(1)
FROM country_dashboard_daily_metric
WHERE sys_origin = 'LIKEI'
GROUP BY stat_date, sys_origin
ORDER BY stat_date DESC;
```
检查周期汇总：
```sql
SELECT period_type, period_key, sys_origin, COUNT(1)
FROM country_dashboard_period_metric
WHERE sys_origin = 'LIKEI'
GROUP BY period_type, period_key, sys_origin
ORDER BY period_type, period_key DESC;
```
检查用户去重明细：
```sql
SELECT period_type, period_key, metric_type, COUNT(1)
FROM country_dashboard_period_user_metric
WHERE sys_origin = 'LIKEI'
GROUP BY period_type, period_key, metric_type
ORDER BY period_type, period_key DESC, metric_type;
```

30
sql/20260507_country_dashboard_daily_metric.sql Normal file
View File

@ -0,0 +1,30 @@
SET NAMES utf8mb4;
CREATE TABLE IF NOT EXISTS `country_dashboard_daily_metric` (
`id` bigint NOT NULL AUTO_INCREMENT COMMENT '主键',
`stat_date` date NOT NULL COMMENT '统计日期',
`date_number` int NOT NULL COMMENT '统计日期yyyyMMdd',
`sys_origin` varchar(32) CHARACTER SET utf8mb4 COLLATE utf8mb4_unicode_ci NOT NULL DEFAULT '' COMMENT '系统来源',
`country_code` varchar(64) CHARACTER SET utf8mb4 COLLATE utf8mb4_unicode_ci NOT NULL DEFAULT 'UNKNOWN' COMMENT '国家码',
`country_name` varchar(128) CHARACTER SET utf8mb4 COLLATE utf8mb4_unicode_ci NOT NULL DEFAULT 'UNKNOWN' COMMENT '国家名称',
`country_new_user` bigint NOT NULL DEFAULT 0 COMMENT '新增用户数',
`new_user_recharge` decimal(20,2) NOT NULL DEFAULT 0.00 COMMENT '新增用户充值',
`official_recharge` decimal(20,2) NOT NULL DEFAULT 0.00 COMMENT '官方充值',
`mifapay_recharge` decimal(20,2) NOT NULL DEFAULT 0.00 COMMENT 'MifaPay充值',
`google_recharge` decimal(20,2) NOT NULL DEFAULT 0.00 COMMENT 'Google充值',
`dealer_recharge` decimal(20,2) NOT NULL DEFAULT 0.00 COMMENT '代理充值',
`salary_exchange` decimal(20,2) NOT NULL DEFAULT 0.00 COMMENT '工资兑换',
`lucky_gift_total_flow` decimal(20,2) NOT NULL DEFAULT 0.00 COMMENT '幸运礼物流水',
`lucky_gift_user` bigint NOT NULL DEFAULT 0 COMMENT '幸运礼物用户数',
`lucky_gift_payout` decimal(20,2) NOT NULL DEFAULT 0.00 COMMENT '幸运礼物返奖',
`game_total_flow` decimal(20,2) NOT NULL DEFAULT 0.00 COMMENT '游戏流水',
`game_user` bigint NOT NULL DEFAULT 0 COMMENT '游戏用户数',
`game_payout` decimal(20,2) NOT NULL DEFAULT 0.00 COMMENT '游戏返奖',
`refreshed_at` datetime NOT NULL DEFAULT CURRENT_TIMESTAMP COMMENT '最近刷新时间',
`create_time` datetime NOT NULL DEFAULT CURRENT_TIMESTAMP COMMENT '创建时间',
`update_time` datetime NOT NULL DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP COMMENT '更新时间',
PRIMARY KEY (`id`),
UNIQUE KEY `uk_date_origin_country` (`stat_date`, `sys_origin`, `country_code`),
KEY `idx_origin_date` (`sys_origin`, `stat_date`),
KEY `idx_country_date` (`country_code`, `stat_date`)
) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_unicode_ci COMMENT='国家数据大屏日预聚合';

52
sql/20260507_country_dashboard_period_metric.sql Normal file
View File

@ -0,0 +1,52 @@
SET NAMES utf8mb4;
CREATE TABLE IF NOT EXISTS `country_dashboard_period_metric` (
`id` bigint NOT NULL AUTO_INCREMENT COMMENT '主键',
`period_type` varchar(16) CHARACTER SET utf8mb4 COLLATE utf8mb4_unicode_ci NOT NULL COMMENT '周期类型:DAY/WEEK/MONTH/ALL',
`period_key` varchar(32) CHARACTER SET utf8mb4 COLLATE utf8mb4_unicode_ci NOT NULL COMMENT '周期键',
`period_name` varchar(64) CHARACTER SET utf8mb4 COLLATE utf8mb4_unicode_ci NOT NULL DEFAULT '' COMMENT '周期名称',
`period_start_date` date DEFAULT NULL COMMENT '周期开始日期',
`period_end_date` date DEFAULT NULL COMMENT '周期结束日期,开区间',
`sys_origin` varchar(32) CHARACTER SET utf8mb4 COLLATE utf8mb4_unicode_ci NOT NULL DEFAULT '' COMMENT '系统来源',
`country_code` varchar(64) CHARACTER SET utf8mb4 COLLATE utf8mb4_unicode_ci NOT NULL DEFAULT 'UNKNOWN' COMMENT '国家码',
`country_name` varchar(128) CHARACTER SET utf8mb4 COLLATE utf8mb4_unicode_ci NOT NULL DEFAULT 'UNKNOWN' COMMENT '国家名称',
`country_new_user` bigint NOT NULL DEFAULT 0 COMMENT '新增用户数',
`new_user_recharge` decimal(20,2) NOT NULL DEFAULT 0.00 COMMENT '新增用户充值',
`official_recharge` decimal(20,2) NOT NULL DEFAULT 0.00 COMMENT '官方充值',
`mifapay_recharge` decimal(20,2) NOT NULL DEFAULT 0.00 COMMENT 'MifaPay充值',
`google_recharge` decimal(20,2) NOT NULL DEFAULT 0.00 COMMENT 'Google充值',
`dealer_recharge` decimal(20,2) NOT NULL DEFAULT 0.00 COMMENT '代理充值',
`salary_exchange` decimal(20,2) NOT NULL DEFAULT 0.00 COMMENT '工资兑换',
`lucky_gift_total_flow` decimal(20,2) NOT NULL DEFAULT 0.00 COMMENT '幸运礼物流水',
`lucky_gift_user` bigint NOT NULL DEFAULT 0 COMMENT '幸运礼物用户数',
`lucky_gift_payout` decimal(20,2) NOT NULL DEFAULT 0.00 COMMENT '幸运礼物返奖',
`game_total_flow` decimal(20,2) NOT NULL DEFAULT 0.00 COMMENT '游戏流水',
`game_user` bigint NOT NULL DEFAULT 0 COMMENT '游戏用户数',
`game_payout` decimal(20,2) NOT NULL DEFAULT 0.00 COMMENT '游戏返奖',
`refreshed_at` datetime NOT NULL DEFAULT CURRENT_TIMESTAMP COMMENT '最近刷新时间',
`create_time` datetime NOT NULL DEFAULT CURRENT_TIMESTAMP COMMENT '创建时间',
`update_time` datetime NOT NULL DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP COMMENT '更新时间',
PRIMARY KEY (`id`),
UNIQUE KEY `uk_period_origin_country` (`period_type`, `period_key`, `sys_origin`, `country_code`),
KEY `idx_origin_period` (`sys_origin`, `period_type`, `period_key`),
KEY `idx_period_date` (`period_type`, `period_start_date`, `period_end_date`)
) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_unicode_ci COMMENT='国家数据大屏周期预聚合';
CREATE TABLE IF NOT EXISTS `country_dashboard_period_user_metric` (
`id` bigint NOT NULL AUTO_INCREMENT COMMENT '主键',
`period_type` varchar(16) CHARACTER SET utf8mb4 COLLATE utf8mb4_unicode_ci NOT NULL COMMENT '周期类型:DAY/WEEK/MONTH/ALL',
`period_key` varchar(32) CHARACTER SET utf8mb4 COLLATE utf8mb4_unicode_ci NOT NULL COMMENT '周期键',
`period_name` varchar(64) CHARACTER SET utf8mb4 COLLATE utf8mb4_unicode_ci NOT NULL DEFAULT '' COMMENT '周期名称',
`period_start_date` date DEFAULT NULL COMMENT '周期开始日期',
`period_end_date` date DEFAULT NULL COMMENT '周期结束日期,开区间',
`sys_origin` varchar(32) CHARACTER SET utf8mb4 COLLATE utf8mb4_unicode_ci NOT NULL DEFAULT '' COMMENT '系统来源',
`country_code` varchar(64) CHARACTER SET utf8mb4 COLLATE utf8mb4_unicode_ci NOT NULL DEFAULT 'UNKNOWN' COMMENT '国家码',
`country_name` varchar(128) CHARACTER SET utf8mb4 COLLATE utf8mb4_unicode_ci NOT NULL DEFAULT 'UNKNOWN' COMMENT '国家名称',
`user_id` bigint NOT NULL COMMENT '用户ID',
`metric_type` varchar(32) CHARACTER SET utf8mb4 COLLATE utf8mb4_unicode_ci NOT NULL COMMENT '人数指标类型',
`create_time` datetime NOT NULL DEFAULT CURRENT_TIMESTAMP COMMENT '创建时间',
PRIMARY KEY (`id`),
UNIQUE KEY `uk_period_user_metric` (`period_type`, `period_key`, `sys_origin`, `country_code`, `user_id`, `metric_type`),
KEY `idx_period_metric_country` (`period_type`, `period_key`, `sys_origin`, `metric_type`, `country_code`),
KEY `idx_origin_user_metric` (`sys_origin`, `user_id`, `metric_type`)
) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_unicode_ci COMMENT='国家数据大屏周期用户去重明细';