diff --git a/rc-service/rc-service-console/console-adapter/src/main/java/com/red/circle/console/adapter/app/user/DatavRestController.java b/rc-service/rc-service-console/console-adapter/src/main/java/com/red/circle/console/adapter/app/user/DatavRestController.java index 3a27ee6c..85f770a5 100644 --- a/rc-service/rc-service-console/console-adapter/src/main/java/com/red/circle/console/adapter/app/user/DatavRestController.java +++ b/rc-service/rc-service-console/console-adapter/src/main/java/com/red/circle/console/adapter/app/user/DatavRestController.java @@ -1,25 +1,29 @@ package com.red.circle.console.adapter.app.user; -import com.red.circle.common.business.core.enums.SysOriginPlatformEnum; import com.red.circle.console.app.dto.clientobject.datav.CountryDashboardCO; import com.red.circle.console.app.dto.clientobject.datav.CountryDashboardRechargeDetailCO; import com.red.circle.console.app.dto.cmd.datav.CountryDashboardQryCmd; import com.red.circle.console.app.dto.cmd.datav.CountryDashboardRechargeDetailQryCmd; +import com.red.circle.console.app.service.app.datav.CountryDashboardDailyMetricService; import com.red.circle.console.app.service.app.datav.CountryDashboardService; import com.red.circle.console.app.service.app.user.DatavService; import com.red.circle.framework.dto.PageResult; -import com.red.circle.framework.web.controller.BaseController; -import com.red.circle.other.inner.model.cmd.user.OnlineCountQryCmd; -import com.red.circle.other.inner.model.dto.user.ActiveUserCountryCodeDTO; -import java.time.LocalDateTime; -import java.util.List; -import java.util.Objects; -import lombok.RequiredArgsConstructor; -import org.springframework.http.MediaType; -import org.springframework.web.bind.annotation.GetMapping; -import org.springframework.web.bind.annotation.RequestMapping; -import org.springframework.web.bind.annotation.RequestParam; -import org.springframework.web.bind.annotation.RestController; +import com.red.circle.framework.web.controller.BaseController; +import com.red.circle.other.inner.model.cmd.user.OnlineCountQryCmd; +import com.red.circle.other.inner.model.dto.user.ActiveUserCountryCodeDTO; +import java.time.LocalDate; +import java.time.LocalDateTime; +import java.util.List; +import java.util.Map; +import java.util.Objects; +import lombok.RequiredArgsConstructor; +import org.springframework.format.annotation.DateTimeFormat; +import org.springframework.http.MediaType; +import org.springframework.web.bind.annotation.GetMapping; +import org.springframework.web.bind.annotation.PostMapping; +import org.springframework.web.bind.annotation.RequestMapping; +import org.springframework.web.bind.annotation.RequestParam; +import org.springframework.web.bind.annotation.RestController; /** * @author pengliang on 2020/10/16 @@ -32,6 +36,7 @@ public class DatavRestController extends BaseController { private final DatavService userExpandService; private final CountryDashboardService countryDashboardService; + private final CountryDashboardDailyMetricService countryDashboardDailyMetricService; @GetMapping("/active/user-country-code") public List latestActiveUserCountryCode() { @@ -59,4 +64,17 @@ public class DatavRestController extends BaseController { return countryDashboardService.pageRechargeDetails(cmd); } + @PostMapping("/country-dashboard/backfill") + public Map backfillCountryDashboard( + @RequestParam @DateTimeFormat(iso = DateTimeFormat.ISO.DATE) LocalDate startDate, + @RequestParam @DateTimeFormat(iso = DateTimeFormat.ISO.DATE) LocalDate endDate, + @RequestParam(value = "sysOrigin", defaultValue = "LIKEI") String sysOrigin) { + countryDashboardDailyMetricService.refreshRange(startDate, endDate, sysOrigin); + return Map.of( + "status", "success", + "startDate", startDate.toString(), + "endDate", endDate.toString(), + "sysOrigin", sysOrigin); + } + } diff --git a/rc-service/rc-service-console/console-application/src/main/java/com/red/circle/console/app/scheduler/CountryDashboardDailyMetricTask.java b/rc-service/rc-service-console/console-application/src/main/java/com/red/circle/console/app/scheduler/CountryDashboardDailyMetricTask.java new file mode 100644 index 00000000..0026bbc9 --- /dev/null +++ b/rc-service/rc-service-console/console-application/src/main/java/com/red/circle/console/app/scheduler/CountryDashboardDailyMetricTask.java @@ -0,0 +1,46 @@ +package com.red.circle.console.app.scheduler; + +import com.red.circle.component.redis.annotation.TaskCacheLock; +import com.red.circle.console.app.service.app.datav.CountryDashboardDailyMetricService; +import lombok.RequiredArgsConstructor; +import lombok.extern.slf4j.Slf4j; +import org.springframework.beans.factory.annotation.Value; +import org.springframework.scheduling.annotation.Scheduled; +import org.springframework.stereotype.Component; + +/** + * 国家数据大屏日预聚合任务. + */ +@Slf4j +@Component +@RequiredArgsConstructor +public class CountryDashboardDailyMetricTask { + + private final CountryDashboardDailyMetricService countryDashboardDailyMetricService; + + @Value("${red-circle.country-dashboard.refresh-today-days:1}") + private int refreshTodayDays; + + @Value("${red-circle.country-dashboard.refresh-recent-days:8}") + private int refreshRecentDays; + + @Scheduled(cron = "${red-circle.country-dashboard.refresh-today-cron:0 */10 * * * ?}", + zone = "Asia/Shanghai") + @TaskCacheLock(key = "COUNTRY_DASHBOARD_DAILY_METRIC_REFRESH_TODAY", expireSecond = 60 * 8) + public void refreshToday() { + long start = System.currentTimeMillis(); + countryDashboardDailyMetricService.refreshRecentDays(refreshTodayDays); + log.info("country dashboard refresh today task finished, days={}, costMs={}", + refreshTodayDays, System.currentTimeMillis() - start); + } + + @Scheduled(cron = "${red-circle.country-dashboard.refresh-recent-cron:0 20 0 * * ?}", + zone = "Asia/Shanghai") + @TaskCacheLock(key = "COUNTRY_DASHBOARD_DAILY_METRIC_REFRESH_RECENT", expireSecond = 60 * 60) + public void refreshRecent() { + long start = System.currentTimeMillis(); + countryDashboardDailyMetricService.refreshRecentDays(refreshRecentDays); + log.info("country dashboard refresh recent task finished, days={}, costMs={}", + refreshRecentDays, System.currentTimeMillis() - start); + } +} diff --git a/rc-service/rc-service-console/console-application/src/main/java/com/red/circle/console/app/service/app/datav/CountryDashboardDailyMetricService.java b/rc-service/rc-service-console/console-application/src/main/java/com/red/circle/console/app/service/app/datav/CountryDashboardDailyMetricService.java new file mode 100644 index 00000000..be4d677d --- /dev/null +++ b/rc-service/rc-service-console/console-application/src/main/java/com/red/circle/console/app/service/app/datav/CountryDashboardDailyMetricService.java @@ -0,0 +1,438 @@ +package com.red.circle.console.app.service.app.datav; + +import com.red.circle.common.business.core.enums.SysOriginPlatformEnum; +import com.red.circle.console.infra.database.rds.dao.datav.CountryDashboardDAO; +import com.red.circle.console.infra.database.rds.dto.datav.CountryDashboardDailyMetricDTO; +import com.red.circle.console.infra.database.rds.dto.datav.CountryDashboardMetricDTO; +import com.red.circle.console.infra.database.rds.dto.datav.CountryDashboardPeriodMetricDTO; +import java.math.BigDecimal; +import java.sql.Timestamp; +import java.time.DayOfWeek; +import java.time.LocalDate; +import java.time.LocalDateTime; +import java.time.format.DateTimeFormatter; +import java.time.temporal.ChronoUnit; +import java.time.temporal.WeekFields; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.LinkedHashSet; +import java.util.LinkedHashMap; +import java.util.List; +import java.util.Locale; +import java.util.Map; +import java.util.Objects; +import java.util.Set; +import lombok.RequiredArgsConstructor; +import lombok.extern.slf4j.Slf4j; +import org.springframework.beans.factory.annotation.Value; +import org.springframework.stereotype.Service; +import org.springframework.transaction.support.TransactionTemplate; + +/** + * 国家数据大屏日预聚合. + */ +@Slf4j +@Service +@RequiredArgsConstructor +public class CountryDashboardDailyMetricService { + + private static final String PERIOD_DAY = "DAY"; + private static final String PERIOD_WEEK = "WEEK"; + private static final String PERIOD_MONTH = "MONTH"; + private static final String PERIOD_ALL = "ALL"; + private static final DateTimeFormatter DATE_NUMBER_FORMATTER = DateTimeFormatter.BASIC_ISO_DATE; + private static final DateTimeFormatter DATE_FORMATTER = DateTimeFormatter.ofPattern("yyyy-MM-dd"); + private static final DateTimeFormatter MONTH_FORMATTER = DateTimeFormatter.ofPattern("yyyy-MM"); + private static final WeekFields WEEK_FIELDS = WeekFields.ISO; + + private final CountryDashboardDAO countryDashboardDAO; + private final TransactionTemplate transactionTemplate; + + @Value("${red-circle.country-dashboard.refresh-sys-origins:LIKEI}") + private String refreshSysOrigins; + + private volatile boolean metricSchemaEnsured; + + public void refreshRecentDays(int days) { + ensureMetricSchema(); + int safeDays = Math.max(days, 1); + LocalDate today = LocalDate.now(); + List statDates = new ArrayList<>(); + for (int i = safeDays - 1; i >= 0; i--) { + statDates.add(today.minusDays(i)); + } + + List sysOrigins = resolveSysOrigins(); + for (String sysOrigin : sysOrigins) { + for (LocalDate statDate : statDates) { + try { + refreshDay(statDate, sysOrigin); + } catch (Exception ex) { + log.error("refresh country dashboard daily metric failed, statDate={}, sysOrigin={}", + statDate, sysOrigin, ex); + } + } + refreshAffectedPeriods(statDates, sysOrigin, false); + try { + refreshAll(sysOrigin); + } catch (Exception ex) { + log.error("refresh country dashboard all metric failed, sysOrigin={}", sysOrigin, ex); + } + } + } + + public void refreshRange(LocalDate startDate, LocalDate endDate, String sysOrigin) { + ensureMetricSchema(); + if (startDate == null || endDate == null) { + throw new IllegalArgumentException("startDate and endDate are required"); + } + if (endDate.isBefore(startDate)) { + throw new IllegalArgumentException("endDate must be greater than or equal to startDate"); + } + long dayCount = ChronoUnit.DAYS.between(startDate, endDate) + 1; + if (dayCount > 366) { + throw new IllegalArgumentException("backfill range can not exceed 366 days"); + } + + String safeSysOrigin = trimToNull(sysOrigin); + if (safeSysOrigin == null) { + safeSysOrigin = SysOriginPlatformEnum.LIKEI.name(); + } + + long start = System.currentTimeMillis(); + List statDates = new ArrayList<>(); + for (long i = 0; i < dayCount; i++) { + statDates.add(startDate.plusDays(i)); + } + + log.info("refresh country dashboard range start, startDate={}, endDate={}, days={}, sysOrigin={}", + startDate, endDate, dayCount, safeSysOrigin); + for (LocalDate statDate : statDates) { + refreshDay(statDate, safeSysOrigin); + } + refreshAffectedPeriods(statDates, safeSysOrigin, true); + refreshAll(safeSysOrigin); + log.info("refresh country dashboard range success, startDate={}, endDate={}, days={}, sysOrigin={}, costMs={}", + startDate, endDate, dayCount, safeSysOrigin, System.currentTimeMillis() - start); + } + + private void ensureMetricSchema() { + if (metricSchemaEnsured) { + return; + } + synchronized (this) { + if (metricSchemaEnsured) { + return; + } + countryDashboardDAO.createDailyMetricTable(); + countryDashboardDAO.createPeriodMetricTable(); + countryDashboardDAO.createPeriodUserMetricTable(); + metricSchemaEnsured = true; + log.info("country dashboard metric tables ensured"); + } + } + + public void refreshDay(LocalDate statDate, String sysOrigin) { + long start = System.currentTimeMillis(); + LocalDateTime startTime = statDate.atStartOfDay(); + LocalDateTime endTime = statDate.plusDays(1).atStartOfDay(); + Long startTimeMillis = Timestamp.valueOf(startTime).getTime(); + Long endTimeMillis = Timestamp.valueOf(endTime).getTime(); + Map rows = new LinkedHashMap<>(); + + mergeSql(rows, countryDashboardDAO.listNewUsers( + PERIOD_DAY, startTime, endTime, null, sysOrigin), statDate, sysOrigin); + mergeSql(rows, countryDashboardDAO.listNewUserRecharge( + PERIOD_DAY, startTime, endTime, null, sysOrigin), statDate, sysOrigin); + mergeSql(rows, countryDashboardDAO.listOfficialRecharge( + PERIOD_DAY, startTime, endTime, null, sysOrigin), statDate, sysOrigin); + mergeSql(rows, countryDashboardDAO.listMifapayRecharge( + PERIOD_DAY, startTime, endTime, null, sysOrigin), statDate, sysOrigin); + mergeSql(rows, countryDashboardDAO.listGoogleRecharge( + PERIOD_DAY, startTime, endTime, null, sysOrigin), statDate, sysOrigin); + mergeSql(rows, countryDashboardDAO.listDealerRecharge( + PERIOD_DAY, startTime, endTime, null, sysOrigin), statDate, sysOrigin); + mergeSql(rows, countryDashboardDAO.listSalaryExchange( + PERIOD_DAY, startTime, endTime, null, sysOrigin), statDate, sysOrigin); + mergeSql(rows, countryDashboardDAO.listLuckyGift( + PERIOD_DAY, startTime, endTime, null, sysOrigin), statDate, sysOrigin); + mergeSql(rows, countryDashboardDAO.listWalletGame( + PERIOD_DAY, startTime, endTime, startTimeMillis, endTimeMillis, null, sysOrigin), + statDate, sysOrigin); + mergeSql(rows, countryDashboardDAO.listLuckyBoxGame( + PERIOD_DAY, startTime, endTime, null, sysOrigin), statDate, sysOrigin); + mergeSql(rows, countryDashboardDAO.listGameUser( + PERIOD_DAY, startTime, endTime, startTimeMillis, endTimeMillis, null, sysOrigin), + statDate, sysOrigin); + + List metrics = new ArrayList<>(rows.values()); + RefreshWriteResult writeResult = transactionTemplate.execute(status -> { + int deleted = countryDashboardDAO.deleteDailyMetrics(statDate, sysOrigin); + int inserted = metrics.isEmpty() ? 0 : countryDashboardDAO.upsertDailyMetrics(metrics); + return new RefreshWriteResult(deleted, inserted, 0); + }); + log.info("refresh country dashboard daily metric success, statDate={}, sysOrigin={}, countries={}, deleted={}, inserted={}, costMs={}", + statDate, sysOrigin, metrics.size(), writeResult.deleted(), writeResult.inserted(), + System.currentTimeMillis() - start); + refreshPeriod(toDayWindow(statDate), sysOrigin); + } + + private void refreshAffectedPeriods(List statDates, String sysOrigin, + boolean failFast) { + Set windows = new LinkedHashSet<>(); + for (LocalDate statDate : statDates) { + windows.add(toWeekWindow(statDate)); + windows.add(toMonthWindow(statDate)); + } + for (PeriodWindow window : windows) { + try { + refreshPeriod(window, sysOrigin); + } catch (Exception ex) { + if (failFast) { + throw ex; + } + log.error("refresh country dashboard period metric failed, periodType={}, periodKey={}, sysOrigin={}", + window.periodType(), window.periodKey(), sysOrigin, ex); + } + } + } + + private void refreshPeriod(PeriodWindow window, String sysOrigin) { + long start = System.currentTimeMillis(); + LocalDateTime startTime = window.periodStartDate().atStartOfDay(); + LocalDateTime endTime = window.periodEndDate().atStartOfDay(); + Long startTimeMillis = Timestamp.valueOf(startTime).getTime(); + Long endTimeMillis = Timestamp.valueOf(endTime).getTime(); + Map rows = new LinkedHashMap<>(); + + mergePeriodAmounts(rows, countryDashboardDAO.listNewUserRecharge( + window.periodType(), startTime, endTime, null, sysOrigin), window, sysOrigin); + mergePeriodAmounts(rows, countryDashboardDAO.listOfficialRecharge( + window.periodType(), startTime, endTime, null, sysOrigin), window, sysOrigin); + mergePeriodAmounts(rows, countryDashboardDAO.listMifapayRecharge( + window.periodType(), startTime, endTime, null, sysOrigin), window, sysOrigin); + mergePeriodAmounts(rows, countryDashboardDAO.listGoogleRecharge( + window.periodType(), startTime, endTime, null, sysOrigin), window, sysOrigin); + mergePeriodAmounts(rows, countryDashboardDAO.listDealerRecharge( + window.periodType(), startTime, endTime, null, sysOrigin), window, sysOrigin); + mergePeriodAmounts(rows, countryDashboardDAO.listSalaryExchange( + window.periodType(), startTime, endTime, null, sysOrigin), window, sysOrigin); + mergePeriodAmounts(rows, countryDashboardDAO.listLuckyGift( + window.periodType(), startTime, endTime, null, sysOrigin), window, sysOrigin); + mergePeriodAmounts(rows, countryDashboardDAO.listWalletGame( + window.periodType(), startTime, endTime, startTimeMillis, endTimeMillis, null, sysOrigin), + window, sysOrigin); + mergePeriodAmounts(rows, countryDashboardDAO.listLuckyBoxGame( + window.periodType(), startTime, endTime, null, sysOrigin), window, sysOrigin); + + RefreshWriteResult writeResult = transactionTemplate.execute(status -> { + countryDashboardDAO.deletePeriodUserMetrics(window.periodType(), window.periodKey(), sysOrigin); + int newUsers = countryDashboardDAO.insertPeriodNewUserMetrics( + window.periodType(), window.periodKey(), window.periodName(), + window.periodStartDate(), window.periodEndDate(), startTime, endTime, sysOrigin); + int luckyGiftUsers = countryDashboardDAO.insertPeriodLuckyGiftUserMetrics( + window.periodType(), window.periodKey(), window.periodName(), + window.periodStartDate(), window.periodEndDate(), startTime, endTime, sysOrigin); + int gameUsers = countryDashboardDAO.insertPeriodGameUserMetrics( + window.periodType(), window.periodKey(), window.periodName(), + window.periodStartDate(), window.periodEndDate(), startTime, endTime, + startTimeMillis, endTimeMillis, sysOrigin); + + mergePeriodUserCounts(rows, countryDashboardDAO.listPeriodUserMetricCounts( + window.periodType(), window.periodKey(), sysOrigin), window, sysOrigin); + List metrics = new ArrayList<>(rows.values()); + int deleted = countryDashboardDAO.deletePeriodMetrics(window.periodType(), window.periodKey(), sysOrigin); + int inserted = metrics.isEmpty() ? 0 : countryDashboardDAO.upsertPeriodMetrics(metrics); + return new RefreshWriteResult(deleted, inserted, newUsers + luckyGiftUsers + gameUsers); + }); + + log.info("refresh country dashboard period metric success, periodType={}, periodKey={}, sysOrigin={}, countries={}, deleted={}, inserted={}, userRows={}, costMs={}", + window.periodType(), window.periodKey(), sysOrigin, rows.size(), writeResult.deleted(), + writeResult.inserted(), writeResult.userRows(), System.currentTimeMillis() - start); + } + + private void refreshAll(String sysOrigin) { + long start = System.currentTimeMillis(); + PeriodWindow window = new PeriodWindow(PERIOD_ALL, PERIOD_ALL, "全部", null, null); + Map rows = new LinkedHashMap<>(); + mergePeriodAmounts(rows, countryDashboardDAO.listAllPeriodAmountsFromDaily(sysOrigin), + window, sysOrigin); + + RefreshWriteResult writeResult = transactionTemplate.execute(status -> { + countryDashboardDAO.deletePeriodUserMetrics(PERIOD_ALL, PERIOD_ALL, sysOrigin); + int userRows = countryDashboardDAO.insertAllPeriodUserMetricsFromDaily(sysOrigin); + mergePeriodUserCounts(rows, countryDashboardDAO.listPeriodUserMetricCounts( + PERIOD_ALL, PERIOD_ALL, sysOrigin), window, sysOrigin); + List metrics = new ArrayList<>(rows.values()); + int deleted = countryDashboardDAO.deletePeriodMetrics(PERIOD_ALL, PERIOD_ALL, sysOrigin); + int inserted = metrics.isEmpty() ? 0 : countryDashboardDAO.upsertPeriodMetrics(metrics); + return new RefreshWriteResult(deleted, inserted, userRows); + }); + + log.info("refresh country dashboard all metric success, sysOrigin={}, countries={}, deleted={}, inserted={}, userRows={}, costMs={}", + sysOrigin, rows.size(), writeResult.deleted(), writeResult.inserted(), + writeResult.userRows(), System.currentTimeMillis() - start); + } + + private void mergeSql(Map rows, + List metrics, LocalDate statDate, String sysOrigin) { + if (metrics == null || metrics.isEmpty()) { + return; + } + for (CountryDashboardMetricDTO metric : metrics) { + CountryDashboardDailyMetricDTO row = getRow(rows, metric, statDate, sysOrigin); + row.setCountryNewUser(safeLong(row.getCountryNewUser()) + safeLong(metric.getCountryNewUser())); + row.setNewUserRecharge(add(row.getNewUserRecharge(), metric.getNewUserRecharge())); + row.setOfficialRecharge(add(row.getOfficialRecharge(), metric.getOfficialRecharge())); + row.setMifapayRecharge(add(row.getMifapayRecharge(), metric.getMifapayRecharge())); + row.setGoogleRecharge(add(row.getGoogleRecharge(), metric.getGoogleRecharge())); + row.setDealerRecharge(add(row.getDealerRecharge(), metric.getDealerRecharge())); + row.setSalaryExchange(add(row.getSalaryExchange(), metric.getSalaryExchange())); + row.setLuckyGiftTotalFlow(add(row.getLuckyGiftTotalFlow(), metric.getLuckyGiftTotalFlow())); + row.setLuckyGiftUser(safeLong(row.getLuckyGiftUser()) + safeLong(metric.getLuckyGiftUser())); + row.setLuckyGiftPayout(add(row.getLuckyGiftPayout(), metric.getLuckyGiftPayout())); + row.setGameTotalFlow(add(row.getGameTotalFlow(), metric.getGameTotalFlow())); + row.setGameUser(safeLong(row.getGameUser()) + safeLong(metric.getGameUser())); + row.setGamePayout(add(row.getGamePayout(), metric.getGamePayout())); + } + } + + private void mergePeriodAmounts(Map rows, + List metrics, PeriodWindow window, String sysOrigin) { + if (metrics == null || metrics.isEmpty()) { + return; + } + for (CountryDashboardMetricDTO metric : metrics) { + CountryDashboardPeriodMetricDTO row = getPeriodRow(rows, metric, window, sysOrigin); + row.setNewUserRecharge(add(row.getNewUserRecharge(), metric.getNewUserRecharge())); + row.setOfficialRecharge(add(row.getOfficialRecharge(), metric.getOfficialRecharge())); + row.setMifapayRecharge(add(row.getMifapayRecharge(), metric.getMifapayRecharge())); + row.setGoogleRecharge(add(row.getGoogleRecharge(), metric.getGoogleRecharge())); + row.setDealerRecharge(add(row.getDealerRecharge(), metric.getDealerRecharge())); + row.setSalaryExchange(add(row.getSalaryExchange(), metric.getSalaryExchange())); + row.setLuckyGiftTotalFlow(add(row.getLuckyGiftTotalFlow(), metric.getLuckyGiftTotalFlow())); + row.setLuckyGiftPayout(add(row.getLuckyGiftPayout(), metric.getLuckyGiftPayout())); + row.setGameTotalFlow(add(row.getGameTotalFlow(), metric.getGameTotalFlow())); + row.setGamePayout(add(row.getGamePayout(), metric.getGamePayout())); + } + } + + private void mergePeriodUserCounts(Map rows, + List metrics, PeriodWindow window, String sysOrigin) { + if (metrics == null || metrics.isEmpty()) { + return; + } + for (CountryDashboardMetricDTO metric : metrics) { + CountryDashboardPeriodMetricDTO row = getPeriodRow(rows, metric, window, sysOrigin); + row.setCountryNewUser(safeLong(metric.getCountryNewUser())); + row.setLuckyGiftUser(safeLong(metric.getLuckyGiftUser())); + row.setGameUser(safeLong(metric.getGameUser())); + } + } + + private CountryDashboardDailyMetricDTO getRow(Map rows, + CountryDashboardMetricDTO metric, LocalDate statDate, String sysOrigin) { + String countryCode = hasText(metric.getCountryCode()) ? metric.getCountryCode() : "UNKNOWN"; + String countryName = hasText(metric.getCountryName()) ? metric.getCountryName() : countryCode; + return rows.computeIfAbsent(countryCode, + key -> new CountryDashboardDailyMetricDTO() + .setStatDate(statDate) + .setDateNumber(Integer.valueOf(statDate.format(DATE_NUMBER_FORMATTER))) + .setSysOrigin(sysOrigin) + .setCountryCode(countryCode) + .setCountryName(countryName)); + } + + private CountryDashboardPeriodMetricDTO getPeriodRow(Map rows, + CountryDashboardMetricDTO metric, PeriodWindow window, String sysOrigin) { + String countryCode = hasText(metric.getCountryCode()) ? metric.getCountryCode() : "UNKNOWN"; + String countryName = hasText(metric.getCountryName()) ? metric.getCountryName() : countryCode; + return rows.computeIfAbsent(countryCode, + key -> new CountryDashboardPeriodMetricDTO() + .setPeriodType(window.periodType()) + .setPeriodKey(window.periodKey()) + .setPeriodName(window.periodName()) + .setPeriodStartDate(window.periodStartDate()) + .setPeriodEndDate(window.periodEndDate()) + .setSysOrigin(sysOrigin) + .setCountryCode(countryCode) + .setCountryName(countryName)); + } + + private PeriodWindow toDayWindow(LocalDate statDate) { + String key = statDate.format(DATE_FORMATTER); + return new PeriodWindow(PERIOD_DAY, key, key, statDate, statDate.plusDays(1)); + } + + private PeriodWindow toWeekWindow(LocalDate statDate) { + LocalDate start = statDate.with(DayOfWeek.MONDAY); + LocalDate end = capCurrentPeriodEnd(start.plusDays(7)); + int weekYear = statDate.get(WEEK_FIELDS.weekBasedYear()); + int week = statDate.get(WEEK_FIELDS.weekOfWeekBasedYear()); + String key = String.format(Locale.ROOT, "%d-W%02d", weekYear, week); + return new PeriodWindow(PERIOD_WEEK, key, key, start, end); + } + + private PeriodWindow toMonthWindow(LocalDate statDate) { + LocalDate start = statDate.withDayOfMonth(1); + LocalDate end = capCurrentPeriodEnd(start.plusMonths(1)); + String key = statDate.format(MONTH_FORMATTER); + return new PeriodWindow(PERIOD_MONTH, key, key, start, end); + } + + private LocalDate capCurrentPeriodEnd(LocalDate naturalEndExclusive) { + LocalDate tomorrow = LocalDate.now().plusDays(1); + return naturalEndExclusive.isAfter(tomorrow) ? tomorrow : naturalEndExclusive; + } + + private List resolveSysOrigins() { + String config = trimToNull(refreshSysOrigins); + if (config == null) { + return List.of(SysOriginPlatformEnum.LIKEI.name()); + } + if ("ALL".equalsIgnoreCase(config)) { + return SysOriginPlatformEnum.getVoiceSystems().stream() + .map(SysOriginPlatformEnum::name) + .filter(Objects::nonNull) + .distinct() + .toList(); + } + List sysOrigins = Arrays.stream(config.split(",")) + .map(String::trim) + .filter(CountryDashboardDailyMetricService::hasText) + .distinct() + .toList(); + return sysOrigins.isEmpty() ? List.of(SysOriginPlatformEnum.LIKEI.name()) : sysOrigins; + } + + private static BigDecimal add(BigDecimal left, BigDecimal right) { + return safe(left).add(safe(right)); + } + + private static BigDecimal safe(BigDecimal value) { + return value == null ? BigDecimal.ZERO : value; + } + + private static long safeLong(Long value) { + return value == null ? 0L : value; + } + + private static boolean hasText(String value) { + return value != null && !value.trim().isEmpty(); + } + + private static String trimToNull(String value) { + if (value == null) { + return null; + } + String trimmed = value.trim(); + return trimmed.isEmpty() ? null : trimmed; + } + + private record PeriodWindow(String periodType, String periodKey, String periodName, + LocalDate periodStartDate, LocalDate periodEndDate) { + } + + private record RefreshWriteResult(int deleted, int inserted, int userRows) { + } +} diff --git a/rc-service/rc-service-console/console-application/src/main/java/com/red/circle/console/app/service/app/datav/CountryDashboardServiceImpl.java b/rc-service/rc-service-console/console-application/src/main/java/com/red/circle/console/app/service/app/datav/CountryDashboardServiceImpl.java index 98dbdda5..643560e1 100644 --- a/rc-service/rc-service-console/console-application/src/main/java/com/red/circle/console/app/service/app/datav/CountryDashboardServiceImpl.java +++ b/rc-service/rc-service-console/console-application/src/main/java/com/red/circle/console/app/service/app/datav/CountryDashboardServiceImpl.java @@ -33,6 +33,7 @@ import java.util.Objects; import java.util.Set; import java.util.function.BiConsumer; import lombok.RequiredArgsConstructor; +import lombok.extern.slf4j.Slf4j; import org.bson.Document; import org.bson.types.Decimal128; import org.springframework.data.mongodb.core.MongoTemplate; @@ -47,6 +48,7 @@ import org.springframework.stereotype.Service; */ @Service @RequiredArgsConstructor +@Slf4j public class CountryDashboardServiceImpl implements CountryDashboardService { private static final String COLLECTION_GIFT_GIVE_RUNNING_WATER = "gift_give_running_water"; @@ -64,6 +66,41 @@ public class CountryDashboardServiceImpl implements CountryDashboardService { QueryCondition condition = QueryCondition.of(cmd); Map rows = new LinkedHashMap<>(); + if (shouldUsePrecomputedMetrics(condition)) { + mergeSql(rows, countryDashboardDAO.listPrecomputedPeriodMetrics( + condition.periodType, condition.startTime, condition.endTime, + condition.countryKeyword, condition.sysOrigin)); + } else { + mergeLiveSql(rows, condition); + } + + mergeMongo(rows, condition, listGiftConsume(condition), + (row, amount) -> row.setGiftConsume(add(row.getGiftConsume(), amount))); + mergeMongo(rows, condition, listLuckyGiftAnchorShare(condition), + (row, amount) -> row.setLuckyGiftAnchorShare(add(row.getLuckyGiftAnchorShare(), amount))); + mergeDailyActive(rows, condition, listDailyActive(condition)); + mergeRetention(rows, condition); + + List records = rows.values().stream() + .peek(this::calculateFormulaFields) + .sorted(Comparator.comparing(CountryDashboardMetricCO::getPeriodKey, + Comparator.nullsLast(Comparator.reverseOrder())) + .thenComparing(CountryDashboardMetricCO::getCountryCode, + Comparator.nullsLast(String::compareTo))) + .toList(); + + CountryDashboardMetricCO total = buildTotal(records, condition.periodType); + + return new CountryDashboardCO() + .setPeriodType(condition.periodType) + .setStartDate(condition.startDate) + .setEndDate(condition.endDate) + .setComputedAt(LocalDateTime.now().format(DateTimeFormatter.ofPattern("yyyy-MM-dd HH:mm:ss"))) + .setTotal(total) + .setRecords(records); + } + + private void mergeLiveSql(Map rows, QueryCondition condition) { mergeSql(rows, countryDashboardDAO.listNewUsers( condition.periodType, condition.startTime, condition.endTime, condition.countryKeyword, condition.sysOrigin)); @@ -90,38 +127,116 @@ public class CountryDashboardServiceImpl implements CountryDashboardService { condition.countryKeyword, condition.sysOrigin)); mergeSql(rows, countryDashboardDAO.listWalletGame( condition.periodType, condition.startTime, condition.endTime, + condition.startTimeMillis(), condition.endTimeMillis(), condition.countryKeyword, condition.sysOrigin)); mergeSql(rows, countryDashboardDAO.listLuckyBoxGame( condition.periodType, condition.startTime, condition.endTime, condition.countryKeyword, condition.sysOrigin)); mergeSql(rows, countryDashboardDAO.listGameUser( condition.periodType, condition.startTime, condition.endTime, + condition.startTimeMillis(), condition.endTimeMillis(), condition.countryKeyword, condition.sysOrigin)); + } - mergeMongo(rows, condition, listGiftConsume(condition), - (row, amount) -> row.setGiftConsume(add(row.getGiftConsume(), amount))); - mergeMongo(rows, condition, listLuckyGiftAnchorShare(condition), - (row, amount) -> row.setLuckyGiftAnchorShare(add(row.getLuckyGiftAnchorShare(), amount))); - mergeDailyActive(rows, condition, listDailyActive(condition)); - mergeRetention(rows, condition); + private boolean shouldUsePrecomputedMetrics(QueryCondition condition) { + if (!hasText(condition.sysOrigin)) { + return false; + } + if (PERIOD_ALL.equals(condition.periodType)) { + if (condition.startTime != null || condition.endTime != null) { + return false; + } + long count = safeLong(countryDashboardDAO.countAllPrecomputedPeriodMetrics(condition.sysOrigin)); + if (count <= 0) { + log.warn("country dashboard all precomputed metric missing, skip live all-history scan, sysOrigin={}", + condition.sysOrigin); + } + return true; + } - List records = rows.values().stream() - .peek(this::calculateFormulaFields) - .sorted(Comparator.comparing(CountryDashboardMetricCO::getPeriodKey, - Comparator.nullsLast(Comparator.reverseOrder())) - .thenComparing(CountryDashboardMetricCO::getCountryCode, - Comparator.nullsLast(String::compareTo))) - .toList(); + List periodKeys = expectedPrecomputedPeriodKeys(condition); + if (periodKeys.isEmpty()) { + return false; + } + long actualPeriods = safeLong(countryDashboardDAO.countPrecomputedPeriods( + condition.periodType, periodKeys, condition.sysOrigin)); + boolean complete = actualPeriods >= periodKeys.size(); + if (!complete) { + log.info("country dashboard use live sql, period precomputed metric incomplete, periodType={}, sysOrigin={}, start={}, end={}, expectedPeriods={}, actualPeriods={}", + condition.periodType, condition.sysOrigin, condition.startDate, condition.endDate, + periodKeys.size(), actualPeriods); + } + return complete; + } - CountryDashboardMetricCO total = buildTotal(records, condition.periodType); + private List expectedPrecomputedPeriodKeys(QueryCondition condition) { + if (condition.startTime == null || condition.endTime == null + || !condition.startTime.isBefore(condition.endTime)) { + return List.of(); + } + LocalDate start = condition.startTime.toLocalDate(); + LocalDate endExclusive = condition.endTime.toLocalDate(); + return switch (condition.periodType) { + case "DAY" -> expectedDayKeys(start, endExclusive); + case "WEEK", "MONTH" -> expectedAlignedPeriodKeys(condition.periodType, start, endExclusive); + default -> List.of(); + }; + } - return new CountryDashboardCO() - .setPeriodType(condition.periodType) - .setStartDate(condition.startDate) - .setEndDate(condition.endDate) - .setComputedAt(LocalDateTime.now().format(DateTimeFormatter.ofPattern("yyyy-MM-dd HH:mm:ss"))) - .setTotal(total) - .setRecords(records); + private List expectedDayKeys(LocalDate start, LocalDate endExclusive) { + List keys = new ArrayList<>(); + for (LocalDate cursor = start; cursor.isBefore(endExclusive); cursor = cursor.plusDays(1)) { + keys.add(cursor.format(DATE_FORMATTER)); + } + return keys; + } + + private List expectedAlignedPeriodKeys(String periodType, LocalDate start, + LocalDate endExclusive) { + List keys = new ArrayList<>(); + LocalDate cursor = start; + LocalDate tomorrow = LocalDate.now().plusDays(1); + while (cursor.isBefore(endExclusive)) { + LocalDate periodStart = periodStart(periodType, cursor); + if (!cursor.equals(periodStart)) { + return List.of(); + } + LocalDate naturalEnd = periodEnd(periodType, periodStart); + LocalDate currentEnd = naturalEnd.isAfter(tomorrow) ? tomorrow : naturalEnd; + LocalDate windowEnd = currentEnd.isAfter(endExclusive) ? endExclusive : currentEnd; + boolean fullPeriod = windowEnd.equals(naturalEnd); + boolean currentPartialPeriod = naturalEnd.isAfter(tomorrow) && windowEnd.equals(tomorrow); + if (!fullPeriod && !currentPartialPeriod) { + return List.of(); + } + keys.add(periodKey(periodType, cursor)); + cursor = windowEnd; + } + return cursor.equals(endExclusive) ? keys : List.of(); + } + + private LocalDate periodStart(String periodType, LocalDate date) { + if ("WEEK".equals(periodType)) { + return date.with(DayOfWeek.MONDAY); + } + return date.withDayOfMonth(1); + } + + private LocalDate periodEnd(String periodType, LocalDate periodStart) { + if ("WEEK".equals(periodType)) { + return periodStart.plusDays(7); + } + return periodStart.plusMonths(1); + } + + private String periodKey(String periodType, LocalDate date) { + if ("WEEK".equals(periodType)) { + WeekFields weekFields = WeekFields.ISO; + int weekYear = date.get(weekFields.weekBasedYear()); + int week = date.get(weekFields.weekOfWeekBasedYear()); + return String.format(Locale.ROOT, "%d-W%02d", weekYear, week); + } + return date.format(MONTH_FORMATTER); } @Override @@ -716,6 +831,14 @@ public class CountryDashboardServiceImpl implements CountryDashboardService { return new PeriodBucket(day, day); } + private Long startTimeMillis() { + return startTime == null ? null : Timestamp.valueOf(startTime).getTime(); + } + + private Long endTimeMillis() { + return endTime == null ? null : Timestamp.valueOf(endTime).getTime(); + } + private static String normalizePeriodType(String periodType) { String value = trimToNull(periodType); if (value == null) { diff --git a/rc-service/rc-service-console/console-infrastructure/src/main/java/com/red/circle/console/infra/database/rds/dao/datav/CountryDashboardDAO.java b/rc-service/rc-service-console/console-infrastructure/src/main/java/com/red/circle/console/infra/database/rds/dao/datav/CountryDashboardDAO.java index 363447fd..aad0b8c3 100644 --- a/rc-service/rc-service-console/console-infrastructure/src/main/java/com/red/circle/console/infra/database/rds/dao/datav/CountryDashboardDAO.java +++ b/rc-service/rc-service-console/console-infrastructure/src/main/java/com/red/circle/console/infra/database/rds/dao/datav/CountryDashboardDAO.java @@ -1,11 +1,14 @@ package com.red.circle.console.infra.database.rds.dao.datav; +import com.red.circle.console.infra.database.rds.dto.datav.CountryDashboardDailyMetricDTO; import com.red.circle.console.infra.database.rds.dto.datav.CountryDashboardMetricDTO; +import com.red.circle.console.infra.database.rds.dto.datav.CountryDashboardPeriodMetricDTO; import com.red.circle.console.infra.database.rds.dto.datav.CountryDashboardRechargeDetailDTO; import com.red.circle.console.infra.database.rds.dto.datav.CountryDashboardRetentionUserDTO; import com.red.circle.console.infra.database.rds.dto.datav.CountryDashboardUserCountryDTO; import com.red.circle.console.infra.database.rds.entity.app.IUserBaseInfo; import com.red.circle.framework.mybatis.dao.BaseDAO; +import java.time.LocalDate; import java.time.LocalDateTime; import java.util.Collection; import java.util.List; @@ -16,6 +19,12 @@ import org.apache.ibatis.annotations.Param; */ public interface CountryDashboardDAO extends BaseDAO { + int createDailyMetricTable(); + + int createPeriodMetricTable(); + + int createPeriodUserMetricTable(); + List listNewUsers( @Param("periodType") String periodType, @Param("startTime") LocalDateTime startTime, @@ -76,6 +85,8 @@ public interface CountryDashboardDAO extends BaseDAO { @Param("periodType") String periodType, @Param("startTime") LocalDateTime startTime, @Param("endTime") LocalDateTime endTime, + @Param("startTimeMillis") Long startTimeMillis, + @Param("endTimeMillis") Long endTimeMillis, @Param("countryKeyword") String countryKeyword, @Param("sysOrigin") String sysOrigin); @@ -90,9 +101,96 @@ public interface CountryDashboardDAO extends BaseDAO { @Param("periodType") String periodType, @Param("startTime") LocalDateTime startTime, @Param("endTime") LocalDateTime endTime, + @Param("startTimeMillis") Long startTimeMillis, + @Param("endTimeMillis") Long endTimeMillis, @Param("countryKeyword") String countryKeyword, @Param("sysOrigin") String sysOrigin); + List listPrecomputedMetrics( + @Param("periodType") String periodType, + @Param("startTime") LocalDateTime startTime, + @Param("endTime") LocalDateTime endTime, + @Param("countryKeyword") String countryKeyword, + @Param("sysOrigin") String sysOrigin); + + List listPrecomputedPeriodMetrics( + @Param("periodType") String periodType, + @Param("startTime") LocalDateTime startTime, + @Param("endTime") LocalDateTime endTime, + @Param("countryKeyword") String countryKeyword, + @Param("sysOrigin") String sysOrigin); + + Long countPrecomputedMetricDates( + @Param("startTime") LocalDateTime startTime, + @Param("endTime") LocalDateTime endTime, + @Param("sysOrigin") String sysOrigin); + + Long countPrecomputedPeriods( + @Param("periodType") String periodType, + @Param("periodKeys") Collection periodKeys, + @Param("sysOrigin") String sysOrigin); + + Long countAllPrecomputedPeriodMetrics(@Param("sysOrigin") String sysOrigin); + + int deleteDailyMetrics( + @Param("statDate") LocalDate statDate, + @Param("sysOrigin") String sysOrigin); + + int upsertDailyMetrics(@Param("metrics") List metrics); + + int deletePeriodMetrics( + @Param("periodType") String periodType, + @Param("periodKey") String periodKey, + @Param("sysOrigin") String sysOrigin); + + int upsertPeriodMetrics(@Param("metrics") List metrics); + + int deletePeriodUserMetrics( + @Param("periodType") String periodType, + @Param("periodKey") String periodKey, + @Param("sysOrigin") String sysOrigin); + + int insertPeriodNewUserMetrics( + @Param("periodType") String periodType, + @Param("periodKey") String periodKey, + @Param("periodName") String periodName, + @Param("periodStartDate") LocalDate periodStartDate, + @Param("periodEndDate") LocalDate periodEndDate, + @Param("startTime") LocalDateTime startTime, + @Param("endTime") LocalDateTime endTime, + @Param("sysOrigin") String sysOrigin); + + int insertPeriodLuckyGiftUserMetrics( + @Param("periodType") String periodType, + @Param("periodKey") String periodKey, + @Param("periodName") String periodName, + @Param("periodStartDate") LocalDate periodStartDate, + @Param("periodEndDate") LocalDate periodEndDate, + @Param("startTime") LocalDateTime startTime, + @Param("endTime") LocalDateTime endTime, + @Param("sysOrigin") String sysOrigin); + + int insertPeriodGameUserMetrics( + @Param("periodType") String periodType, + @Param("periodKey") String periodKey, + @Param("periodName") String periodName, + @Param("periodStartDate") LocalDate periodStartDate, + @Param("periodEndDate") LocalDate periodEndDate, + @Param("startTime") LocalDateTime startTime, + @Param("endTime") LocalDateTime endTime, + @Param("startTimeMillis") Long startTimeMillis, + @Param("endTimeMillis") Long endTimeMillis, + @Param("sysOrigin") String sysOrigin); + + int insertAllPeriodUserMetricsFromDaily(@Param("sysOrigin") String sysOrigin); + + List listPeriodUserMetricCounts( + @Param("periodType") String periodType, + @Param("periodKey") String periodKey, + @Param("sysOrigin") String sysOrigin); + + List listAllPeriodAmountsFromDaily(@Param("sysOrigin") String sysOrigin); + List listNewUserCohorts( @Param("periodType") String periodType, @Param("startTime") LocalDateTime startTime, diff --git a/rc-service/rc-service-console/console-infrastructure/src/main/java/com/red/circle/console/infra/database/rds/dto/datav/CountryDashboardDailyMetricDTO.java b/rc-service/rc-service-console/console-infrastructure/src/main/java/com/red/circle/console/infra/database/rds/dto/datav/CountryDashboardDailyMetricDTO.java new file mode 100644 index 00000000..df124c4c --- /dev/null +++ b/rc-service/rc-service-console/console-infrastructure/src/main/java/com/red/circle/console/infra/database/rds/dto/datav/CountryDashboardDailyMetricDTO.java @@ -0,0 +1,50 @@ +package com.red.circle.console.infra.database.rds.dto.datav; + +import java.math.BigDecimal; +import java.time.LocalDate; +import lombok.Data; +import lombok.experimental.Accessors; + +/** + * 国家数据大屏日预聚合结果. + */ +@Data +@Accessors(chain = true) +public class CountryDashboardDailyMetricDTO { + + private LocalDate statDate; + + private Integer dateNumber; + + private String sysOrigin; + + private String countryCode; + + private String countryName; + + private Long countryNewUser; + + private BigDecimal newUserRecharge; + + private BigDecimal officialRecharge; + + private BigDecimal mifapayRecharge; + + private BigDecimal googleRecharge; + + private BigDecimal dealerRecharge; + + private BigDecimal salaryExchange; + + private BigDecimal luckyGiftTotalFlow; + + private Long luckyGiftUser; + + private BigDecimal luckyGiftPayout; + + private BigDecimal gameTotalFlow; + + private Long gameUser; + + private BigDecimal gamePayout; +} diff --git a/rc-service/rc-service-console/console-infrastructure/src/main/java/com/red/circle/console/infra/database/rds/dto/datav/CountryDashboardPeriodMetricDTO.java b/rc-service/rc-service-console/console-infrastructure/src/main/java/com/red/circle/console/infra/database/rds/dto/datav/CountryDashboardPeriodMetricDTO.java new file mode 100644 index 00000000..f9be711f --- /dev/null +++ b/rc-service/rc-service-console/console-infrastructure/src/main/java/com/red/circle/console/infra/database/rds/dto/datav/CountryDashboardPeriodMetricDTO.java @@ -0,0 +1,56 @@ +package com.red.circle.console.infra.database.rds.dto.datav; + +import java.math.BigDecimal; +import java.time.LocalDate; +import lombok.Data; +import lombok.experimental.Accessors; + +/** + * 国家数据大屏周期预聚合结果. + */ +@Data +@Accessors(chain = true) +public class CountryDashboardPeriodMetricDTO { + + private String periodType; + + private String periodKey; + + private String periodName; + + private LocalDate periodStartDate; + + private LocalDate periodEndDate; + + private String sysOrigin; + + private String countryCode; + + private String countryName; + + private Long countryNewUser; + + private BigDecimal newUserRecharge; + + private BigDecimal officialRecharge; + + private BigDecimal mifapayRecharge; + + private BigDecimal googleRecharge; + + private BigDecimal dealerRecharge; + + private BigDecimal salaryExchange; + + private BigDecimal luckyGiftTotalFlow; + + private Long luckyGiftUser; + + private BigDecimal luckyGiftPayout; + + private BigDecimal gameTotalFlow; + + private Long gameUser; + + private BigDecimal gamePayout; +} diff --git a/rc-service/rc-service-console/console-infrastructure/src/main/resources/dao/datav/CountryDashboardDAO.xml b/rc-service/rc-service-console/console-infrastructure/src/main/resources/dao/datav/CountryDashboardDAO.xml index b4017f39..a8438e14 100644 --- a/rc-service/rc-service-console/console-infrastructure/src/main/resources/dao/datav/CountryDashboardDAO.xml +++ b/rc-service/rc-service-console/console-infrastructure/src/main/resources/dao/datav/CountryDashboardDAO.xml @@ -3,6 +3,92 @@ "http://mybatis.org/dtd/mybatis-3-mapper.dtd"> + + CREATE TABLE IF NOT EXISTS `country_dashboard_daily_metric` ( + `id` bigint NOT NULL AUTO_INCREMENT COMMENT '主键', + `stat_date` date NOT NULL COMMENT '统计日期', + `date_number` int NOT NULL COMMENT '统计日期yyyyMMdd', + `sys_origin` varchar(32) CHARACTER SET utf8mb4 COLLATE utf8mb4_unicode_ci NOT NULL DEFAULT '' COMMENT '系统来源', + `country_code` varchar(64) CHARACTER SET utf8mb4 COLLATE utf8mb4_unicode_ci NOT NULL DEFAULT 'UNKNOWN' COMMENT '国家码', + `country_name` varchar(128) CHARACTER SET utf8mb4 COLLATE utf8mb4_unicode_ci NOT NULL DEFAULT 'UNKNOWN' COMMENT '国家名称', + `country_new_user` bigint NOT NULL DEFAULT 0 COMMENT '新增用户数', + `new_user_recharge` decimal(20,2) NOT NULL DEFAULT 0.00 COMMENT '新增用户充值', + `official_recharge` decimal(20,2) NOT NULL DEFAULT 0.00 COMMENT '官方充值', + `mifapay_recharge` decimal(20,2) NOT NULL DEFAULT 0.00 COMMENT 'MifaPay充值', + `google_recharge` decimal(20,2) NOT NULL DEFAULT 0.00 COMMENT 'Google充值', + `dealer_recharge` decimal(20,2) NOT NULL DEFAULT 0.00 COMMENT '代理充值', + `salary_exchange` decimal(20,2) NOT NULL DEFAULT 0.00 COMMENT '工资兑换', + `lucky_gift_total_flow` decimal(20,2) NOT NULL DEFAULT 0.00 COMMENT '幸运礼物流水', + `lucky_gift_user` bigint NOT NULL DEFAULT 0 COMMENT '幸运礼物用户数', + `lucky_gift_payout` decimal(20,2) NOT NULL DEFAULT 0.00 COMMENT '幸运礼物返奖', + `game_total_flow` decimal(20,2) NOT NULL DEFAULT 0.00 COMMENT '游戏流水', + `game_user` bigint NOT NULL DEFAULT 0 COMMENT '游戏用户数', + `game_payout` decimal(20,2) NOT NULL DEFAULT 0.00 COMMENT '游戏返奖', + `refreshed_at` datetime NOT NULL DEFAULT CURRENT_TIMESTAMP COMMENT '最近刷新时间', + `create_time` datetime NOT NULL DEFAULT CURRENT_TIMESTAMP COMMENT '创建时间', + `update_time` datetime NOT NULL DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP COMMENT '更新时间', + PRIMARY KEY (`id`), + UNIQUE KEY `uk_date_origin_country` (`stat_date`, `sys_origin`, `country_code`), + KEY `idx_origin_date` (`sys_origin`, `stat_date`), + KEY `idx_country_date` (`country_code`, `stat_date`) + ) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_unicode_ci COMMENT='国家数据大屏日预聚合' + + + + CREATE TABLE IF NOT EXISTS `country_dashboard_period_metric` ( + `id` bigint NOT NULL AUTO_INCREMENT COMMENT '主键', + `period_type` varchar(16) CHARACTER SET utf8mb4 COLLATE utf8mb4_unicode_ci NOT NULL COMMENT '周期类型:DAY/WEEK/MONTH/ALL', + `period_key` varchar(32) CHARACTER SET utf8mb4 COLLATE utf8mb4_unicode_ci NOT NULL COMMENT '周期键', + `period_name` varchar(64) CHARACTER SET utf8mb4 COLLATE utf8mb4_unicode_ci NOT NULL DEFAULT '' COMMENT '周期名称', + `period_start_date` date DEFAULT NULL COMMENT '周期开始日期', + `period_end_date` date DEFAULT NULL COMMENT '周期结束日期,开区间', + `sys_origin` varchar(32) CHARACTER SET utf8mb4 COLLATE utf8mb4_unicode_ci NOT NULL DEFAULT '' COMMENT '系统来源', + `country_code` varchar(64) CHARACTER SET utf8mb4 COLLATE utf8mb4_unicode_ci NOT NULL DEFAULT 'UNKNOWN' COMMENT '国家码', + `country_name` varchar(128) CHARACTER SET utf8mb4 COLLATE utf8mb4_unicode_ci NOT NULL DEFAULT 'UNKNOWN' COMMENT '国家名称', + `country_new_user` bigint NOT NULL DEFAULT 0 COMMENT '新增用户数', + `new_user_recharge` decimal(20,2) NOT NULL DEFAULT 0.00 COMMENT '新增用户充值', + `official_recharge` decimal(20,2) NOT NULL DEFAULT 0.00 COMMENT '官方充值', + `mifapay_recharge` decimal(20,2) NOT NULL DEFAULT 0.00 COMMENT 'MifaPay充值', + `google_recharge` decimal(20,2) NOT NULL DEFAULT 0.00 COMMENT 'Google充值', + `dealer_recharge` decimal(20,2) NOT NULL DEFAULT 0.00 COMMENT '代理充值', + `salary_exchange` decimal(20,2) NOT NULL DEFAULT 0.00 COMMENT '工资兑换', + `lucky_gift_total_flow` decimal(20,2) NOT NULL DEFAULT 0.00 COMMENT '幸运礼物流水', + `lucky_gift_user` bigint NOT NULL DEFAULT 0 COMMENT '幸运礼物用户数', + `lucky_gift_payout` decimal(20,2) NOT NULL DEFAULT 0.00 COMMENT '幸运礼物返奖', + `game_total_flow` decimal(20,2) NOT NULL DEFAULT 0.00 COMMENT '游戏流水', + `game_user` bigint NOT NULL DEFAULT 0 COMMENT '游戏用户数', + `game_payout` decimal(20,2) NOT NULL DEFAULT 0.00 COMMENT '游戏返奖', + `refreshed_at` datetime NOT NULL DEFAULT CURRENT_TIMESTAMP COMMENT '最近刷新时间', + `create_time` datetime NOT NULL DEFAULT CURRENT_TIMESTAMP COMMENT '创建时间', + `update_time` datetime NOT NULL DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP COMMENT '更新时间', + PRIMARY KEY (`id`), + UNIQUE KEY `uk_period_origin_country` (`period_type`, `period_key`, `sys_origin`, `country_code`), + KEY `idx_origin_period` (`sys_origin`, `period_type`, `period_key`), + KEY `idx_period_date` (`period_type`, `period_start_date`, `period_end_date`) + ) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_unicode_ci COMMENT='国家数据大屏周期预聚合' + + + + CREATE TABLE IF NOT EXISTS `country_dashboard_period_user_metric` ( + `id` bigint NOT NULL AUTO_INCREMENT COMMENT '主键', + `period_type` varchar(16) CHARACTER SET utf8mb4 COLLATE utf8mb4_unicode_ci NOT NULL COMMENT '周期类型:DAY/WEEK/MONTH/ALL', + `period_key` varchar(32) CHARACTER SET utf8mb4 COLLATE utf8mb4_unicode_ci NOT NULL COMMENT '周期键', + `period_name` varchar(64) CHARACTER SET utf8mb4 COLLATE utf8mb4_unicode_ci NOT NULL DEFAULT '' COMMENT '周期名称', + `period_start_date` date DEFAULT NULL COMMENT '周期开始日期', + `period_end_date` date DEFAULT NULL COMMENT '周期结束日期,开区间', + `sys_origin` varchar(32) CHARACTER SET utf8mb4 COLLATE utf8mb4_unicode_ci NOT NULL DEFAULT '' COMMENT '系统来源', + `country_code` varchar(64) CHARACTER SET utf8mb4 COLLATE utf8mb4_unicode_ci NOT NULL DEFAULT 'UNKNOWN' COMMENT '国家码', + `country_name` varchar(128) CHARACTER SET utf8mb4 COLLATE utf8mb4_unicode_ci NOT NULL DEFAULT 'UNKNOWN' COMMENT '国家名称', + `user_id` bigint NOT NULL COMMENT '用户ID', + `metric_type` varchar(32) CHARACTER SET utf8mb4 COLLATE utf8mb4_unicode_ci NOT NULL COMMENT '人数指标类型', + `create_time` datetime NOT NULL DEFAULT CURRENT_TIMESTAMP COMMENT '创建时间', + PRIMARY KEY (`id`), + UNIQUE KEY `uk_period_user_metric` (`period_type`, `period_key`, `sys_origin`, `country_code`, `user_id`, `metric_type`), + KEY `idx_period_metric_country` (`period_type`, `period_key`, `sys_origin`, `metric_type`, `country_code`), + KEY `idx_origin_user_metric` (`sys_origin`, `user_id`, `metric_type`) + ) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_unicode_ci COMMENT='国家数据大屏周期用户去重明细' + + NULLIF(u.country_code, '') AS countryCode, NULLIF(u.country_name, '') AS countryName @@ -79,6 +165,23 @@ + + + DATE_FORMAT(t.stat_date, '%Y-%m-%d') + DATE_FORMAT(t.stat_date, '%x-W%v') + DATE_FORMAT(t.stat_date, '%Y-%m') + 'ALL' + + AS periodKey, + + DATE_FORMAT(t.stat_date, '%Y-%m-%d') + DATE_FORMAT(t.stat_date, '%x-W%v') + DATE_FORMAT(t.stat_date, '%Y-%m') + '全部' + + AS periodName + + AND FIND_IN_SET(u.country_code, #{countryCodes}) @@ -138,42 +241,68 @@ + + + + create_time >= #{startTimeMillis} + + + AND create_time < #{endTimeMillis} + + + AND sys_origin = #{sysOrigin} + + + + SELECT user_id, sys_origin, type, penny_amount, event_type, FROM_UNIXTIME(create_time / 1000) AS create_time FROM likei_wallet.wallet_gold_asset_record_1 + UNION ALL SELECT user_id, sys_origin, type, penny_amount, event_type, FROM_UNIXTIME(create_time / 1000) AS create_time FROM likei_wallet.wallet_gold_asset_record_2 + UNION ALL SELECT user_id, sys_origin, type, penny_amount, event_type, FROM_UNIXTIME(create_time / 1000) AS create_time FROM likei_wallet.wallet_gold_asset_record_3 + UNION ALL SELECT user_id, sys_origin, type, penny_amount, event_type, FROM_UNIXTIME(create_time / 1000) AS create_time FROM likei_wallet.wallet_gold_asset_record_4 + UNION ALL SELECT user_id, sys_origin, type, penny_amount, event_type, FROM_UNIXTIME(create_time / 1000) AS create_time FROM likei_wallet.wallet_gold_asset_record_5 + UNION ALL SELECT user_id, sys_origin, type, penny_amount, event_type, FROM_UNIXTIME(create_time / 1000) AS create_time FROM likei_wallet.wallet_gold_asset_record_6 + UNION ALL SELECT user_id, sys_origin, type, penny_amount, event_type, FROM_UNIXTIME(create_time / 1000) AS create_time FROM likei_wallet.wallet_gold_asset_record_7 + UNION ALL SELECT user_id, sys_origin, type, penny_amount, event_type, FROM_UNIXTIME(create_time / 1000) AS create_time FROM likei_wallet.wallet_gold_asset_record_8 + UNION ALL SELECT user_id, sys_origin, type, penny_amount, event_type, FROM_UNIXTIME(create_time / 1000) AS create_time FROM likei_wallet.wallet_gold_asset_record_9 + UNION ALL SELECT user_id, sys_origin, type, penny_amount, event_type, FROM_UNIXTIME(create_time / 1000) AS create_time FROM likei_wallet.wallet_gold_asset_record_10 + UNION ALL SELECT user_id, sys_origin, type, penny_amount, event_type, FROM_UNIXTIME(create_time / 1000) AS create_time FROM likei_wallet.wallet_gold_asset_record_11 + UNION ALL SELECT user_id, sys_origin, type, penny_amount, event_type, FROM_UNIXTIME(create_time / 1000) AS create_time FROM likei_wallet.wallet_gold_asset_record_12 + @@ -213,6 +342,476 @@ AND t.event_type NOT LIKE 'ACCEPT_GIFT%' + + + + + + + + + + + + DELETE FROM country_dashboard_daily_metric + WHERE stat_date = #{statDate} + + AND sys_origin = #{sysOrigin} + + + + + INSERT INTO country_dashboard_daily_metric ( + stat_date, + date_number, + sys_origin, + country_code, + country_name, + country_new_user, + new_user_recharge, + official_recharge, + mifapay_recharge, + google_recharge, + dealer_recharge, + salary_exchange, + lucky_gift_total_flow, + lucky_gift_user, + lucky_gift_payout, + game_total_flow, + game_user, + game_payout, + refreshed_at + ) + VALUES + + ( + #{item.statDate}, + #{item.dateNumber}, + #{item.sysOrigin}, + #{item.countryCode}, + #{item.countryName}, + IFNULL(#{item.countryNewUser}, 0), + IFNULL(#{item.newUserRecharge}, 0), + IFNULL(#{item.officialRecharge}, 0), + IFNULL(#{item.mifapayRecharge}, 0), + IFNULL(#{item.googleRecharge}, 0), + IFNULL(#{item.dealerRecharge}, 0), + IFNULL(#{item.salaryExchange}, 0), + IFNULL(#{item.luckyGiftTotalFlow}, 0), + IFNULL(#{item.luckyGiftUser}, 0), + IFNULL(#{item.luckyGiftPayout}, 0), + IFNULL(#{item.gameTotalFlow}, 0), + IFNULL(#{item.gameUser}, 0), + IFNULL(#{item.gamePayout}, 0), + NOW() + ) + + ON DUPLICATE KEY UPDATE + country_name = VALUES(country_name), + country_new_user = VALUES(country_new_user), + new_user_recharge = VALUES(new_user_recharge), + official_recharge = VALUES(official_recharge), + mifapay_recharge = VALUES(mifapay_recharge), + google_recharge = VALUES(google_recharge), + dealer_recharge = VALUES(dealer_recharge), + salary_exchange = VALUES(salary_exchange), + lucky_gift_total_flow = VALUES(lucky_gift_total_flow), + lucky_gift_user = VALUES(lucky_gift_user), + lucky_gift_payout = VALUES(lucky_gift_payout), + game_total_flow = VALUES(game_total_flow), + game_user = VALUES(game_user), + game_payout = VALUES(game_payout), + refreshed_at = NOW() + + + + DELETE FROM country_dashboard_period_metric + WHERE period_type = #{periodType} + AND period_key = #{periodKey} + + AND sys_origin = #{sysOrigin} + + + + + INSERT INTO country_dashboard_period_metric ( + period_type, + period_key, + period_name, + period_start_date, + period_end_date, + sys_origin, + country_code, + country_name, + country_new_user, + new_user_recharge, + official_recharge, + mifapay_recharge, + google_recharge, + dealer_recharge, + salary_exchange, + lucky_gift_total_flow, + lucky_gift_user, + lucky_gift_payout, + game_total_flow, + game_user, + game_payout, + refreshed_at + ) + VALUES + + ( + #{item.periodType}, + #{item.periodKey}, + #{item.periodName}, + #{item.periodStartDate}, + #{item.periodEndDate}, + #{item.sysOrigin}, + #{item.countryCode}, + #{item.countryName}, + IFNULL(#{item.countryNewUser}, 0), + IFNULL(#{item.newUserRecharge}, 0), + IFNULL(#{item.officialRecharge}, 0), + IFNULL(#{item.mifapayRecharge}, 0), + IFNULL(#{item.googleRecharge}, 0), + IFNULL(#{item.dealerRecharge}, 0), + IFNULL(#{item.salaryExchange}, 0), + IFNULL(#{item.luckyGiftTotalFlow}, 0), + IFNULL(#{item.luckyGiftUser}, 0), + IFNULL(#{item.luckyGiftPayout}, 0), + IFNULL(#{item.gameTotalFlow}, 0), + IFNULL(#{item.gameUser}, 0), + IFNULL(#{item.gamePayout}, 0), + NOW() + ) + + ON DUPLICATE KEY UPDATE + period_name = VALUES(period_name), + period_start_date = VALUES(period_start_date), + period_end_date = VALUES(period_end_date), + country_name = VALUES(country_name), + country_new_user = VALUES(country_new_user), + new_user_recharge = VALUES(new_user_recharge), + official_recharge = VALUES(official_recharge), + mifapay_recharge = VALUES(mifapay_recharge), + google_recharge = VALUES(google_recharge), + dealer_recharge = VALUES(dealer_recharge), + salary_exchange = VALUES(salary_exchange), + lucky_gift_total_flow = VALUES(lucky_gift_total_flow), + lucky_gift_user = VALUES(lucky_gift_user), + lucky_gift_payout = VALUES(lucky_gift_payout), + game_total_flow = VALUES(game_total_flow), + game_user = VALUES(game_user), + game_payout = VALUES(game_payout), + refreshed_at = NOW() + + + + DELETE FROM country_dashboard_period_user_metric + WHERE period_type = #{periodType} + AND period_key = #{periodKey} + + AND sys_origin = #{sysOrigin} + + + + + INSERT IGNORE INTO country_dashboard_period_user_metric ( + period_type, + period_key, + period_name, + period_start_date, + period_end_date, + sys_origin, + country_code, + country_name, + user_id, + metric_type + ) + SELECT DISTINCT + #{periodType}, + #{periodKey}, + #{periodName}, + #{periodStartDate}, + #{periodEndDate}, + #{sysOrigin}, + IFNULL(NULLIF(u.country_code, ''), 'UNKNOWN') AS countryCode, + IFNULL(NULLIF(u.country_name, ''), IFNULL(NULLIF(u.country_code, ''), 'UNKNOWN')) AS countryName, + u.id AS userId, + 'COUNTRY_NEW_USER' + FROM user_base_info u + WHERE 1 = 1 + AND (u.is_del = 0 OR u.is_del IS NULL) + + AND u.origin_sys = #{sysOrigin} + + + + + + INSERT IGNORE INTO country_dashboard_period_user_metric ( + period_type, + period_key, + period_name, + period_start_date, + period_end_date, + sys_origin, + country_code, + country_name, + user_id, + metric_type + ) + SELECT DISTINCT + #{periodType}, + #{periodKey}, + #{periodName}, + #{periodStartDate}, + #{periodEndDate}, + #{sysOrigin}, + IFNULL(NULLIF(u.country_code, ''), 'UNKNOWN') AS countryCode, + IFNULL(NULLIF(u.country_name, ''), IFNULL(NULLIF(u.country_code, ''), 'UNKNOWN')) AS countryName, + t.user_id AS userId, + 'LUCKY_GIFT_USER' + FROM game_lucky_gift_count t + INNER JOIN user_base_info u ON u.id = t.user_id + WHERE 1 = 1 + AND (u.is_del = 0 OR u.is_del IS NULL) + + AND u.origin_sys = #{sysOrigin} + + + + + + + INSERT IGNORE INTO country_dashboard_period_user_metric ( + period_type, + period_key, + period_name, + period_start_date, + period_end_date, + sys_origin, + country_code, + country_name, + user_id, + metric_type + ) + SELECT DISTINCT + #{periodType}, + #{periodKey}, + #{periodName}, + #{periodStartDate}, + #{periodEndDate}, + #{sysOrigin}, + IFNULL(NULLIF(u.country_code, ''), 'UNKNOWN') AS countryCode, + IFNULL(NULLIF(u.country_name, ''), IFNULL(NULLIF(u.country_code, ''), 'UNKNOWN')) AS countryName, + s.user_id AS userId, + 'GAME_USER' + FROM ( + SELECT t.user_id + FROM ( + + ) t + WHERE + UNION + SELECT t.user_id + FROM game_lucky_box_count t + WHERE 1 = 1 + + + ) s + INNER JOIN user_base_info u ON u.id = s.user_id + WHERE 1 = 1 + AND (u.is_del = 0 OR u.is_del IS NULL) + + AND u.origin_sys = #{sysOrigin} + + + + + INSERT IGNORE INTO country_dashboard_period_user_metric ( + period_type, + period_key, + period_name, + period_start_date, + period_end_date, + sys_origin, + country_code, + country_name, + user_id, + metric_type + ) + SELECT DISTINCT + 'ALL', + 'ALL', + '全部', + NULL, + NULL, + t.sys_origin, + t.country_code, + t.country_name, + t.user_id, + t.metric_type + FROM country_dashboard_period_user_metric t + WHERE t.period_type = 'DAY' + + AND t.sys_origin = #{sysOrigin} + + + + + + +