497 lines
17 KiB
Go
497 lines
17 KiB
Go
package main
|
||
|
||
import (
|
||
"context"
|
||
"crypto/sha256"
|
||
"database/sql"
|
||
"encoding/hex"
|
||
"flag"
|
||
"fmt"
|
||
"log"
|
||
"os"
|
||
"strings"
|
||
"time"
|
||
|
||
"google.golang.org/protobuf/proto"
|
||
roomeventsv1 "hyapp.local/api/proto/events/room/v1"
|
||
|
||
"hyapp/pkg/appcode"
|
||
"hyapp/pkg/gamemq"
|
||
"hyapp/pkg/roommq"
|
||
"hyapp/pkg/usermq"
|
||
"hyapp/pkg/walletmq"
|
||
statapp "hyapp/services/statistics-service/internal/app"
|
||
mysqlstorage "hyapp/services/statistics-service/internal/storage/mysql"
|
||
|
||
_ "github.com/go-sql-driver/mysql"
|
||
)
|
||
|
||
const replayEventIDMaxLen = 160
|
||
|
||
type replayWindow struct {
|
||
startDay string
|
||
endDay string
|
||
startMS int64
|
||
endMS int64
|
||
}
|
||
|
||
type replaySummary struct {
|
||
userRows int64
|
||
walletRows int64
|
||
roomRows int64
|
||
gameRows int64
|
||
}
|
||
|
||
func main() {
|
||
var (
|
||
appCode = flag.String("app", envDefault("APP_CODE", "lalu"), "app_code to replay")
|
||
statTZ = flag.String("timezone", envDefault("STAT_TZ", mysqlstorage.StatTZAsiaShanghai), "stat_tz to rebuild: UTC or Asia/Shanghai")
|
||
startDay = flag.String("start-day", "", "inclusive stat day in selected timezone, yyyy-mm-dd")
|
||
endDay = flag.String("end-day", "", "inclusive stat day in selected timezone, yyyy-mm-dd")
|
||
apply = flag.Bool("apply", false, "apply mutations; dry-run only counts source rows by default")
|
||
statisticsDSN = flag.String("statistics-dsn", firstNonEmptyEnv("STATISTICS_MYSQL_DSN", "MYSQL_DSN"), "statistics mysql dsn")
|
||
userDSN = flag.String("user-dsn", firstNonEmptyEnv("USER_MYSQL_DSN", "MYSQL_USER_DSN"), "user-service mysql dsn")
|
||
walletDSN = flag.String("wallet-dsn", firstNonEmptyEnv("WALLET_MYSQL_DSN", "MYSQL_WALLET_DSN"), "wallet-service mysql dsn")
|
||
roomDSN = flag.String("room-dsn", firstNonEmptyEnv("ROOM_MYSQL_DSN", "MYSQL_ROOM_DSN"), "room-service mysql dsn")
|
||
gameDSN = flag.String("game-dsn", firstNonEmptyEnv("GAME_MYSQL_DSN", "MYSQL_GAME_DSN"), "game-service mysql dsn")
|
||
)
|
||
flag.Parse()
|
||
|
||
statTZValue, loc, err := normalizeReplayStatTZ(*statTZ)
|
||
if err != nil {
|
||
log.Fatal(err)
|
||
}
|
||
window, err := resolveReplayWindow(*startDay, *endDay, loc)
|
||
if err != nil {
|
||
log.Fatal(err)
|
||
}
|
||
if *apply && strings.TrimSpace(*statisticsDSN) == "" {
|
||
log.Fatal("statistics-dsn is required when apply is true")
|
||
}
|
||
if *apply && (strings.TrimSpace(*userDSN) == "" || strings.TrimSpace(*walletDSN) == "" || strings.TrimSpace(*roomDSN) == "" || strings.TrimSpace(*gameDSN) == "") {
|
||
log.Fatal("apply requires user-dsn, wallet-dsn, room-dsn and game-dsn so the rebuilt stat_tz window is complete for owner outbox facts")
|
||
}
|
||
|
||
ctx := context.Background()
|
||
dbs := map[string]*sql.DB{}
|
||
for name, dsn := range map[string]string{"user": *userDSN, "wallet": *walletDSN, "room": *roomDSN, "game": *gameDSN} {
|
||
if strings.TrimSpace(dsn) == "" {
|
||
continue
|
||
}
|
||
db, err := openDB(ctx, dsn)
|
||
if err != nil {
|
||
log.Fatalf("open %s db: %v", name, err)
|
||
}
|
||
defer db.Close()
|
||
dbs[name] = db
|
||
}
|
||
|
||
summary, err := countSources(ctx, dbs, *appCode, window)
|
||
if err != nil {
|
||
log.Fatalf("count sources: %v", err)
|
||
}
|
||
if !*apply {
|
||
fmt.Printf("dry-run app=%s stat_tz=%s days=%s..%s source_rows user=%d wallet=%d room=%d game=%d\n",
|
||
*appCode, statTZValue, window.startDay, window.endDay, summary.userRows, summary.walletRows, summary.roomRows, summary.gameRows)
|
||
return
|
||
}
|
||
statsDB, err := openDB(ctx, *statisticsDSN)
|
||
if err != nil {
|
||
log.Fatalf("open statistics db: %v", err)
|
||
}
|
||
defer statsDB.Close()
|
||
repo, err := mysqlstorage.Open(ctx, *statisticsDSN)
|
||
if err != nil {
|
||
log.Fatalf("open statistics repository: %v", err)
|
||
}
|
||
defer repo.Close()
|
||
if err := resetReplayTarget(ctx, statsDB, *appCode, statTZValue, window); err != nil {
|
||
log.Fatalf("reset target: %v", err)
|
||
}
|
||
dimensionRows, err := primeUserDimensions(ctx, statsDB, dbs["user"], *appCode)
|
||
if err != nil {
|
||
log.Fatalf("prime user dimensions: %v", err)
|
||
}
|
||
scopeCtx := mysqlstorage.WithStatTZScope(ctx, statTZValue)
|
||
if err := replayUser(ctx, scopeCtx, repo, dbs["user"], *appCode, statTZValue, window); err != nil {
|
||
log.Fatalf("replay user outbox: %v", err)
|
||
}
|
||
if err := replayWallet(ctx, scopeCtx, repo, dbs["wallet"], *appCode, statTZValue, window); err != nil {
|
||
log.Fatalf("replay wallet outbox: %v", err)
|
||
}
|
||
if err := replayRoom(ctx, scopeCtx, repo, dbs["room"], *appCode, statTZValue, window); err != nil {
|
||
log.Fatalf("replay room outbox: %v", err)
|
||
}
|
||
if err := replayGame(ctx, scopeCtx, repo, dbs["game"], *appCode, statTZValue, window); err != nil {
|
||
log.Fatalf("replay game outbox: %v", err)
|
||
}
|
||
fmt.Printf("applied app=%s stat_tz=%s days=%s..%s source_rows user=%d wallet=%d room=%d game=%d user_dimension_rows=%d\n",
|
||
*appCode, statTZValue, window.startDay, window.endDay, summary.userRows, summary.walletRows, summary.roomRows, summary.gameRows, dimensionRows)
|
||
}
|
||
|
||
func normalizeReplayStatTZ(value string) (string, *time.Location, error) {
|
||
switch strings.TrimSpace(value) {
|
||
case mysqlstorage.StatTZUTC:
|
||
return mysqlstorage.StatTZUTC, time.UTC, nil
|
||
case mysqlstorage.StatTZAsiaShanghai:
|
||
return mysqlstorage.StatTZAsiaShanghai, time.FixedZone(mysqlstorage.StatTZAsiaShanghai, 8*60*60), nil
|
||
default:
|
||
return "", nil, fmt.Errorf("unsupported stat_tz %q; only UTC and Asia/Shanghai are supported", value)
|
||
}
|
||
}
|
||
|
||
func resolveReplayWindow(startDay string, endDay string, loc *time.Location) (replayWindow, error) {
|
||
now := time.Now().In(loc)
|
||
if strings.TrimSpace(startDay) == "" || strings.TrimSpace(endDay) == "" {
|
||
// 默认重跑“最近 30 个已完成自然日”，避开当天实时消费者仍在写入造成 replay 后缀和原 event_id 同时入账。
|
||
yesterday := time.Date(now.Year(), now.Month(), now.Day(), 0, 0, 0, 0, loc).AddDate(0, 0, -1)
|
||
start := yesterday.AddDate(0, 0, -29)
|
||
startDay = start.Format("2006-01-02")
|
||
endDay = yesterday.Format("2006-01-02")
|
||
}
|
||
start, err := time.ParseInLocation("2006-01-02", startDay, loc)
|
||
if err != nil {
|
||
return replayWindow{}, fmt.Errorf("parse start-day: %w", err)
|
||
}
|
||
end, err := time.ParseInLocation("2006-01-02", endDay, loc)
|
||
if err != nil {
|
||
return replayWindow{}, fmt.Errorf("parse end-day: %w", err)
|
||
}
|
||
if end.Before(start) {
|
||
return replayWindow{}, fmt.Errorf("end-day %s is before start-day %s", endDay, startDay)
|
||
}
|
||
return replayWindow{
|
||
startDay: start.Format("2006-01-02"),
|
||
endDay: end.Format("2006-01-02"),
|
||
startMS: start.UnixMilli(),
|
||
endMS: end.AddDate(0, 0, 1).UnixMilli(),
|
||
}, nil
|
||
}
|
||
|
||
func openDB(ctx context.Context, dsn string) (*sql.DB, error) {
|
||
db, err := sql.Open("mysql", strings.TrimSpace(dsn))
|
||
if err != nil {
|
||
return nil, err
|
||
}
|
||
if err := db.PingContext(ctx); err != nil {
|
||
_ = db.Close()
|
||
return nil, err
|
||
}
|
||
return db, nil
|
||
}
|
||
|
||
func countSources(ctx context.Context, dbs map[string]*sql.DB, app string, window replayWindow) (replaySummary, error) {
|
||
var out replaySummary
|
||
var err error
|
||
if db := dbs["user"]; db != nil {
|
||
out.userRows, err = countOutboxRows(ctx, db, "user_outbox", app, userReplayEventTypes(), window)
|
||
if err != nil {
|
||
return replaySummary{}, err
|
||
}
|
||
}
|
||
if db := dbs["wallet"]; db != nil {
|
||
out.walletRows, err = countOutboxRows(ctx, db, "wallet_outbox", app, walletReplayEventTypes(), window)
|
||
if err != nil {
|
||
return replaySummary{}, err
|
||
}
|
||
}
|
||
if db := dbs["room"]; db != nil {
|
||
out.roomRows, err = countOutboxRows(ctx, db, "room_outbox", app, []string{"RoomGiftSent", "RoomUserJoined"}, window)
|
||
if err != nil {
|
||
return replaySummary{}, err
|
||
}
|
||
}
|
||
if db := dbs["game"]; db != nil {
|
||
out.gameRows, err = countOutboxRows(ctx, db, "game_outbox", app, gameReplayEventTypes(), window)
|
||
if err != nil {
|
||
return replaySummary{}, err
|
||
}
|
||
}
|
||
return out, nil
|
||
}
|
||
|
||
func countOutboxRows(ctx context.Context, db *sql.DB, table string, app string, eventTypes []string, window replayWindow) (int64, error) {
|
||
query, args := outboxWindowQuery("COUNT(*)", table, app, eventTypes, window, "")
|
||
row := db.QueryRowContext(ctx, query, args...)
|
||
var count int64
|
||
return count, row.Scan(&count)
|
||
}
|
||
|
||
func resetReplayTarget(ctx context.Context, db *sql.DB, app string, statTZ string, window replayWindow) error {
|
||
tx, err := db.BeginTx(ctx, nil)
|
||
if err != nil {
|
||
return err
|
||
}
|
||
defer tx.Rollback()
|
||
// 目标表只删除当前 stat_tz 的可重建 owner-outbox 聚合；H5 上报没有 owner outbox，不在这里删除，避免抹掉部署后实时进入的 H5 事件。
|
||
for _, spec := range []struct {
|
||
table string
|
||
day string
|
||
}{
|
||
{"stat_app_day_country", "stat_day"},
|
||
{"stat_user_day_activity", "stat_day"},
|
||
{"stat_user_registration", "registered_day"},
|
||
{"stat_user_day_mic", "stat_day"},
|
||
{"stat_recharge_day_payers", "stat_day"},
|
||
{"stat_lucky_gift_day_payers", "stat_day"},
|
||
{"stat_lucky_gift_pool_day_country", "stat_day"},
|
||
{"stat_real_room_robot_gift_day_rooms", "stat_day"},
|
||
{"stat_game_day_country", "stat_day"},
|
||
{"stat_game_day_players", "stat_day"},
|
||
{"stat_self_game_match_day", "stat_day"},
|
||
{"stat_self_game_pool_day", "stat_day"},
|
||
{"stat_self_game_participant_day", "stat_day"},
|
||
{"stat_self_game_user_day", "stat_day"},
|
||
} {
|
||
if _, err := tx.ExecContext(ctx, "DELETE FROM "+spec.table+" WHERE app_code = ? AND stat_tz = ? AND "+spec.day+" BETWEEN ? AND ?", app, statTZ, window.startDay, window.endDay); err != nil {
|
||
return err
|
||
}
|
||
}
|
||
token := replayToken(statTZ)
|
||
for _, source := range []string{mysqlstorage.SourceUser, mysqlstorage.SourceWallet, mysqlstorage.SourceRoom, mysqlstorage.SourceGame} {
|
||
if _, err := tx.ExecContext(ctx, `
|
||
DELETE FROM statistics_event_consumption
|
||
WHERE app_code = ? AND source = ? AND (event_id LIKE ? OR event_id LIKE ?)`,
|
||
app, source, "%:stat_tz:"+token, "replay:"+token+":%"); err != nil {
|
||
return err
|
||
}
|
||
}
|
||
return tx.Commit()
|
||
}
|
||
|
||
func replayUser(ctx context.Context, scopeCtx context.Context, repo *mysqlstorage.Repository, db *sql.DB, app string, statTZ string, window replayWindow) error {
|
||
query, args := outboxWindowQuery(`
|
||
app_code, event_id, event_type, aggregate_type, aggregate_id, CAST(payload_json AS CHAR), created_at_ms`, "user_outbox", app,
|
||
userReplayEventTypes(), window, "ORDER BY created_at_ms ASC, event_id ASC")
|
||
rows, err := db.QueryContext(ctx, query, args...)
|
||
if err != nil {
|
||
return err
|
||
}
|
||
defer rows.Close()
|
||
for rows.Next() {
|
||
var msg usermq.UserOutboxMessage
|
||
if err := rows.Scan(&msg.AppCode, &msg.EventID, &msg.EventType, &msg.AggregateType, &msg.AggregateID, &msg.PayloadJSON, &msg.OccurredAtMS); err != nil {
|
||
return err
|
||
}
|
||
msg.EventID = replayEventID(msg.EventID, statTZ)
|
||
body, err := usermq.EncodeUserOutboxMessage(msg)
|
||
if err != nil {
|
||
return err
|
||
}
|
||
if err := statapp.ConsumeUserOutboxBody(scopeCtx, repo, body); err != nil {
|
||
return err
|
||
}
|
||
}
|
||
return rows.Err()
|
||
}
|
||
|
||
func primeUserDimensions(ctx context.Context, statsDB *sql.DB, userDB *sql.DB, app string) (int64, error) {
|
||
rows, err := userDB.QueryContext(ctx, `
|
||
SELECT u.user_id, COALESCE(c.country_id, 0), COALESCE(u.region_id, 0)
|
||
FROM users u
|
||
LEFT JOIN countries c ON c.app_code = u.app_code AND c.country_code = u.country
|
||
WHERE u.app_code = ?
|
||
AND u.user_id > 0
|
||
AND (COALESCE(c.country_id, 0) > 0 OR COALESCE(u.region_id, 0) > 0)`,
|
||
appcode.Normalize(app))
|
||
if err != nil {
|
||
return 0, err
|
||
}
|
||
defer rows.Close()
|
||
|
||
tx, err := statsDB.BeginTx(ctx, nil)
|
||
if err != nil {
|
||
return 0, err
|
||
}
|
||
defer tx.Rollback()
|
||
|
||
stmt, err := tx.PrepareContext(ctx, `
|
||
INSERT INTO stat_user_dimension (app_code, user_id, country_id, region_id, updated_at_ms)
|
||
VALUES (?, ?, ?, ?, ?)
|
||
ON DUPLICATE KEY UPDATE
|
||
country_id = CASE WHEN VALUES(country_id) > 0 THEN VALUES(country_id) ELSE country_id END,
|
||
region_id = CASE WHEN VALUES(region_id) > 0 THEN VALUES(region_id) ELSE region_id END,
|
||
updated_at_ms = VALUES(updated_at_ms)`)
|
||
if err != nil {
|
||
return 0, err
|
||
}
|
||
defer stmt.Close()
|
||
|
||
nowMS := time.Now().UnixMilli()
|
||
var count int64
|
||
for rows.Next() {
|
||
var userID int64
|
||
var countryID int64
|
||
var regionID int64
|
||
if err := rows.Scan(&userID, &countryID, ®ionID); err != nil {
|
||
return 0, err
|
||
}
|
||
if _, err := stmt.ExecContext(ctx, appcode.Normalize(app), userID, countryID, regionID, nowMS); err != nil {
|
||
return 0, err
|
||
}
|
||
count++
|
||
}
|
||
if err := rows.Err(); err != nil {
|
||
return 0, err
|
||
}
|
||
if err := tx.Commit(); err != nil {
|
||
return 0, err
|
||
}
|
||
return count, nil
|
||
}
|
||
|
||
func replayWallet(ctx context.Context, scopeCtx context.Context, repo *mysqlstorage.Repository, db *sql.DB, app string, statTZ string, window replayWindow) error {
|
||
query, args := outboxWindowQuery(`
|
||
app_code, event_id, event_type, transaction_id, command_id, user_id, asset_type,
|
||
available_delta, frozen_delta, CAST(payload AS CHAR), created_at_ms`, "wallet_outbox", app,
|
||
walletReplayEventTypes(), window, "ORDER BY created_at_ms ASC, event_id ASC")
|
||
rows, err := db.QueryContext(ctx, query, args...)
|
||
if err != nil {
|
||
return err
|
||
}
|
||
defer rows.Close()
|
||
for rows.Next() {
|
||
var msg walletmq.WalletOutboxMessage
|
||
if err := rows.Scan(&msg.AppCode, &msg.EventID, &msg.EventType, &msg.TransactionID, &msg.CommandID, &msg.UserID, &msg.AssetType, &msg.AvailableDelta, &msg.FrozenDelta, &msg.PayloadJSON, &msg.OccurredAtMS); err != nil {
|
||
return err
|
||
}
|
||
msg.EventID = replayEventID(msg.EventID, statTZ)
|
||
body, err := walletmq.EncodeWalletOutboxMessage(msg)
|
||
if err != nil {
|
||
return err
|
||
}
|
||
if err := statapp.ConsumeWalletOutboxBody(scopeCtx, repo, body); err != nil {
|
||
return err
|
||
}
|
||
}
|
||
return rows.Err()
|
||
}
|
||
|
||
func replayRoom(ctx context.Context, scopeCtx context.Context, repo *mysqlstorage.Repository, db *sql.DB, app string, statTZ string, window replayWindow) error {
|
||
query, args := outboxWindowQuery("envelope", "room_outbox", app, []string{"RoomGiftSent", "RoomUserJoined"}, window, "ORDER BY created_at_ms ASC, event_id ASC")
|
||
rows, err := db.QueryContext(ctx, query, args...)
|
||
if err != nil {
|
||
return err
|
||
}
|
||
defer rows.Close()
|
||
for rows.Next() {
|
||
var raw []byte
|
||
if err := rows.Scan(&raw); err != nil {
|
||
return err
|
||
}
|
||
var envelope roomeventsv1.EventEnvelope
|
||
if err := proto.Unmarshal(raw, &envelope); err != nil {
|
||
return err
|
||
}
|
||
envelope.EventId = replayEventID(envelope.GetEventId(), statTZ)
|
||
body, err := roommq.EncodeRoomOutboxMessage(&envelope)
|
||
if err != nil {
|
||
return err
|
||
}
|
||
if err := statapp.ConsumeRoomOutboxBody(scopeCtx, repo, body); err != nil {
|
||
return err
|
||
}
|
||
}
|
||
return rows.Err()
|
||
}
|
||
|
||
func replayGame(ctx context.Context, scopeCtx context.Context, repo *mysqlstorage.Repository, db *sql.DB, app string, statTZ string, window replayWindow) error {
|
||
query, args := outboxWindowQuery(`
|
||
app_code, event_id, event_type, order_id, user_id, platform_code, game_id,
|
||
op_type, coin_amount, CAST(payload_json AS CHAR), created_at_ms`, "game_outbox", app, gameReplayEventTypes(), window, "ORDER BY created_at_ms ASC, event_id ASC")
|
||
rows, err := db.QueryContext(ctx, query, args...)
|
||
if err != nil {
|
||
return err
|
||
}
|
||
defer rows.Close()
|
||
for rows.Next() {
|
||
var msg gamemq.GameOutboxMessage
|
||
if err := rows.Scan(&msg.AppCode, &msg.EventID, &msg.EventType, &msg.OrderID, &msg.UserID, &msg.PlatformCode, &msg.GameID, &msg.OpType, &msg.CoinAmount, &msg.PayloadJSON, &msg.OccurredAtMS); err != nil {
|
||
return err
|
||
}
|
||
msg.EventID = replayEventID(msg.EventID, statTZ)
|
||
body, err := gamemq.EncodeGameOutboxMessage(msg)
|
||
if err != nil {
|
||
return err
|
||
}
|
||
if err := statapp.ConsumeGameOutboxBody(scopeCtx, repo, body); err != nil {
|
||
return err
|
||
}
|
||
}
|
||
return rows.Err()
|
||
}
|
||
|
||
func outboxWindowQuery(selectExpr string, table string, app string, eventTypes []string, window replayWindow, suffix string) (string, []any) {
|
||
placeholders := strings.TrimRight(strings.Repeat("?,", len(eventTypes)), ",")
|
||
args := []any{app}
|
||
for _, eventType := range eventTypes {
|
||
args = append(args, eventType)
|
||
}
|
||
args = append(args, window.startMS, window.endMS)
|
||
return fmt.Sprintf("SELECT %s FROM %s WHERE app_code = ? AND event_type IN (%s) AND created_at_ms >= ? AND created_at_ms < ? %s", selectExpr, table, placeholders, suffix), args
|
||
}
|
||
|
||
func userReplayEventTypes() []string {
|
||
return []string{
|
||
"UserRegistered",
|
||
"UserRegionChanged",
|
||
"UserMicDailyStatsUpdated",
|
||
}
|
||
}
|
||
|
||
func walletReplayEventTypes() []string {
|
||
return []string{
|
||
"WalletRechargeRecorded",
|
||
"WalletCoinSellerStockPurchased",
|
||
"WalletBalanceChanged",
|
||
"WalletTaskRewardCredited",
|
||
"WalletLuckyGiftRewardCredited",
|
||
"WalletWheelRewardCredited",
|
||
"WalletRoomTurnoverRewardCredited",
|
||
"WalletInviteActivityRewardCredited",
|
||
"WalletAgencyOpeningRewardCredited",
|
||
}
|
||
}
|
||
|
||
func gameReplayEventTypes() []string {
|
||
return []string{
|
||
gamemq.EventTypeGameOrderSettled,
|
||
gamemq.EventTypeSelfGameMatchCreated,
|
||
gamemq.EventTypeSelfGameMatchReady,
|
||
gamemq.EventTypeSelfGameMatchSettled,
|
||
gamemq.EventTypeSelfGameMatchCanceled,
|
||
gamemq.EventTypeSelfGameMatchFailed,
|
||
gamemq.EventTypeSelfGamePoolAdjusted,
|
||
}
|
||
}
|
||
|
||
func replayEventID(eventID string, statTZ string) string {
|
||
token := replayToken(statTZ)
|
||
suffixed := eventID + ":stat_tz:" + token
|
||
if len(suffixed) <= replayEventIDMaxLen {
|
||
return suffixed
|
||
}
|
||
sum := sha256.Sum256([]byte(eventID + "|" + statTZ))
|
||
return "replay:" + token + ":" + hex.EncodeToString(sum[:])[:40]
|
||
}
|
||
|
||
func replayToken(statTZ string) string {
|
||
return strings.NewReplacer("/", "_", ":", "_").Replace(strings.ToLower(strings.TrimSpace(statTZ)))
|
||
}
|
||
|
||
func envDefault(key string, fallback string) string {
|
||
if value := strings.TrimSpace(os.Getenv(key)); value != "" {
|
||
return value
|
||
}
|
||
return fallback
|
||
}
|
||
|
||
func firstNonEmptyEnv(keys ...string) string {
|
||
for _, key := range keys {
|
||
if value := strings.TrimSpace(os.Getenv(key)); value != "" {
|
||
return value
|
||
}
|
||
}
|
||
return ""
|
||
}
|