429 lines
13 KiB
Go
429 lines
13 KiB
Go
package app
|
||
|
||
import (
|
||
"context"
|
||
"errors"
|
||
"fmt"
|
||
"log/slog"
|
||
"net"
|
||
"strings"
|
||
"sync"
|
||
"time"
|
||
|
||
"google.golang.org/grpc"
|
||
"google.golang.org/grpc/credentials/insecure"
|
||
healthgrpc "google.golang.org/grpc/health/grpc_health_v1"
|
||
walletv1 "hyapp.local/api/proto/wallet/v1"
|
||
"hyapp/pkg/appcode"
|
||
"hyapp/pkg/grpchealth"
|
||
"hyapp/pkg/grpcshutdown"
|
||
"hyapp/pkg/healthhttp"
|
||
"hyapp/pkg/logx"
|
||
"hyapp/pkg/rocketmqx"
|
||
"hyapp/pkg/walletmq"
|
||
"hyapp/services/wallet-service/internal/client"
|
||
"hyapp/services/wallet-service/internal/client/googleplay"
|
||
"hyapp/services/wallet-service/internal/config"
|
||
walletservice "hyapp/services/wallet-service/internal/service/wallet"
|
||
mysqlstorage "hyapp/services/wallet-service/internal/storage/mysql"
|
||
grpcserver "hyapp/services/wallet-service/internal/transport/grpc"
|
||
)
|
||
|
||
// App 装配 wallet-service gRPC 入口。
|
||
type App struct {
|
||
server *grpc.Server
|
||
listener net.Listener
|
||
health *grpchealth.ServingChecker
|
||
healthHTTP *healthhttp.Server
|
||
mysqlRepo *mysqlstorage.Repository
|
||
walletSvc *walletservice.Service
|
||
activityConn *grpc.ClientConn
|
||
outboxProducer *rocketmqx.Producer
|
||
outboxWorkerCfg config.OutboxWorkerConfig
|
||
walletOutboxTopic string
|
||
nodeID string
|
||
redPacketExpiryWorkerCfg config.RedPacketExpiryWorkerConfig
|
||
stopWorker context.CancelFunc
|
||
workers sync.WaitGroup
|
||
closeOnce sync.Once
|
||
}
|
||
|
||
// New 初始化 wallet-service。
|
||
func New(cfg config.Config) (*App, error) {
|
||
startupCtx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
|
||
defer cancel()
|
||
|
||
// wallet-service 的账户、交易、分录和 outbox 必须共享同一个 MySQL 事务边界。
|
||
repository, err := mysqlstorage.Open(startupCtx, cfg.MySQLDSN)
|
||
if err != nil {
|
||
return nil, err
|
||
}
|
||
|
||
listener, err := net.Listen("tcp", cfg.GRPCAddr)
|
||
if err != nil {
|
||
_ = repository.Close()
|
||
return nil, err
|
||
}
|
||
activityConn, err := grpc.Dial(cfg.ActivityServiceAddr, grpc.WithTransportCredentials(insecure.NewCredentials()))
|
||
if err != nil {
|
||
_ = listener.Close()
|
||
_ = repository.Close()
|
||
return nil, err
|
||
}
|
||
var outboxProducer *rocketmqx.Producer
|
||
if cfg.RocketMQ.WalletOutbox.Enabled {
|
||
outboxProducer, err = rocketmqx.NewProducer(rocketMQProducerConfig(cfg.RocketMQ))
|
||
if err != nil {
|
||
_ = activityConn.Close()
|
||
_ = listener.Close()
|
||
_ = repository.Close()
|
||
return nil, err
|
||
}
|
||
}
|
||
|
||
server := grpc.NewServer(grpc.UnaryInterceptor(logx.UnaryServerInterceptor("wallet-service")))
|
||
svc := walletservice.New(repository, client.NewActivityAchievementClient(activityConn))
|
||
if cfg.GooglePlay.Enabled {
|
||
googleClient, err := googleplay.New(cfg.GooglePlay)
|
||
if err != nil {
|
||
if outboxProducer != nil {
|
||
_ = outboxProducer.Shutdown()
|
||
}
|
||
_ = activityConn.Close()
|
||
_ = listener.Close()
|
||
_ = repository.Close()
|
||
return nil, err
|
||
}
|
||
svc.SetGooglePlayClient(googleClient)
|
||
}
|
||
walletv1.RegisterWalletServiceServer(server, grpcserver.NewServer(svc))
|
||
health := grpchealth.NewServingChecker("wallet-service", grpchealth.Dependency{
|
||
Name: "mysql",
|
||
Check: repository.Ping,
|
||
})
|
||
healthgrpc.RegisterHealthServer(server, grpchealth.NewServer(health))
|
||
healthHTTP, err := healthhttp.New(cfg.HealthHTTPAddr, cfg.NodeID, health)
|
||
if err != nil {
|
||
if outboxProducer != nil {
|
||
_ = outboxProducer.Shutdown()
|
||
}
|
||
_ = activityConn.Close()
|
||
_ = listener.Close()
|
||
_ = repository.Close()
|
||
return nil, err
|
||
}
|
||
|
||
return &App{
|
||
server: server,
|
||
listener: listener,
|
||
health: health,
|
||
healthHTTP: healthHTTP,
|
||
mysqlRepo: repository,
|
||
walletSvc: svc,
|
||
activityConn: activityConn,
|
||
outboxProducer: outboxProducer,
|
||
outboxWorkerCfg: cfg.OutboxWorker,
|
||
walletOutboxTopic: cfg.RocketMQ.WalletOutbox.Topic,
|
||
nodeID: cfg.NodeID,
|
||
redPacketExpiryWorkerCfg: cfg.RedPacketExpiryWorker,
|
||
}, nil
|
||
}
|
||
|
||
// Run 启动 gRPC 服务。
|
||
func (a *App) Run() error {
|
||
a.runHealthHTTP()
|
||
if err := a.startMQ(); err != nil {
|
||
a.health.MarkStopped()
|
||
return err
|
||
}
|
||
a.runGiftWallProjectionWorker()
|
||
a.health.MarkServing()
|
||
err := a.server.Serve(a.listener)
|
||
a.health.MarkStopped()
|
||
a.closeGiftWallProjectionWorker()
|
||
a.shutdownMQ()
|
||
if errors.Is(err, grpc.ErrServerStopped) {
|
||
return nil
|
||
}
|
||
|
||
return err
|
||
}
|
||
|
||
// Close 关闭 gRPC 服务。
|
||
func (a *App) Close() {
|
||
a.closeOnce.Do(func() {
|
||
a.health.MarkDraining()
|
||
a.closeGiftWallProjectionWorker()
|
||
grpcshutdown.GracefulStop(a.server, 15*time.Second)
|
||
a.closeHealthHTTP()
|
||
if a.activityConn != nil {
|
||
_ = a.activityConn.Close()
|
||
}
|
||
a.shutdownMQ()
|
||
if a.mysqlRepo != nil {
|
||
// MySQL 连接池最后关闭，避免正在 drain 的扣费请求丢失提交能力。
|
||
_ = a.mysqlRepo.Close()
|
||
}
|
||
})
|
||
}
|
||
|
||
func (a *App) runGiftWallProjectionWorker() {
|
||
if a.walletSvc == nil {
|
||
return
|
||
}
|
||
ctx, cancel := context.WithCancel(context.Background())
|
||
a.stopWorker = cancel
|
||
giftWallWorkerID := "gift-wall-" + a.nodeID
|
||
a.workers.Add(1)
|
||
go func() {
|
||
defer a.workers.Done()
|
||
ticker := time.NewTicker(2 * time.Second)
|
||
defer ticker.Stop()
|
||
for {
|
||
// 礼物墙是展示读模型，worker 常驻在 wallet-service 内消费账务 outbox。
|
||
// 服务重启会回滚未提交的投影事务；已完成事件由 projection done 状态防重。
|
||
processed, err := a.walletSvc.ProjectPendingGiftWallEvents(ctx, giftWallWorkerID, 50, 30*time.Second)
|
||
if err != nil && !errors.Is(err, context.Canceled) {
|
||
logx.Error(ctx, "gift_wall_projection_failed", err, slog.String("worker_id", giftWallWorkerID))
|
||
}
|
||
if processed >= 50 {
|
||
// 批次打满说明还有积压，立即继续 drain，避免固定 tick 放大展示延迟。
|
||
continue
|
||
}
|
||
select {
|
||
case <-ctx.Done():
|
||
return
|
||
case <-ticker.C:
|
||
}
|
||
}
|
||
}()
|
||
badgeWorkerID := "badge-grant-" + a.nodeID
|
||
a.workers.Add(1)
|
||
go func() {
|
||
defer a.workers.Done()
|
||
ticker := time.NewTicker(2 * time.Second)
|
||
defer ticker.Stop()
|
||
for {
|
||
// 徽章展示槽位属于 activity-service 读模型；wallet 只 relay 自己 outbox 中的资源赠送事实。
|
||
processed, err := a.walletSvc.ProjectPendingBadgeGrantEvents(ctx, badgeWorkerID, 50, 30*time.Second)
|
||
if err != nil && !errors.Is(err, context.Canceled) {
|
||
logx.Error(ctx, "badge_grant_projection_failed", err, slog.String("worker_id", badgeWorkerID))
|
||
}
|
||
if processed >= 50 {
|
||
continue
|
||
}
|
||
select {
|
||
case <-ctx.Done():
|
||
return
|
||
case <-ticker.C:
|
||
}
|
||
}
|
||
}()
|
||
if a.outboxWorkerCfg.Enabled && a.outboxProducer != nil {
|
||
outboxWorkerID := "wallet-outbox-" + a.nodeID
|
||
a.workers.Add(1)
|
||
go func() {
|
||
defer a.workers.Done()
|
||
a.runWalletOutboxWorker(ctx, outboxWorkerID)
|
||
}()
|
||
}
|
||
if a.redPacketExpiryWorkerCfg.Enabled {
|
||
redPacketWorkerID := "red-packet-expiry-" + a.nodeID
|
||
a.workers.Add(1)
|
||
go func() {
|
||
defer a.workers.Done()
|
||
ticker := time.NewTicker(a.redPacketExpiryWorkerCfg.PollInterval)
|
||
defer ticker.Stop()
|
||
for {
|
||
result, err := a.walletSvc.ExpireRedPackets(ctx, a.redPacketExpiryWorkerCfg.AppCode, a.redPacketExpiryWorkerCfg.BatchSize)
|
||
if err != nil && !errors.Is(err, context.Canceled) {
|
||
logx.Error(ctx, "red_packet_expiry_failed", err, slog.String("worker_id", redPacketWorkerID))
|
||
}
|
||
if result.ExpiredCount > 0 {
|
||
logx.Info(ctx, "red_packet_expired",
|
||
slog.String("worker_id", redPacketWorkerID),
|
||
slog.Int64("expired_count", int64(result.ExpiredCount)),
|
||
slog.Int64("refunded_amount", result.RefundedAmount),
|
||
)
|
||
}
|
||
if result.ExpiredCount >= a.redPacketExpiryWorkerCfg.BatchSize {
|
||
continue
|
||
}
|
||
select {
|
||
case <-ctx.Done():
|
||
return
|
||
case <-ticker.C:
|
||
}
|
||
}
|
||
}()
|
||
}
|
||
}
|
||
|
||
func (a *App) runWalletOutboxWorker(ctx context.Context, workerID string) {
|
||
options := a.outboxWorkerCfg
|
||
ticker := time.NewTicker(options.PollInterval)
|
||
defer ticker.Stop()
|
||
for {
|
||
processed, err := a.processWalletOutboxBatch(ctx, workerID, options)
|
||
if err != nil && !errors.Is(err, context.Canceled) {
|
||
logx.Error(ctx, "wallet_outbox_publish_failed", err, slog.String("worker_id", workerID))
|
||
}
|
||
if processed >= options.BatchSize {
|
||
continue
|
||
}
|
||
select {
|
||
case <-ctx.Done():
|
||
return
|
||
case <-ticker.C:
|
||
}
|
||
}
|
||
}
|
||
|
||
func (a *App) processWalletOutboxBatch(ctx context.Context, workerID string, options config.OutboxWorkerConfig) (int, error) {
|
||
if a.mysqlRepo == nil || a.outboxProducer == nil {
|
||
return 0, nil
|
||
}
|
||
records, err := a.mysqlRepo.ClaimPendingWalletOutbox(ctx, workerID, options.BatchSize, time.Now().UTC().Add(options.PublishTimeout).UnixMilli())
|
||
if err != nil {
|
||
return 0, err
|
||
}
|
||
for _, record := range records {
|
||
if record.RetryCount >= options.MaxRetryCount {
|
||
markCtx, cancel := context.WithTimeout(appcode.WithContext(context.Background(), record.AppCode), options.PublishTimeout)
|
||
err := a.mysqlRepo.MarkWalletOutboxDead(markCtx, record.EventID, deadWalletOutboxReason(record))
|
||
cancel()
|
||
if err != nil {
|
||
return 0, err
|
||
}
|
||
continue
|
||
}
|
||
publishCtx, cancel := context.WithTimeout(appcode.WithContext(context.Background(), record.AppCode), options.PublishTimeout)
|
||
err := a.publishWalletOutboxRecord(publishCtx, record)
|
||
cancel()
|
||
if err != nil {
|
||
nextRetryAtMS := time.Now().UTC().Add(walletOutboxBackoff(record.RetryCount+1, options)).UnixMilli()
|
||
markCtx, markCancel := context.WithTimeout(appcode.WithContext(context.Background(), record.AppCode), options.PublishTimeout)
|
||
markErr := a.mysqlRepo.MarkWalletOutboxRetryable(markCtx, record.EventID, err.Error(), nextRetryAtMS)
|
||
markCancel()
|
||
if markErr != nil {
|
||
return 0, markErr
|
||
}
|
||
continue
|
||
}
|
||
markCtx, markCancel := context.WithTimeout(appcode.WithContext(context.Background(), record.AppCode), options.PublishTimeout)
|
||
markErr := a.mysqlRepo.MarkWalletOutboxDelivered(markCtx, record.EventID)
|
||
markCancel()
|
||
if markErr != nil {
|
||
return 0, markErr
|
||
}
|
||
}
|
||
return len(records), nil
|
||
}
|
||
|
||
func (a *App) publishWalletOutboxRecord(ctx context.Context, record mysqlstorage.WalletOutboxRecord) error {
|
||
body, err := walletmq.EncodeWalletOutboxMessage(walletmq.WalletOutboxMessage{
|
||
AppCode: record.AppCode,
|
||
EventID: record.EventID,
|
||
EventType: record.EventType,
|
||
TransactionID: record.TransactionID,
|
||
CommandID: record.CommandID,
|
||
UserID: record.UserID,
|
||
AssetType: record.AssetType,
|
||
AvailableDelta: record.AvailableDelta,
|
||
FrozenDelta: record.FrozenDelta,
|
||
PayloadJSON: record.PayloadJSON,
|
||
OccurredAtMS: record.CreatedAtMS,
|
||
})
|
||
if err != nil {
|
||
return err
|
||
}
|
||
return a.outboxProducer.SendSync(ctx, rocketmqx.Message{
|
||
Topic: a.walletOutboxTopic,
|
||
Tag: walletmq.TagWalletOutboxEvent,
|
||
Keys: []string{record.EventID, record.TransactionID, record.CommandID},
|
||
Body: body,
|
||
})
|
||
}
|
||
|
||
func deadWalletOutboxReason(record mysqlstorage.WalletOutboxRecord) string {
|
||
last := strings.TrimSpace(record.LastError)
|
||
if last == "" {
|
||
return fmt.Sprintf("wallet outbox exceeded retry limit: retry_count=%d", record.RetryCount)
|
||
}
|
||
return last
|
||
}
|
||
|
||
func walletOutboxBackoff(retryCount int, options config.OutboxWorkerConfig) time.Duration {
|
||
if retryCount <= 0 {
|
||
return options.InitialBackoff
|
||
}
|
||
backoff := options.InitialBackoff
|
||
for i := 1; i < retryCount; i++ {
|
||
backoff *= 2
|
||
if backoff >= options.MaxBackoff {
|
||
return options.MaxBackoff
|
||
}
|
||
}
|
||
return backoff
|
||
}
|
||
|
||
func (a *App) closeGiftWallProjectionWorker() {
|
||
if a.stopWorker != nil {
|
||
a.stopWorker()
|
||
}
|
||
a.workers.Wait()
|
||
}
|
||
|
||
func (a *App) startMQ() error {
|
||
if a.outboxProducer == nil {
|
||
return nil
|
||
}
|
||
return a.outboxProducer.Start()
|
||
}
|
||
|
||
func (a *App) shutdownMQ() {
|
||
if a.outboxProducer == nil {
|
||
return
|
||
}
|
||
if err := a.outboxProducer.Shutdown(); err != nil {
|
||
logx.Warn(context.Background(), "rocketmq_producer_shutdown_failed", slog.String("error", err.Error()))
|
||
}
|
||
}
|
||
|
||
func rocketMQProducerConfig(cfg config.RocketMQConfig) rocketmqx.ProducerConfig {
|
||
return rocketmqx.ProducerConfig{
|
||
EndpointConfig: rocketmqx.EndpointConfig{
|
||
NameServers: cfg.NameServers,
|
||
NameServerDomain: cfg.NameServerDomain,
|
||
AccessKey: cfg.AccessKey,
|
||
SecretKey: cfg.SecretKey,
|
||
SecurityToken: cfg.SecurityToken,
|
||
Namespace: cfg.Namespace,
|
||
VIPChannel: cfg.VIPChannel,
|
||
},
|
||
GroupName: cfg.WalletOutbox.ProducerGroup,
|
||
SendTimeout: cfg.SendTimeout,
|
||
Retry: cfg.Retry,
|
||
}
|
||
}
|
||
|
||
func (a *App) runHealthHTTP() {
|
||
if a.healthHTTP == nil {
|
||
return
|
||
}
|
||
go func() {
|
||
if err := a.healthHTTP.Run(); err != nil {
|
||
logx.Error(context.Background(), "health_http_run_failed", err)
|
||
}
|
||
}()
|
||
}
|
||
|
||
func (a *App) closeHealthHTTP() {
|
||
if a.healthHTTP == nil {
|
||
return
|
||
}
|
||
ctx, cancel := context.WithTimeout(context.Background(), 2*time.Second)
|
||
defer cancel()
|
||
_ = a.healthHTTP.Close(ctx)
|
||
}
|