354 lines
12 KiB
Go
354 lines
12 KiB
Go
// Package app 负责把 room-service 的领域服务、存储、路由目录和 gRPC 入口装配成进程。
|
||
package app
|
||
|
||
import (
|
||
"context"
|
||
"errors"
|
||
"log/slog"
|
||
"net"
|
||
"sync"
|
||
"time"
|
||
|
||
"google.golang.org/grpc"
|
||
"google.golang.org/grpc/credentials/insecure"
|
||
healthgrpc "google.golang.org/grpc/health/grpc_health_v1"
|
||
activityv1 "hyapp.local/api/proto/activity/v1"
|
||
roomv1 "hyapp.local/api/proto/room/v1"
|
||
"hyapp/pkg/grpchealth"
|
||
"hyapp/pkg/grpcshutdown"
|
||
"hyapp/pkg/healthhttp"
|
||
"hyapp/pkg/logx"
|
||
"hyapp/pkg/tencentim"
|
||
"hyapp/services/room-service/internal/config"
|
||
"hyapp/services/room-service/internal/healthcheck"
|
||
"hyapp/services/room-service/internal/integration"
|
||
roomservice "hyapp/services/room-service/internal/room/service"
|
||
"hyapp/services/room-service/internal/router"
|
||
mysqlstorage "hyapp/services/room-service/internal/storage/mysql"
|
||
grpcserver "hyapp/services/room-service/internal/transport/grpc"
|
||
)
|
||
|
||
// App 负责装配 room-service 运行时依赖。
|
||
type App struct {
|
||
// cfg 保留启动配置，便于后续扩展后台 worker 时复用。
|
||
cfg config.Config
|
||
// grpcServer 承载 RoomCommandService、RoomGuardService 和标准 gRPC health。
|
||
grpcServer *grpc.Server
|
||
// service 是 Room Cell 领域入口，后台 presence worker 复用它执行清理命令。
|
||
service *roomservice.Service
|
||
// listener 是 room-service 的唯一网络入口。
|
||
listener net.Listener
|
||
// walletConn 是 SendGift 同步扣费依赖。
|
||
walletConn *grpc.ClientConn
|
||
// activityConn 是 room outbox 事实投递到 activity-service 的 best-effort 通道。
|
||
activityConn *grpc.ClientConn
|
||
// repository 是 MySQL 持久化实现，保存 meta、command log、snapshot 和 outbox。
|
||
repository *mysqlstorage.Repository
|
||
// redisClose 关闭 Redis client，Redis directory 本身不拥有额外生命周期。
|
||
redisClose func() error
|
||
// nodeRegistry 保存当前节点可被 owner 转发直连的实例地址。
|
||
nodeRegistry router.NodeRegistry
|
||
// health 汇总 gRPC、runtime、MySQL 和 Redis lease 探测状态。
|
||
health *healthcheck.State
|
||
// healthHTTP 给 CLB 和发布脚本提供 HTTP readiness，不承载业务请求。
|
||
healthHTTP *healthhttp.Server
|
||
// workerCtx 统一控制本节点后台 worker，关闭时必须先停止 worker 再释放 MySQL/Redis。
|
||
workerCtx context.Context
|
||
// workerCancel 停止 presence stale 和 outbox 补偿 worker。
|
||
workerCancel context.CancelFunc
|
||
// workerWG 等待后台 worker 当前事件或当前扫描批次安全退出。
|
||
workerWG sync.WaitGroup
|
||
// closeOnce 防止信号退出和 Serve 错误同时触发重复关闭。
|
||
closeOnce sync.Once
|
||
}
|
||
|
||
// New 初始化 room-service 应用。
|
||
func New(cfg config.Config) (*App, error) {
|
||
normalized, err := config.Normalize(cfg)
|
||
if err != nil {
|
||
return nil, err
|
||
}
|
||
cfg = normalized
|
||
|
||
// wallet-service 是 SendGift 主链路依赖，连接对象创建失败时不能启动。
|
||
walletConn, err := grpc.Dial(cfg.WalletServiceAddr, grpc.WithTransportCredentials(insecure.NewCredentials()))
|
||
if err != nil {
|
||
return nil, err
|
||
}
|
||
activityConn, err := grpc.Dial(cfg.ActivityServiceAddr, grpc.WithTransportCredentials(insecure.NewCredentials()))
|
||
if err != nil {
|
||
_ = walletConn.Close()
|
||
return nil, err
|
||
}
|
||
|
||
startupCtx, cancel := context.WithTimeout(context.Background(), 15*time.Second)
|
||
defer cancel()
|
||
|
||
// MySQL 是房间恢复的持久来源，启动阶段必须确认连接池可用。
|
||
repository, err := mysqlstorage.Open(startupCtx, cfg.MySQLDSN, cfg.MySQLMaxOpenConns, cfg.MySQLMaxIdleConns)
|
||
if err != nil {
|
||
_ = activityConn.Close()
|
||
_ = walletConn.Close()
|
||
return nil, err
|
||
}
|
||
|
||
if cfg.MySQLAutoMigrate {
|
||
// 本地和测试环境可自动建表，线上应通过显式迁移控制结构变更。
|
||
if err := repository.Migrate(startupCtx); err != nil {
|
||
_ = repository.Close()
|
||
_ = activityConn.Close()
|
||
_ = walletConn.Close()
|
||
return nil, err
|
||
}
|
||
}
|
||
|
||
// Redis directory 保存 room owner lease，是单房间单活的运行时仲裁来源。
|
||
redisClient, err := router.NewRedisClient(startupCtx, cfg.RedisAddr, cfg.RedisPassword, cfg.RedisDB)
|
||
if err != nil {
|
||
_ = repository.Close()
|
||
_ = activityConn.Close()
|
||
_ = walletConn.Close()
|
||
return nil, err
|
||
}
|
||
|
||
directory := router.NewRedisDirectory(redisClient)
|
||
roomPublisher := integration.NewNoopRoomEventPublisher()
|
||
activityPublisher := integration.NewActivityOutboxPublisher(activityv1.NewRoomEventConsumerServiceClient(activityConn), time.Second)
|
||
outboxPublishers := []integration.OutboxPublisher{activityPublisher}
|
||
if cfg.TencentIM.Enabled {
|
||
// 腾讯云 IM 替代自研 IM；room-service 只负责服务端 REST 群消息桥接。
|
||
tencentClient, err := tencentim.NewRESTClient(cfg.TencentIM.RESTConfig())
|
||
if err != nil {
|
||
_ = repository.Close()
|
||
_ = activityConn.Close()
|
||
_ = walletConn.Close()
|
||
_ = redisClient.Close()
|
||
return nil, err
|
||
}
|
||
tencentPublisher := integration.NewTencentIMPublisher(tencentClient)
|
||
roomPublisher = tencentPublisher
|
||
outboxPublishers = append(outboxPublishers, tencentPublisher)
|
||
}
|
||
outboxPublisher := integration.NewCompositeOutboxPublisher(outboxPublishers...)
|
||
|
||
// 领域服务只依赖接口，App 层负责选择 MySQL/Redis/gRPC 具体实现。
|
||
svc := roomservice.New(roomservice.Config{
|
||
NodeID: cfg.NodeID,
|
||
LeaseTTL: cfg.LeaseTTL,
|
||
RankLimit: cfg.RankLimit,
|
||
SnapshotEveryN: cfg.SnapshotEveryN,
|
||
PresenceStaleAfter: cfg.PresenceStaleAfter,
|
||
MicPublishTimeout: cfg.MicPublishTimeout,
|
||
}, directory, repository, integration.NewGRPCWalletClient(walletConn), roomPublisher, outboxPublisher)
|
||
|
||
server := grpc.NewServer(grpc.UnaryInterceptor(logx.UnaryServerInterceptor("room-service")))
|
||
roomServer := grpcserver.NewServer(svc, grpcserver.WithOwnerForwarding(cfg.NodeID, directory, directory, cfg.OwnerForwardTimeout))
|
||
// 命令服务处理房间状态变更，守卫服务给腾讯云 IM 回调或 gateway 做发言和进群校验。
|
||
roomv1.RegisterRoomCommandServiceServer(server, roomServer)
|
||
roomv1.RegisterRoomGuardServiceServer(server, roomServer)
|
||
roomv1.RegisterRoomQueryServiceServer(server, roomServer)
|
||
healthState := healthcheck.NewState(cfg.NodeID, svc, repository, directory)
|
||
healthgrpc.RegisterHealthServer(server, grpchealth.NewServer(healthState))
|
||
healthHTTP, err := healthhttp.New(cfg.HealthHTTPAddr, cfg.NodeID, healthState)
|
||
if err != nil {
|
||
_ = activityConn.Close()
|
||
_ = walletConn.Close()
|
||
_ = repository.Close()
|
||
_ = redisClient.Close()
|
||
return nil, err
|
||
}
|
||
|
||
// listener 在 New 阶段创建，可以把端口占用作为启动失败暴露。
|
||
listener, err := net.Listen("tcp", cfg.GRPCAddr)
|
||
if err != nil {
|
||
_ = healthHTTP.Close(context.Background())
|
||
_ = activityConn.Close()
|
||
_ = walletConn.Close()
|
||
_ = repository.Close()
|
||
_ = redisClient.Close()
|
||
return nil, err
|
||
}
|
||
workerCtx, workerCancel := context.WithCancel(context.Background())
|
||
|
||
return &App{
|
||
cfg: cfg,
|
||
grpcServer: server,
|
||
service: svc,
|
||
listener: listener,
|
||
walletConn: walletConn,
|
||
activityConn: activityConn,
|
||
repository: repository,
|
||
redisClose: redisClient.Close,
|
||
nodeRegistry: directory,
|
||
health: healthState,
|
||
healthHTTP: healthHTTP,
|
||
workerCtx: workerCtx,
|
||
workerCancel: workerCancel,
|
||
}, nil
|
||
}
|
||
|
||
// Run 启动 gRPC 服务。
|
||
func (a *App) Run() error {
|
||
a.runHealthHTTP()
|
||
// health 先标记 serving，再进入 Serve，ready 才能在监听开始后通过。
|
||
a.health.MarkGRPCServing()
|
||
if err := a.registerRoomNode(context.Background()); err != nil {
|
||
a.health.MarkGRPCStopped()
|
||
return err
|
||
}
|
||
defer func() {
|
||
if a.workerCancel != nil {
|
||
a.workerCancel()
|
||
}
|
||
a.workerWG.Wait()
|
||
}()
|
||
a.workerWG.Go(func() {
|
||
a.runNodeRegistryHeartbeat(a.workerCtx)
|
||
})
|
||
if a.cfg.PresenceStaleScanInterval > 0 {
|
||
// presence worker 只清理本节点已装载 Cell，命令仍走 Room Cell 持久化链路。
|
||
a.workerWG.Go(func() {
|
||
a.service.RunPresenceStaleWorker(a.workerCtx, a.cfg.PresenceStaleScanInterval)
|
||
})
|
||
}
|
||
if a.cfg.MicPublishScanInterval > 0 {
|
||
// MicUp 只代表业务占麦；后台 worker 负责释放未确认 RTC 发流的 pending_publish 麦位。
|
||
a.workerWG.Go(func() {
|
||
a.service.RunMicPublishTimeoutWorker(a.workerCtx, a.cfg.MicPublishScanInterval)
|
||
})
|
||
}
|
||
if a.cfg.OutboxWorker.Enabled {
|
||
a.workerWG.Go(func() {
|
||
a.service.RunOutboxWorker(a.workerCtx, roomservice.OutboxWorkerOptions{
|
||
PollInterval: a.cfg.OutboxWorker.PollInterval,
|
||
BatchSize: a.cfg.OutboxWorker.BatchSize,
|
||
PublishTimeout: a.cfg.OutboxWorker.PublishTimeout,
|
||
RetryStrategy: a.cfg.OutboxWorker.RetryStrategy,
|
||
MaxRetryCount: a.cfg.OutboxWorker.MaxRetryCount,
|
||
InitialBackoff: a.cfg.OutboxWorker.InitialBackoff,
|
||
MaxBackoff: a.cfg.OutboxWorker.MaxBackoff,
|
||
})
|
||
})
|
||
}
|
||
|
||
err := a.grpcServer.Serve(a.listener)
|
||
a.health.MarkGRPCStopped()
|
||
if errors.Is(err, grpc.ErrServerStopped) {
|
||
// GracefulStop 触发的退出不是故障。
|
||
return nil
|
||
}
|
||
|
||
return err
|
||
}
|
||
|
||
// Close 关闭底层 gRPC 服务。
|
||
func (a *App) Close() {
|
||
a.closeOnce.Do(func() {
|
||
// 先进入 draining，避免下线中的节点继续接管房间 lease 或接收新命令。
|
||
a.health.MarkDraining()
|
||
if a.service != nil {
|
||
a.service.MarkDraining()
|
||
}
|
||
if a.workerCancel != nil {
|
||
// 先停后台 worker 并等待当前投递结果落库，再关闭 gRPC/MySQL/Redis。
|
||
a.workerCancel()
|
||
}
|
||
a.workerWG.Wait()
|
||
// GracefulStop 让已进入的 gRPC 请求自然结束。
|
||
grpcshutdown.GracefulStop(a.grpcServer, 15*time.Second)
|
||
if a.service != nil {
|
||
// gRPC drain 完成后释放本节点持有的 room route，降低滚动发布时等待 Redis TTL 的窗口。
|
||
releaseCtx, releaseCancel := context.WithTimeout(context.Background(), 5*time.Second)
|
||
a.service.ReleaseLoadedRoomLeases(releaseCtx)
|
||
releaseCancel()
|
||
}
|
||
a.unregisterRoomNode()
|
||
a.closeHealthHTTP()
|
||
if a.walletConn != nil {
|
||
_ = a.walletConn.Close()
|
||
}
|
||
if a.activityConn != nil {
|
||
_ = a.activityConn.Close()
|
||
}
|
||
if a.repository != nil {
|
||
// MySQL 连接池最后释放，保证 GracefulStop 期间仍可完成持久化。
|
||
_ = a.repository.Close()
|
||
}
|
||
if a.redisClose != nil {
|
||
// Redis client 关闭后本节点不再续约或接管房间。
|
||
_ = a.redisClose()
|
||
}
|
||
})
|
||
}
|
||
|
||
func (a *App) registerRoomNode(ctx context.Context) error {
|
||
if a.nodeRegistry == nil {
|
||
return nil
|
||
}
|
||
if _, ok := ctx.Deadline(); !ok {
|
||
var cancel context.CancelFunc
|
||
ctx, cancel = context.WithTimeout(ctx, 2*time.Second)
|
||
defer cancel()
|
||
}
|
||
return a.nodeRegistry.RegisterNode(ctx, router.NodeRegistration{
|
||
NodeID: a.cfg.NodeID,
|
||
GRPCAddr: a.cfg.AdvertiseAddr,
|
||
}, a.cfg.NodeRegistryTTL)
|
||
}
|
||
|
||
func (a *App) runNodeRegistryHeartbeat(ctx context.Context) {
|
||
if a.nodeRegistry == nil {
|
||
return
|
||
}
|
||
ticker := time.NewTicker(a.cfg.NodeRegistryHeartbeatInterval)
|
||
defer ticker.Stop()
|
||
for {
|
||
select {
|
||
case <-ctx.Done():
|
||
return
|
||
case <-ticker.C:
|
||
if err := a.registerRoomNode(ctx); err != nil {
|
||
logx.Warn(ctx, "room_node_register_failed",
|
||
slog.String("node_id", a.cfg.NodeID),
|
||
slog.String("advertise_addr", a.cfg.AdvertiseAddr),
|
||
slog.String("error", err.Error()),
|
||
)
|
||
}
|
||
}
|
||
}
|
||
}
|
||
|
||
func (a *App) unregisterRoomNode() {
|
||
if a.nodeRegistry == nil {
|
||
return
|
||
}
|
||
ctx, cancel := context.WithTimeout(context.Background(), 2*time.Second)
|
||
defer cancel()
|
||
if err := a.nodeRegistry.UnregisterNode(ctx, a.cfg.NodeID); err != nil {
|
||
logx.Warn(ctx, "room_node_unregister_failed",
|
||
slog.String("node_id", a.cfg.NodeID),
|
||
slog.String("error", err.Error()),
|
||
)
|
||
}
|
||
}
|
||
|
||
func (a *App) runHealthHTTP() {
|
||
if a.healthHTTP == nil {
|
||
return
|
||
}
|
||
go func() {
|
||
if err := a.healthHTTP.Run(); err != nil {
|
||
logx.Error(context.Background(), "health_http_run_failed", err)
|
||
}
|
||
}()
|
||
}
|
||
|
||
func (a *App) closeHealthHTTP() {
|
||
if a.healthHTTP == nil {
|
||
return
|
||
}
|
||
ctx, cancel := context.WithTimeout(context.Background(), 2*time.Second)
|
||
defer cancel()
|
||
_ = a.healthHTTP.Close(ctx)
|
||
}
|