// Package app 负责把 room-service 的领域服务、存储、路由目录和 gRPC 入口装配成进程。 package app import ( "context" "errors" "log/slog" "net" "sync" "time" "google.golang.org/grpc" "google.golang.org/grpc/credentials/insecure" healthgrpc "google.golang.org/grpc/health/grpc_health_v1" activityv1 "hyapp.local/api/proto/activity/v1" roomv1 "hyapp.local/api/proto/room/v1" "hyapp/pkg/appcode" "hyapp/pkg/grpchealth" "hyapp/pkg/grpcshutdown" "hyapp/pkg/healthhttp" "hyapp/pkg/logx" "hyapp/pkg/rocketmqx" "hyapp/pkg/roommq" "hyapp/pkg/tencentim" "hyapp/pkg/tencentrtc" "hyapp/pkg/usermq" "hyapp/services/room-service/internal/config" "hyapp/services/room-service/internal/healthcheck" "hyapp/services/room-service/internal/integration" roomservice "hyapp/services/room-service/internal/room/service" "hyapp/services/room-service/internal/router" mysqlstorage "hyapp/services/room-service/internal/storage/mysql" grpcserver "hyapp/services/room-service/internal/transport/grpc" ) // App 负责装配 room-service 运行时依赖。 type App struct { // cfg 保留启动配置,便于后续扩展后台 worker 时复用。 cfg config.Config // grpcServer 承载 RoomCommandService、RoomGuardService 和标准 gRPC health。 grpcServer *grpc.Server // service 是 Room Cell 领域入口,后台 presence worker 复用它执行清理命令。 service *roomservice.Service // listener 是 room-service 的唯一网络入口。 listener net.Listener // walletConn 是 SendGift 同步扣费依赖。 walletConn *grpc.ClientConn // activityConn 是 room outbox 事实投递到 activity-service 的 best-effort 通道。 activityConn *grpc.ClientConn // repository 是 MySQL 持久化实现,保存 meta、command log、snapshot 和 outbox。 repository *mysqlstorage.Repository // redisClose 关闭 Redis client,Redis directory 本身不拥有额外生命周期。 redisClose func() error // nodeRegistry 保存当前节点可被 owner 转发直连的实例地址。 nodeRegistry router.NodeRegistry // health 汇总 gRPC、runtime、MySQL 和 Redis lease 探测状态。 health *healthcheck.State // healthHTTP 给 CLB 和发布脚本提供 HTTP readiness,不承载业务请求。 healthHTTP *healthhttp.Server // mqProducers 在 outbox worker 启动前打开,承载 fanout 和延迟发射唤醒。 mqProducers []*rocketmqx.Producer // mqConsumers 在 gRPC 服务启动前订阅,承接 room_outbox fanout 和 delayed open。 mqConsumers []*rocketmqx.Consumer // workerCtx 统一控制本节点后台 worker,关闭时必须先停止 worker 再释放 MySQL/Redis。 workerCtx context.Context // workerCancel 停止 presence stale 和 outbox 补偿 worker。 workerCancel context.CancelFunc // workerWG 等待后台 worker 当前事件或当前扫描批次安全退出。 workerWG sync.WaitGroup // closeOnce 防止信号退出和 Serve 错误同时触发重复关闭。 closeOnce sync.Once } // New 初始化 room-service 应用。 func New(cfg config.Config) (*App, error) { normalized, err := config.Normalize(cfg) if err != nil { return nil, err } cfg = normalized // wallet-service 是 SendGift 主链路依赖,连接对象创建失败时不能启动。 walletConn, err := grpc.Dial(cfg.WalletServiceAddr, grpc.WithTransportCredentials(insecure.NewCredentials())) if err != nil { return nil, err } var activityConn *grpc.ClientConn if cfg.ActivityServiceAddr != "" { activityConn, err = grpc.Dial(cfg.ActivityServiceAddr, grpc.WithTransportCredentials(insecure.NewCredentials())) if err != nil { _ = walletConn.Close() return nil, err } } startupCtx, cancel := context.WithTimeout(context.Background(), 15*time.Second) defer cancel() // MySQL 是房间恢复的持久来源,启动阶段必须确认连接池可用。 repository, err := mysqlstorage.Open(startupCtx, cfg.MySQLDSN, cfg.MySQLMaxOpenConns, cfg.MySQLMaxIdleConns) if err != nil { closeClientConn(activityConn) _ = walletConn.Close() return nil, err } if cfg.MySQLAutoMigrate { // 本地和测试环境可自动建表,线上应通过显式迁移控制结构变更。 if err := repository.Migrate(startupCtx); err != nil { _ = repository.Close() closeClientConn(activityConn) _ = walletConn.Close() return nil, err } } // Redis directory 保存 room owner lease,是单房间单活的运行时仲裁来源。 redisClient, err := router.NewRedisClient(startupCtx, cfg.RedisAddr, cfg.RedisPassword, cfg.RedisDB) if err != nil { _ = repository.Close() closeClientConn(activityConn) _ = walletConn.Close() return nil, err } directory := router.NewRedisDirectory(redisClient) roomPublisher := integration.NewNoopRoomEventPublisher() outboxPublishers := make([]integration.OutboxPublisher, 0, 3) var rtcUserRemover integration.RTCUserRemover var tencentPublisher *integration.TencentIMPublisher if cfg.TencentIM.Enabled { // 腾讯云 IM 替代自研 IM;room-service 只负责服务端 REST 群消息桥接。 tencentClient, err := tencentim.NewRESTClient(cfg.TencentIM.RESTConfig()) if err != nil { _ = repository.Close() closeClientConn(activityConn) _ = walletConn.Close() _ = redisClient.Close() return nil, err } tencentPublisher = integration.NewTencentIMPublisher(tencentClient) roomPublisher = tencentPublisher } if cfg.TencentRTC.Enabled { rtcClient, err := tencentrtc.NewRESTClient(cfg.TencentRTC.RESTConfig()) if err != nil { _ = repository.Close() closeClientConn(activityConn) _ = walletConn.Close() _ = redisClient.Close() return nil, err } rtcUserRemover = rtcClient } mqProducers := make([]*rocketmqx.Producer, 0, 2) mqConsumers := make([]*rocketmqx.Consumer, 0, 3) var rocketLaunchScheduler integration.RoomRocketLaunchScheduler if cfg.OutboxWorker.PublishMode == config.OutboxPublishModeDirect || cfg.OutboxWorker.PublishMode == config.OutboxPublishModeDual { if activityConn != nil { activityPublisher := integration.NewActivityOutboxPublisher(activityv1.NewRoomEventConsumerServiceClient(activityConn), time.Second) outboxPublishers = append(outboxPublishers, activityPublisher) } if tencentPublisher != nil { outboxPublishers = append(outboxPublishers, tencentPublisher) } } if cfg.OutboxWorker.PublishMode == config.OutboxPublishModeMQ || cfg.OutboxWorker.PublishMode == config.OutboxPublishModeDual { outboxProducer, err := rocketmqx.NewProducer(rocketMQProducerConfig(cfg.RocketMQ, cfg.RocketMQ.RoomOutbox.ProducerGroup)) if err != nil { _ = repository.Close() closeClientConn(activityConn) _ = walletConn.Close() _ = redisClient.Close() return nil, err } mqProducers = append(mqProducers, outboxProducer) outboxPublishers = append(outboxPublishers, integration.NewRocketMQOutboxPublisher(outboxProducer, cfg.RocketMQ.RoomOutbox.Topic)) } if cfg.RocketMQ.RocketLaunch.Enabled { rocketProducer, err := rocketmqx.NewProducer(rocketMQProducerConfig(cfg.RocketMQ, cfg.RocketMQ.RocketLaunch.ProducerGroup)) if err != nil { _ = repository.Close() closeClientConn(activityConn) _ = walletConn.Close() _ = redisClient.Close() return nil, err } mqProducers = append(mqProducers, rocketProducer) rocketLaunchScheduler = integration.NewRocketMQRocketLaunchScheduler(rocketProducer, cfg.RocketMQ.RocketLaunch.Topic) } outboxPublisher := integration.NewCompositeOutboxPublisher(outboxPublishers...) var luckyGiftClient integration.LuckyGiftClient if activityConn != nil { luckyGiftClient = integration.NewGRPCLuckyGiftClient(activityConn) } // 领域服务只依赖接口,App 层负责选择 MySQL/Redis/gRPC 具体实现。 svc := roomservice.New(roomservice.Config{ NodeID: cfg.NodeID, LeaseTTL: cfg.LeaseTTL, RankLimit: cfg.RankLimit, SnapshotEveryN: cfg.SnapshotEveryN, PresenceStaleAfter: cfg.PresenceStaleAfter, MicPublishTimeout: cfg.MicPublishTimeout, RTCUserRemover: rtcUserRemover, RoomRocketLaunchScheduler: rocketLaunchScheduler, RoomGiftLeaderboard: roomservice.NewRedisRoomGiftLeaderboardStore(redisClient), LuckyGiftSendLocker: roomservice.NewRedisLuckyGiftSendLocker(redisClient), LuckyGiftSendLockTTL: cfg.LuckyGiftSendLockTTL, }, directory, repository, integration.NewGRPCWalletClient(walletConn), roomPublisher, outboxPublisher, luckyGiftClient) if cfg.RocketMQ.RocketLaunch.Enabled { rocketConsumer, err := rocketmqx.NewConsumer(rocketMQConsumerConfig(cfg.RocketMQ, cfg.RocketMQ.RocketLaunch.ConsumerGroup, cfg.RocketMQ.RocketLaunch.ConsumerMaxReconsumeTimes)) if err != nil { _ = repository.Close() closeClientConn(activityConn) _ = walletConn.Close() _ = redisClient.Close() return nil, err } if err := rocketConsumer.Subscribe(cfg.RocketMQ.RocketLaunch.Topic, roommq.TagRoomRocketLaunchDue, func(ctx context.Context, message rocketmqx.ConsumedMessage) error { due, err := roommq.DecodeRoomRocketLaunchDueMessage(message.Body) if err != nil { return err } return svc.HandleRoomRocketLaunchDue(ctx, due) }); err != nil { _ = repository.Close() closeClientConn(activityConn) _ = walletConn.Close() _ = redisClient.Close() return nil, err } mqConsumers = append(mqConsumers, rocketConsumer) } if cfg.RocketMQ.RoomOutbox.TencentIMConsumerEnabled && tencentPublisher == nil { _ = repository.Close() closeClientConn(activityConn) _ = walletConn.Close() _ = redisClient.Close() return nil, errors.New("rocketmq room_outbox tencent_im consumer requires tencent_im.enabled") } if cfg.RocketMQ.RoomOutbox.TencentIMConsumerEnabled && tencentPublisher != nil { imConsumer, err := rocketmqx.NewConsumer(rocketMQConsumerConfig(cfg.RocketMQ, cfg.RocketMQ.RoomOutbox.TencentIMConsumerGroup, cfg.RocketMQ.RoomOutbox.ConsumerMaxReconsumeTimes)) if err != nil { _ = repository.Close() closeClientConn(activityConn) _ = walletConn.Close() _ = redisClient.Close() return nil, err } if err := imConsumer.Subscribe(cfg.RocketMQ.RoomOutbox.Topic, roommq.TagRoomOutboxEvent, func(ctx context.Context, message rocketmqx.ConsumedMessage) error { envelope, _, err := roommq.DecodeRoomOutboxMessage(message.Body) if err != nil { return err } return tencentPublisher.PublishOutboxEvent(appcode.WithContext(ctx, envelope.GetAppCode()), envelope) }); err != nil { _ = repository.Close() closeClientConn(activityConn) _ = walletConn.Close() _ = redisClient.Close() return nil, err } mqConsumers = append(mqConsumers, imConsumer) } if cfg.RocketMQ.UserOutbox.Enabled { userConsumer, err := rocketmqx.NewConsumer(rocketMQConsumerConfig(cfg.RocketMQ, cfg.RocketMQ.UserOutbox.ConsumerGroup, cfg.RocketMQ.UserOutbox.ConsumerMaxReconsumeTimes)) if err != nil { _ = repository.Close() closeClientConn(activityConn) _ = walletConn.Close() _ = redisClient.Close() return nil, err } if err := userConsumer.Subscribe(cfg.RocketMQ.UserOutbox.Topic, usermq.TagUserOutboxEvent, func(ctx context.Context, message rocketmqx.ConsumedMessage) error { outboxMessage, err := usermq.DecodeUserOutboxMessage(message.Body) if err != nil { return err } return svc.HandleUserOutboxMessage(ctx, outboxMessage) }); err != nil { _ = repository.Close() closeClientConn(activityConn) _ = walletConn.Close() _ = redisClient.Close() return nil, err } mqConsumers = append(mqConsumers, userConsumer) } server := grpc.NewServer(grpc.UnaryInterceptor(logx.UnaryServerInterceptor("room-service"))) roomServer := grpcserver.NewServer(svc, grpcserver.WithOwnerForwarding(cfg.NodeID, directory, directory, cfg.OwnerForwardTimeout)) // 命令服务处理房间状态变更,守卫服务给腾讯云 IM 回调或 gateway 做发言和进群校验。 roomv1.RegisterRoomCommandServiceServer(server, roomServer) roomv1.RegisterRoomGuardServiceServer(server, roomServer) roomv1.RegisterRoomQueryServiceServer(server, roomServer) healthState := healthcheck.NewState(cfg.NodeID, svc, repository, directory) healthgrpc.RegisterHealthServer(server, grpchealth.NewServer(healthState)) healthHTTP, err := healthhttp.New(cfg.HealthHTTPAddr, cfg.NodeID, healthState) if err != nil { closeClientConn(activityConn) _ = walletConn.Close() _ = repository.Close() _ = redisClient.Close() return nil, err } // listener 在 New 阶段创建,可以把端口占用作为启动失败暴露。 listener, err := net.Listen("tcp", cfg.GRPCAddr) if err != nil { _ = healthHTTP.Close(context.Background()) closeClientConn(activityConn) _ = walletConn.Close() _ = repository.Close() _ = redisClient.Close() return nil, err } workerCtx, workerCancel := context.WithCancel(context.Background()) return &App{ cfg: cfg, grpcServer: server, service: svc, listener: listener, walletConn: walletConn, activityConn: activityConn, repository: repository, redisClose: redisClient.Close, nodeRegistry: directory, health: healthState, healthHTTP: healthHTTP, mqProducers: mqProducers, mqConsumers: mqConsumers, workerCtx: workerCtx, workerCancel: workerCancel, }, nil } // Run 启动 gRPC 服务。 func (a *App) Run() error { a.runHealthHTTP() // health 先标记 serving,再进入 Serve,ready 才能在监听开始后通过。 a.health.MarkGRPCServing() if err := a.registerRoomNode(context.Background()); err != nil { a.health.MarkGRPCStopped() return err } if err := a.startMQ(); err != nil { a.health.MarkGRPCStopped() return err } defer func() { if a.workerCancel != nil { a.workerCancel() } a.workerWG.Wait() a.shutdownMQ() }() a.workerWG.Go(func() { a.runNodeRegistryHeartbeat(a.workerCtx) }) if a.cfg.PresenceStaleScanInterval > 0 { // presence worker 只清理本节点已装载 Cell,命令仍走 Room Cell 持久化链路。 a.workerWG.Go(func() { a.service.RunPresenceStaleWorker(a.workerCtx, a.cfg.PresenceStaleScanInterval) }) } if a.cfg.MicPublishScanInterval > 0 { // MicUp 只代表业务占麦;后台 worker 负责释放未确认 RTC 发流的 pending_publish 麦位。 a.workerWG.Go(func() { a.service.RunMicPublishTimeoutWorker(a.workerCtx, a.cfg.MicPublishScanInterval) }) } if a.cfg.RoomRocketLaunchScanInterval > 0 { // 火箭发射仍通过 Room Cell 命令链路提交,worker 只负责触发到点的系统命令。 a.workerWG.Go(func() { a.service.RunRoomRocketLaunchWorker(a.workerCtx, a.cfg.RoomRocketLaunchScanInterval) }) } if a.cfg.OutboxWorker.Enabled { a.workerWG.Go(func() { a.service.RunOutboxWorker(a.workerCtx, roomservice.OutboxWorkerOptions{ PollInterval: a.cfg.OutboxWorker.PollInterval, BatchSize: a.cfg.OutboxWorker.BatchSize, PublishTimeout: a.cfg.OutboxWorker.PublishTimeout, RetryStrategy: a.cfg.OutboxWorker.RetryStrategy, MaxRetryCount: a.cfg.OutboxWorker.MaxRetryCount, InitialBackoff: a.cfg.OutboxWorker.InitialBackoff, MaxBackoff: a.cfg.OutboxWorker.MaxBackoff, }) }) } err := a.grpcServer.Serve(a.listener) a.health.MarkGRPCStopped() if errors.Is(err, grpc.ErrServerStopped) { // GracefulStop 触发的退出不是故障。 return nil } return err } // Close 关闭底层 gRPC 服务。 func (a *App) Close() { a.closeOnce.Do(func() { // 先进入 draining,避免下线中的节点继续接管房间 lease 或接收新命令。 a.health.MarkDraining() if a.service != nil { a.service.MarkDraining() } if a.workerCancel != nil { // 先停后台 worker 并等待当前投递结果落库,再关闭 gRPC/MySQL/Redis。 a.workerCancel() } a.workerWG.Wait() a.shutdownMQ() // GracefulStop 让已进入的 gRPC 请求自然结束。 grpcshutdown.GracefulStop(a.grpcServer, 15*time.Second) if a.service != nil { // gRPC drain 完成后释放本节点持有的 room route,降低滚动发布时等待 Redis TTL 的窗口。 releaseCtx, releaseCancel := context.WithTimeout(context.Background(), 5*time.Second) a.service.ReleaseLoadedRoomLeases(releaseCtx) releaseCancel() } a.unregisterRoomNode() a.closeHealthHTTP() if a.walletConn != nil { _ = a.walletConn.Close() } if a.activityConn != nil { _ = a.activityConn.Close() } if a.repository != nil { // MySQL 连接池最后释放,保证 GracefulStop 期间仍可完成持久化。 _ = a.repository.Close() } if a.redisClose != nil { // Redis client 关闭后本节点不再续约或接管房间。 _ = a.redisClose() } }) } func (a *App) registerRoomNode(ctx context.Context) error { if a.nodeRegistry == nil { return nil } if _, ok := ctx.Deadline(); !ok { var cancel context.CancelFunc ctx, cancel = context.WithTimeout(ctx, 2*time.Second) defer cancel() } return a.nodeRegistry.RegisterNode(ctx, router.NodeRegistration{ NodeID: a.cfg.NodeID, GRPCAddr: a.cfg.AdvertiseAddr, }, a.cfg.NodeRegistryTTL) } func (a *App) runNodeRegistryHeartbeat(ctx context.Context) { if a.nodeRegistry == nil { return } ticker := time.NewTicker(a.cfg.NodeRegistryHeartbeatInterval) defer ticker.Stop() for { select { case <-ctx.Done(): return case <-ticker.C: if err := a.registerRoomNode(ctx); err != nil { logx.Warn(ctx, "room_node_register_failed", slog.String("node_id", a.cfg.NodeID), slog.String("advertise_addr", a.cfg.AdvertiseAddr), slog.String("error", err.Error()), ) } } } } func (a *App) unregisterRoomNode() { if a.nodeRegistry == nil { return } ctx, cancel := context.WithTimeout(context.Background(), 2*time.Second) defer cancel() if err := a.nodeRegistry.UnregisterNode(ctx, a.cfg.NodeID); err != nil { logx.Warn(ctx, "room_node_unregister_failed", slog.String("node_id", a.cfg.NodeID), slog.String("error", err.Error()), ) } } func (a *App) runHealthHTTP() { if a.healthHTTP == nil { return } go func() { if err := a.healthHTTP.Run(); err != nil { logx.Error(context.Background(), "health_http_run_failed", err) } }() } func (a *App) closeHealthHTTP() { if a.healthHTTP == nil { return } ctx, cancel := context.WithTimeout(context.Background(), 2*time.Second) defer cancel() _ = a.healthHTTP.Close(ctx) } func (a *App) startMQ() error { for _, producer := range a.mqProducers { if err := producer.Start(); err != nil { a.shutdownMQ() return err } } for _, consumer := range a.mqConsumers { if err := consumer.Start(); err != nil { a.shutdownMQ() return err } } return nil } func (a *App) shutdownMQ() { for _, consumer := range a.mqConsumers { if err := consumer.Shutdown(); err != nil { logx.Warn(context.Background(), "rocketmq_consumer_shutdown_failed", slog.String("error", err.Error())) } } for _, producer := range a.mqProducers { if err := producer.Shutdown(); err != nil { logx.Warn(context.Background(), "rocketmq_producer_shutdown_failed", slog.String("error", err.Error())) } } } func rocketMQProducerConfig(cfg config.RocketMQConfig, groupName string) rocketmqx.ProducerConfig { return rocketmqx.ProducerConfig{ EndpointConfig: rocketMQEndpointConfig(cfg), GroupName: groupName, SendTimeout: cfg.SendTimeout, Retry: cfg.Retry, } } func rocketMQConsumerConfig(cfg config.RocketMQConfig, groupName string, maxReconsumeTimes int32) rocketmqx.ConsumerConfig { return rocketmqx.ConsumerConfig{ EndpointConfig: rocketMQEndpointConfig(cfg), GroupName: groupName, MaxReconsumeTimes: maxReconsumeTimes, ConsumeRetryDelay: time.Second, ConsumePullBatch: 32, ConsumePullTimeout: 15 * time.Minute, } } func rocketMQEndpointConfig(cfg config.RocketMQConfig) rocketmqx.EndpointConfig { return rocketmqx.EndpointConfig{ NameServers: cfg.NameServers, NameServerDomain: cfg.NameServerDomain, AccessKey: cfg.AccessKey, SecretKey: cfg.SecretKey, SecurityToken: cfg.SecurityToken, Namespace: cfg.Namespace, VIPChannel: cfg.VIPChannel, } } func closeClientConn(conn *grpc.ClientConn) { if conn != nil { _ = conn.Close() } }