package service import ( "bytes" "context" "errors" "fmt" "log/slog" "strings" "sync" "time" "google.golang.org/grpc/codes" "google.golang.org/grpc/status" "google.golang.org/protobuf/proto" roomv1 "hyapp.local/api/proto/room/v1" "hyapp/pkg/appcode" "hyapp/pkg/xerr" "hyapp/services/room-service/internal/room/command" ) const ( giftOperationPersistTimeout = 2 * time.Second giftOperationRetryDelay = time.Second // 每条恢复任务从进入 wallet/lucky/Room Cell 到返回都受独立 deadline 约束;不能复用 worker 的长生命周期 ctx。 giftOperationAttemptTimeout = 8 * time.Second // claim 必须覆盖尝试窗口、Room Cell 已扣费后的受保护提交和失败状态落盘,并保留抖动余量。 giftOperationClaimTTL = 20 * time.Second ) func (f *giftFlow) loadGiftOperation() error { operation, exists, err := f.s.repository.GetGiftOperation(f.ctx, f.cmd.RoomID(), f.cmd.ID()) if err != nil || !exists { return err } currentPayload, err := command.IdempotencyPayloadForCommand(f.cmd) if err != nil { return err } storedPayload, err := command.IdempotencyPayload(operation.CommandPayload) if err != nil { return err } if !bytes.Equal(currentPayload, storedPayload) { return xerr.New(xerr.Conflict, "command_id payload conflict") } if operation.Status == GiftOperationStatusCommitted { // committed 与 command log 在同一 MySQL 事务写入;只出现一边说明事实被人工破坏,不能再调用钱包重建。 if _, exists, err := f.s.repository.GetCommand(f.ctx, f.cmd.RoomID(), f.cmd.ID()); err != nil { return err } else if !exists { return xerr.New(xerr.Internal, "committed gift operation is missing command result") } return nil } if operation.Status != GiftOperationStatusPending && operation.Status != GiftOperationStatusDebited && operation.Status != GiftOperationStatusDrawn { return nil } return f.resumeGiftOperation(operation) } func (f *giftFlow) resumeGiftOperation(operation GiftOperation) error { var storedRequest roomv1.SendGiftRequest if err := proto.Unmarshal(operation.RequestPayload, &storedRequest); err != nil { return xerr.New(xerr.Internal, "stored send gift request cannot be decoded") } storedCommand, err := command.Deserialize((command.SendGift{}).Type(), operation.CommandPayload) if err != nil { return xerr.New(xerr.Internal, "stored send gift command cannot be decoded") } giftCommand, ok := storedCommand.(*command.SendGift) if !ok || giftCommand == nil { return xerr.New(xerr.Internal, "stored send gift command type is invalid") } f.req = &storedRequest f.cmd = *giftCommand f.settledCommand = *giftCommand f.operation = operation f.operationStarted = true f.recovering = true f.debitCompleted = operation.Status == GiftOperationStatusDebited || operation.Status == GiftOperationStatusDrawn f.drawCompleted = operation.Status == GiftOperationStatusDrawn return nil } func (f *giftFlow) ensureGiftOperation(now time.Time) error { if f.operationStarted { return nil } commandPayload, err := command.Serialize(f.cmd) if err != nil { return err } request := proto.Clone(f.req).(*roomv1.SendGiftRequest) // batch 首次校验可能裁掉已离房目标;恢复只能结算首次确认的实际目标,不能因之后 presence 变化重新扩张或缩小账单。 request.TargetUserIds = append([]int64(nil), f.cmd.TargetUserIDs...) request.TargetUserId = f.cmd.TargetUserID request.TargetHostScopes = filterGiftTargetHostScopesProto(request.GetTargetHostScopes(), f.cmd.TargetUserIDs) request.TargetDisplayProfiles = filterGiftDisplayProfilesProto(request.GetTargetDisplayProfiles(), f.cmd.TargetUserIDs) requestPayload, err := proto.Marshal(request) if err != nil { return xerr.New(xerr.Internal, "send gift recovery request cannot be encoded") } operation, created, err := f.s.repository.EnsureGiftOperation(f.ctx, GiftOperation{ AppCode: appcode.FromContext(f.ctx), RoomID: f.cmd.RoomID(), CommandID: f.cmd.ID(), ActorUserID: f.cmd.ActorUserID(), CommandPayload: commandPayload, RequestPayload: requestPayload, Status: GiftOperationStatusPending, CreatedAtMS: now.UnixMilli(), UpdatedAtMS: now.UnixMilli(), }) if err != nil { return err } storedIDPayload, err := command.IdempotencyPayload(operation.CommandPayload) if err != nil { return err } currentIDPayload, err := command.IdempotencyPayload(commandPayload) if err != nil { return err } if !bytes.Equal(storedIDPayload, currentIDPayload) { return xerr.New(xerr.Conflict, "command_id payload conflict") } if operation.Status == GiftOperationStatusCompensated { // Reactivation is deliberately after the normalized payload comparison above. Reversing this // order would let a conflicting request expose the old stored request to the background worker, // which could turn an HTTP Conflict into an unrelated wallet debit. operation, _, err = f.s.repository.ReactivateCompensatedGiftOperation(f.ctx, f.cmd.RoomID(), f.cmd.ID(), now.UnixMilli()) if err != nil { return err } if operation.Status == GiftOperationStatusCompensated { return xerr.New(xerr.Unavailable, "gift operation reactivation did not advance") } } if !created { // loadGiftOperation executes before entering the Cell and can legitimately miss a row created by // an overlapping request. Once Ensure returns that row, all financial calls must use its exact // stored request/command snapshots. The idempotency comparison intentionally ignores gateway- // resolved region/host/display fields, but wallet request_hash does not; continuing with this // caller's refreshed snapshots would turn a valid retry into a ledger conflict. return f.resumeGiftOperation(operation) } f.operation = operation f.operationStarted = true return nil } func (f *giftFlow) markGiftOperationDebited(now time.Time) error { f.debitCompleted = true billingPayload, err := proto.Marshal(f.billing) if err != nil { return xerr.New(xerr.Internal, "gift billing result cannot be encoded") } if err := f.s.repository.SaveGiftOperationProgress(f.ctx, GiftOperationProgress{ RoomID: f.cmd.RoomID(), CommandID: f.cmd.ID(), Status: GiftOperationStatusDebited, BillingPayload: billingPayload, UpdatedAtMS: now.UnixMilli(), }); err != nil { return err } f.operation.Status = GiftOperationStatusDebited f.operation.BillingPayload = billingPayload return nil } func (f *giftFlow) markGiftOperationDrawn(now time.Time) error { f.drawCompleted = true luckyPayload, err := proto.Marshal(&roomv1.SendGiftResultV2{ ApiVersion: 2, CommandId: f.cmd.ID(), LuckyGift: f.luckyGift, LuckyGifts: append([]*roomv1.LuckyGiftDrawResult(nil), f.luckyGifts...), }) if err != nil { return xerr.New(xerr.Internal, "gift lucky result cannot be encoded") } if err := f.s.repository.SaveGiftOperationProgress(f.ctx, GiftOperationProgress{ RoomID: f.cmd.RoomID(), CommandID: f.cmd.ID(), Status: GiftOperationStatusDrawn, LuckyPayload: luckyPayload, UpdatedAtMS: now.UnixMilli(), }); err != nil { return err } f.operation.Status = GiftOperationStatusDrawn f.operation.LuckyPayload = luckyPayload return nil } func (f *giftFlow) handleGiftOperationFailure(err error) { if !f.operationStarted || err == nil { return } ctx, cancel := context.WithTimeout(context.WithoutCancel(f.ctx), giftOperationPersistTimeout) defer cancel() now := f.s.clock.Now() if !f.debitCompleted && !giftOperationErrorRetryable(err) && !isRoomOwnedByOtherNodeError(err) { // wallet 明确拒绝(余额不足、配置非法等)保证没有扣款,pending 可以安全关闭;同载荷的用户主动重试会重新激活。 // 跨节点执行权冲突不在此列:它只说明本节点不是 owner,首次尝试可能已经扣款,必须留给 owner 节点重试发现回执。 if persistErr := f.s.repository.MarkGiftOperationCompensated(ctx, f.cmd.RoomID(), f.cmd.ID(), err.Error(), now.UnixMilli()); persistErr != nil { slog.WarnContext(ctx, "mark gift operation compensated failed", "room_id", f.cmd.RoomID(), "command_id", f.cmd.ID(), "error", persistErr) } return } nextRetryAtMS := now.Add(giftOperationRetryDelay).UnixMilli() preserveLease := giftOperationFailureMustKeepClaim(f.recovering, err) if persistErr := f.s.repository.MarkGiftOperationRetry(ctx, f.cmd.RoomID(), f.cmd.ID(), err.Error(), nextRetryAtMS, now.UnixMilli(), preserveLease); persistErr != nil { slog.WarnContext(ctx, "mark gift operation retry failed", "room_id", f.cmd.RoomID(), "command_id", f.cmd.ID(), "error", persistErr) } } func giftOperationFailureMustKeepClaim(recovering bool, err error) bool { if !recovering || err == nil { return false } if errors.Is(err, context.DeadlineExceeded) || errors.Is(err, context.Canceled) { return true } switch status.Code(err) { case codes.DeadlineExceeded, codes.Canceled: return true default: return false } } func giftOperationErrorRetryable(err error) bool { code := xerr.CodeOf(err) if _, ok := status.FromError(err); ok { code = xerr.ReasonFromGRPC(err) } if code == xerr.Internal || code == xerr.Unavailable { // unknown/timeout may have reached wallet; retrying the same command_id is the only safe way to discover the stable receipt. return true } return xerr.SpecOf(code).Retryable } func (s *Service) ensureNoRecoverableGiftOperations(ctx context.Context, roomID string) error { open, err := s.repository.HasRecoverableGiftOperations(ctx, roomID) if err != nil { return err } if open { // 关闭/删除会让恢复请求无法通过房间生命周期校验,并且硬删除还会抹掉 saga;Room Cell 串行保证 // 本检查与 SendGift 创建 operation 不会交错,因此明确阻止生命周期终态直到 worker 完成结算。 return xerr.New(xerr.Conflict, "room has recovering gift operations") } return nil } func filterGiftTargetHostScopesProto(scopes []*roomv1.SendGiftTargetHostScope, targets []int64) []*roomv1.SendGiftTargetHostScope { allowed := positiveInt64Set(targets) out := make([]*roomv1.SendGiftTargetHostScope, 0, len(scopes)) for _, scope := range scopes { if scope != nil && allowed[scope.GetTargetUserId()] { out = append(out, proto.Clone(scope).(*roomv1.SendGiftTargetHostScope)) } } return out } func filterGiftDisplayProfilesProto(profiles []*roomv1.SendGiftDisplayProfile, targets []int64) []*roomv1.SendGiftDisplayProfile { allowed := positiveInt64Set(targets) out := make([]*roomv1.SendGiftDisplayProfile, 0, len(profiles)) for _, profile := range profiles { if profile != nil && allowed[profile.GetUserId()] { out = append(out, proto.Clone(profile).(*roomv1.SendGiftDisplayProfile)) } } return out } func positiveInt64Set(values []int64) map[int64]bool { out := make(map[int64]bool, len(values)) for _, value := range values { if value > 0 { out[value] = true } } return out } // RunGiftOperationRecoveryWorker 持续完成可能已经扣款的 saga;它只复用稳定 command_id 调 owner 接口,不自行改房间状态。 func (s *Service) RunGiftOperationRecoveryWorker(ctx context.Context, interval time.Duration) { if interval <= 0 { interval = time.Second } workerID := fmt.Sprintf("%s-gift-recovery", strings.TrimSpace(s.nodeID)) ticker := time.NewTicker(interval) defer ticker.Stop() for { s.recoverGiftOperationsOnce(ctx, workerID) select { case <-ctx.Done(): return case <-ticker.C: } } } func (s *Service) recoverGiftOperationsOnce(ctx context.Context, workerID string) { now := s.clock.Now() operations, err := s.repository.ClaimRecoverableGiftOperations(ctx, workerID, 20, now.UnixMilli(), now.Add(giftOperationClaimTTL).UnixMilli()) if err != nil { slog.WarnContext(ctx, "claim gift operation recovery failed", "error", err) return } // 被同一批 claim 的任务必须立即开始计时;若继续串行执行,队尾可能尚未进入 Room Cell 就超过租约, // 被其他实例重复领取。不同房间天然可并行,同房间仍由 Room Cell 串行且等待受各自 deadline 限制。 var wait sync.WaitGroup for _, operation := range operations { operation := operation wait.Add(1) go func() { defer wait.Done() s.recoverGiftOperation(ctx, operation) }() } wait.Wait() } func (s *Service) recoverGiftOperation(workerCtx context.Context, operation GiftOperation) { baseCtx := appcode.WithContext(workerCtx, operation.AppCode) operationCtx, cancel := context.WithTimeout(baseCtx, giftOperationAttemptTimeout) defer cancel() now := s.clock.Now() if ownerNodeID, ownedElsewhere, err := s.roomLeaseOwnedElsewhere(operationCtx, operation.AppCode, operation.RoomID); err == nil && ownedElsewhere { // 非 owner 节点执行恢复只会在 ensureCell 得到跨节点 CONFLICT;释放 claim 让 owner 节点的 // worker 在下一轮完成结算,避免恢复任务在错误节点空转。目录查询失败时按本地可执行处理, // ensureCell 会再做权威判定。 if persistErr := s.repository.MarkGiftOperationRetry(operationCtx, operation.RoomID, operation.CommandID, roomOwnedByOtherNodeMsgPrefix+ownerNodeID, now.Add(giftOperationRetryDelay).UnixMilli(), now.UnixMilli(), false); persistErr != nil { slog.WarnContext(operationCtx, "defer gift operation to owner failed", "room_id", operation.RoomID, "command_id", operation.CommandID, "error", persistErr) } return } var req roomv1.SendGiftRequest if err := proto.Unmarshal(operation.RequestPayload, &req); err != nil { _ = s.repository.MarkGiftOperationRetry(operationCtx, operation.RoomID, operation.CommandID, "stored request decode failed", now.Add(time.Minute).UnixMilli(), now.UnixMilli(), false) return } options := giftSendOptions{BatchDisplay: strings.EqualFold(strings.TrimSpace(req.GetDisplayMode()), "batch")} if _, err := s.sendGiftWithRecovery(operationCtx, &req, options, operation); err != nil { // sendGiftWithRecovery 用同一 command_id 记录 retry;deadline 后下一轮只发现/重放首次 wallet/lucky 结果,绝不新建账单。 slog.WarnContext(operationCtx, "recover gift operation failed", "room_id", operation.RoomID, "command_id", operation.CommandID, "status", operation.Status, "error", err) } }