02117bca4c
机器人房写命令间隔常大于 lease TTL(10s)，双节点 orchestrator 无差别 起全量 runtime 导致互相抢占 ownership，跨节点命令全天 CONFLICT 刷屏， 礼物恢复 worker 在非 owner 节点把 pending saga 误标 compensated (~200/h)。 - startRobotRoomRuntime 启动前经 EnsureOwner 认领 lease，非 owner 跳过 - 每个 runtime 增加 lease keeper 按 TTL/3 主动续租，发现执行权被接管立即停止本地循环 - startActiveRobotRooms 改为 reconcile：lease 迁走或配置停用的房间停掉本地残留 runtime - 礼物恢复 worker 先查房间归属，归他人时释放 claim 让 owner 节点结算 - 跨节点 ownership 冲突不再进入 compensated 终态，改走 retry（pending 不保证钱包未扣款） Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>
358 lines
14 KiB
Go
358 lines
14 KiB
Go
package service
|
||
|
||
import (
|
||
"bytes"
|
||
"context"
|
||
"errors"
|
||
"fmt"
|
||
"log/slog"
|
||
"strings"
|
||
"sync"
|
||
"time"
|
||
|
||
"google.golang.org/grpc/codes"
|
||
"google.golang.org/grpc/status"
|
||
"google.golang.org/protobuf/proto"
|
||
roomv1 "hyapp.local/api/proto/room/v1"
|
||
"hyapp/pkg/appcode"
|
||
"hyapp/pkg/xerr"
|
||
"hyapp/services/room-service/internal/room/command"
|
||
)
|
||
|
||
const (
|
||
giftOperationPersistTimeout = 2 * time.Second
|
||
giftOperationRetryDelay = time.Second
|
||
// 每条恢复任务从进入 wallet/lucky/Room Cell 到返回都受独立 deadline 约束；不能复用 worker 的长生命周期 ctx。
|
||
giftOperationAttemptTimeout = 8 * time.Second
|
||
// claim 必须覆盖尝试窗口、Room Cell 已扣费后的受保护提交和失败状态落盘，并保留抖动余量。
|
||
giftOperationClaimTTL = 20 * time.Second
|
||
)
|
||
|
||
func (f *giftFlow) loadGiftOperation() error {
|
||
operation, exists, err := f.s.repository.GetGiftOperation(f.ctx, f.cmd.RoomID(), f.cmd.ID())
|
||
if err != nil || !exists {
|
||
return err
|
||
}
|
||
currentPayload, err := command.IdempotencyPayloadForCommand(f.cmd)
|
||
if err != nil {
|
||
return err
|
||
}
|
||
storedPayload, err := command.IdempotencyPayload(operation.CommandPayload)
|
||
if err != nil {
|
||
return err
|
||
}
|
||
if !bytes.Equal(currentPayload, storedPayload) {
|
||
return xerr.New(xerr.Conflict, "command_id payload conflict")
|
||
}
|
||
if operation.Status == GiftOperationStatusCommitted {
|
||
// committed 与 command log 在同一 MySQL 事务写入；只出现一边说明事实被人工破坏，不能再调用钱包重建。
|
||
if _, exists, err := f.s.repository.GetCommand(f.ctx, f.cmd.RoomID(), f.cmd.ID()); err != nil {
|
||
return err
|
||
} else if !exists {
|
||
return xerr.New(xerr.Internal, "committed gift operation is missing command result")
|
||
}
|
||
return nil
|
||
}
|
||
if operation.Status != GiftOperationStatusPending && operation.Status != GiftOperationStatusDebited && operation.Status != GiftOperationStatusDrawn {
|
||
return nil
|
||
}
|
||
return f.resumeGiftOperation(operation)
|
||
}
|
||
|
||
func (f *giftFlow) resumeGiftOperation(operation GiftOperation) error {
|
||
var storedRequest roomv1.SendGiftRequest
|
||
if err := proto.Unmarshal(operation.RequestPayload, &storedRequest); err != nil {
|
||
return xerr.New(xerr.Internal, "stored send gift request cannot be decoded")
|
||
}
|
||
storedCommand, err := command.Deserialize((command.SendGift{}).Type(), operation.CommandPayload)
|
||
if err != nil {
|
||
return xerr.New(xerr.Internal, "stored send gift command cannot be decoded")
|
||
}
|
||
giftCommand, ok := storedCommand.(*command.SendGift)
|
||
if !ok || giftCommand == nil {
|
||
return xerr.New(xerr.Internal, "stored send gift command type is invalid")
|
||
}
|
||
f.req = &storedRequest
|
||
f.cmd = *giftCommand
|
||
f.settledCommand = *giftCommand
|
||
f.operation = operation
|
||
f.operationStarted = true
|
||
f.recovering = true
|
||
f.debitCompleted = operation.Status == GiftOperationStatusDebited || operation.Status == GiftOperationStatusDrawn
|
||
f.drawCompleted = operation.Status == GiftOperationStatusDrawn
|
||
return nil
|
||
}
|
||
|
||
func (f *giftFlow) ensureGiftOperation(now time.Time) error {
|
||
if f.operationStarted {
|
||
return nil
|
||
}
|
||
commandPayload, err := command.Serialize(f.cmd)
|
||
if err != nil {
|
||
return err
|
||
}
|
||
request := proto.Clone(f.req).(*roomv1.SendGiftRequest)
|
||
// batch 首次校验可能裁掉已离房目标；恢复只能结算首次确认的实际目标，不能因之后 presence 变化重新扩张或缩小账单。
|
||
request.TargetUserIds = append([]int64(nil), f.cmd.TargetUserIDs...)
|
||
request.TargetUserId = f.cmd.TargetUserID
|
||
request.TargetHostScopes = filterGiftTargetHostScopesProto(request.GetTargetHostScopes(), f.cmd.TargetUserIDs)
|
||
request.TargetDisplayProfiles = filterGiftDisplayProfilesProto(request.GetTargetDisplayProfiles(), f.cmd.TargetUserIDs)
|
||
requestPayload, err := proto.Marshal(request)
|
||
if err != nil {
|
||
return xerr.New(xerr.Internal, "send gift recovery request cannot be encoded")
|
||
}
|
||
operation, created, err := f.s.repository.EnsureGiftOperation(f.ctx, GiftOperation{
|
||
AppCode: appcode.FromContext(f.ctx),
|
||
RoomID: f.cmd.RoomID(),
|
||
CommandID: f.cmd.ID(),
|
||
ActorUserID: f.cmd.ActorUserID(),
|
||
CommandPayload: commandPayload,
|
||
RequestPayload: requestPayload,
|
||
Status: GiftOperationStatusPending,
|
||
CreatedAtMS: now.UnixMilli(),
|
||
UpdatedAtMS: now.UnixMilli(),
|
||
})
|
||
if err != nil {
|
||
return err
|
||
}
|
||
storedIDPayload, err := command.IdempotencyPayload(operation.CommandPayload)
|
||
if err != nil {
|
||
return err
|
||
}
|
||
currentIDPayload, err := command.IdempotencyPayload(commandPayload)
|
||
if err != nil {
|
||
return err
|
||
}
|
||
if !bytes.Equal(storedIDPayload, currentIDPayload) {
|
||
return xerr.New(xerr.Conflict, "command_id payload conflict")
|
||
}
|
||
if operation.Status == GiftOperationStatusCompensated {
|
||
// Reactivation is deliberately after the normalized payload comparison above. Reversing this
|
||
// order would let a conflicting request expose the old stored request to the background worker,
|
||
// which could turn an HTTP Conflict into an unrelated wallet debit.
|
||
operation, _, err = f.s.repository.ReactivateCompensatedGiftOperation(f.ctx, f.cmd.RoomID(), f.cmd.ID(), now.UnixMilli())
|
||
if err != nil {
|
||
return err
|
||
}
|
||
if operation.Status == GiftOperationStatusCompensated {
|
||
return xerr.New(xerr.Unavailable, "gift operation reactivation did not advance")
|
||
}
|
||
}
|
||
if !created {
|
||
// loadGiftOperation executes before entering the Cell and can legitimately miss a row created by
|
||
// an overlapping request. Once Ensure returns that row, all financial calls must use its exact
|
||
// stored request/command snapshots. The idempotency comparison intentionally ignores gateway-
|
||
// resolved region/host/display fields, but wallet request_hash does not; continuing with this
|
||
// caller's refreshed snapshots would turn a valid retry into a ledger conflict.
|
||
return f.resumeGiftOperation(operation)
|
||
}
|
||
f.operation = operation
|
||
f.operationStarted = true
|
||
return nil
|
||
}
|
||
|
||
func (f *giftFlow) markGiftOperationDebited(now time.Time) error {
|
||
f.debitCompleted = true
|
||
billingPayload, err := proto.Marshal(f.billing)
|
||
if err != nil {
|
||
return xerr.New(xerr.Internal, "gift billing result cannot be encoded")
|
||
}
|
||
if err := f.s.repository.SaveGiftOperationProgress(f.ctx, GiftOperationProgress{
|
||
RoomID: f.cmd.RoomID(),
|
||
CommandID: f.cmd.ID(),
|
||
Status: GiftOperationStatusDebited,
|
||
BillingPayload: billingPayload,
|
||
UpdatedAtMS: now.UnixMilli(),
|
||
}); err != nil {
|
||
return err
|
||
}
|
||
f.operation.Status = GiftOperationStatusDebited
|
||
f.operation.BillingPayload = billingPayload
|
||
return nil
|
||
}
|
||
|
||
func (f *giftFlow) markGiftOperationDrawn(now time.Time) error {
|
||
f.drawCompleted = true
|
||
luckyPayload, err := proto.Marshal(&roomv1.SendGiftResultV2{
|
||
ApiVersion: 2,
|
||
CommandId: f.cmd.ID(),
|
||
LuckyGift: f.luckyGift,
|
||
LuckyGifts: append([]*roomv1.LuckyGiftDrawResult(nil), f.luckyGifts...),
|
||
})
|
||
if err != nil {
|
||
return xerr.New(xerr.Internal, "gift lucky result cannot be encoded")
|
||
}
|
||
if err := f.s.repository.SaveGiftOperationProgress(f.ctx, GiftOperationProgress{
|
||
RoomID: f.cmd.RoomID(),
|
||
CommandID: f.cmd.ID(),
|
||
Status: GiftOperationStatusDrawn,
|
||
LuckyPayload: luckyPayload,
|
||
UpdatedAtMS: now.UnixMilli(),
|
||
}); err != nil {
|
||
return err
|
||
}
|
||
f.operation.Status = GiftOperationStatusDrawn
|
||
f.operation.LuckyPayload = luckyPayload
|
||
return nil
|
||
}
|
||
|
||
func (f *giftFlow) handleGiftOperationFailure(err error) {
|
||
if !f.operationStarted || err == nil {
|
||
return
|
||
}
|
||
ctx, cancel := context.WithTimeout(context.WithoutCancel(f.ctx), giftOperationPersistTimeout)
|
||
defer cancel()
|
||
now := f.s.clock.Now()
|
||
if !f.debitCompleted && !giftOperationErrorRetryable(err) && !isRoomOwnedByOtherNodeError(err) {
|
||
// wallet 明确拒绝（余额不足、配置非法等）保证没有扣款，pending 可以安全关闭；同载荷的用户主动重试会重新激活。
|
||
// 跨节点执行权冲突不在此列：它只说明本节点不是 owner，首次尝试可能已经扣款，必须留给 owner 节点重试发现回执。
|
||
if persistErr := f.s.repository.MarkGiftOperationCompensated(ctx, f.cmd.RoomID(), f.cmd.ID(), err.Error(), now.UnixMilli()); persistErr != nil {
|
||
slog.WarnContext(ctx, "mark gift operation compensated failed", "room_id", f.cmd.RoomID(), "command_id", f.cmd.ID(), "error", persistErr)
|
||
}
|
||
return
|
||
}
|
||
nextRetryAtMS := now.Add(giftOperationRetryDelay).UnixMilli()
|
||
preserveLease := giftOperationFailureMustKeepClaim(f.recovering, err)
|
||
if persistErr := f.s.repository.MarkGiftOperationRetry(ctx, f.cmd.RoomID(), f.cmd.ID(), err.Error(), nextRetryAtMS, now.UnixMilli(), preserveLease); persistErr != nil {
|
||
slog.WarnContext(ctx, "mark gift operation retry failed", "room_id", f.cmd.RoomID(), "command_id", f.cmd.ID(), "error", persistErr)
|
||
}
|
||
}
|
||
|
||
func giftOperationFailureMustKeepClaim(recovering bool, err error) bool {
|
||
if !recovering || err == nil {
|
||
return false
|
||
}
|
||
if errors.Is(err, context.DeadlineExceeded) || errors.Is(err, context.Canceled) {
|
||
return true
|
||
}
|
||
switch status.Code(err) {
|
||
case codes.DeadlineExceeded, codes.Canceled:
|
||
return true
|
||
default:
|
||
return false
|
||
}
|
||
}
|
||
|
||
func giftOperationErrorRetryable(err error) bool {
|
||
code := xerr.CodeOf(err)
|
||
if _, ok := status.FromError(err); ok {
|
||
code = xerr.ReasonFromGRPC(err)
|
||
}
|
||
if code == xerr.Internal || code == xerr.Unavailable {
|
||
// unknown/timeout may have reached wallet; retrying the same command_id is the only safe way to discover the stable receipt.
|
||
return true
|
||
}
|
||
return xerr.SpecOf(code).Retryable
|
||
}
|
||
|
||
func (s *Service) ensureNoRecoverableGiftOperations(ctx context.Context, roomID string) error {
|
||
open, err := s.repository.HasRecoverableGiftOperations(ctx, roomID)
|
||
if err != nil {
|
||
return err
|
||
}
|
||
if open {
|
||
// 关闭/删除会让恢复请求无法通过房间生命周期校验，并且硬删除还会抹掉 saga；Room Cell 串行保证
|
||
// 本检查与 SendGift 创建 operation 不会交错，因此明确阻止生命周期终态直到 worker 完成结算。
|
||
return xerr.New(xerr.Conflict, "room has recovering gift operations")
|
||
}
|
||
return nil
|
||
}
|
||
|
||
func filterGiftTargetHostScopesProto(scopes []*roomv1.SendGiftTargetHostScope, targets []int64) []*roomv1.SendGiftTargetHostScope {
|
||
allowed := positiveInt64Set(targets)
|
||
out := make([]*roomv1.SendGiftTargetHostScope, 0, len(scopes))
|
||
for _, scope := range scopes {
|
||
if scope != nil && allowed[scope.GetTargetUserId()] {
|
||
out = append(out, proto.Clone(scope).(*roomv1.SendGiftTargetHostScope))
|
||
}
|
||
}
|
||
return out
|
||
}
|
||
|
||
func filterGiftDisplayProfilesProto(profiles []*roomv1.SendGiftDisplayProfile, targets []int64) []*roomv1.SendGiftDisplayProfile {
|
||
allowed := positiveInt64Set(targets)
|
||
out := make([]*roomv1.SendGiftDisplayProfile, 0, len(profiles))
|
||
for _, profile := range profiles {
|
||
if profile != nil && allowed[profile.GetUserId()] {
|
||
out = append(out, proto.Clone(profile).(*roomv1.SendGiftDisplayProfile))
|
||
}
|
||
}
|
||
return out
|
||
}
|
||
|
||
func positiveInt64Set(values []int64) map[int64]bool {
|
||
out := make(map[int64]bool, len(values))
|
||
for _, value := range values {
|
||
if value > 0 {
|
||
out[value] = true
|
||
}
|
||
}
|
||
return out
|
||
}
|
||
|
||
// RunGiftOperationRecoveryWorker 持续完成可能已经扣款的 saga；它只复用稳定 command_id 调 owner 接口，不自行改房间状态。
|
||
func (s *Service) RunGiftOperationRecoveryWorker(ctx context.Context, interval time.Duration) {
|
||
if interval <= 0 {
|
||
interval = time.Second
|
||
}
|
||
workerID := fmt.Sprintf("%s-gift-recovery", strings.TrimSpace(s.nodeID))
|
||
ticker := time.NewTicker(interval)
|
||
defer ticker.Stop()
|
||
for {
|
||
s.recoverGiftOperationsOnce(ctx, workerID)
|
||
select {
|
||
case <-ctx.Done():
|
||
return
|
||
case <-ticker.C:
|
||
}
|
||
}
|
||
}
|
||
|
||
func (s *Service) recoverGiftOperationsOnce(ctx context.Context, workerID string) {
|
||
now := s.clock.Now()
|
||
operations, err := s.repository.ClaimRecoverableGiftOperations(ctx, workerID, 20, now.UnixMilli(), now.Add(giftOperationClaimTTL).UnixMilli())
|
||
if err != nil {
|
||
slog.WarnContext(ctx, "claim gift operation recovery failed", "error", err)
|
||
return
|
||
}
|
||
// 被同一批 claim 的任务必须立即开始计时；若继续串行执行，队尾可能尚未进入 Room Cell 就超过租约，
|
||
// 被其他实例重复领取。不同房间天然可并行，同房间仍由 Room Cell 串行且等待受各自 deadline 限制。
|
||
var wait sync.WaitGroup
|
||
for _, operation := range operations {
|
||
operation := operation
|
||
wait.Add(1)
|
||
go func() {
|
||
defer wait.Done()
|
||
s.recoverGiftOperation(ctx, operation)
|
||
}()
|
||
}
|
||
wait.Wait()
|
||
}
|
||
|
||
func (s *Service) recoverGiftOperation(workerCtx context.Context, operation GiftOperation) {
|
||
baseCtx := appcode.WithContext(workerCtx, operation.AppCode)
|
||
operationCtx, cancel := context.WithTimeout(baseCtx, giftOperationAttemptTimeout)
|
||
defer cancel()
|
||
|
||
now := s.clock.Now()
|
||
if ownerNodeID, ownedElsewhere, err := s.roomLeaseOwnedElsewhere(operationCtx, operation.AppCode, operation.RoomID); err == nil && ownedElsewhere {
|
||
// 非 owner 节点执行恢复只会在 ensureCell 得到跨节点 CONFLICT；释放 claim 让 owner 节点的
|
||
// worker 在下一轮完成结算，避免恢复任务在错误节点空转。目录查询失败时按本地可执行处理，
|
||
// ensureCell 会再做权威判定。
|
||
if persistErr := s.repository.MarkGiftOperationRetry(operationCtx, operation.RoomID, operation.CommandID, roomOwnedByOtherNodeMsgPrefix+ownerNodeID, now.Add(giftOperationRetryDelay).UnixMilli(), now.UnixMilli(), false); persistErr != nil {
|
||
slog.WarnContext(operationCtx, "defer gift operation to owner failed", "room_id", operation.RoomID, "command_id", operation.CommandID, "error", persistErr)
|
||
}
|
||
return
|
||
}
|
||
var req roomv1.SendGiftRequest
|
||
if err := proto.Unmarshal(operation.RequestPayload, &req); err != nil {
|
||
_ = s.repository.MarkGiftOperationRetry(operationCtx, operation.RoomID, operation.CommandID, "stored request decode failed", now.Add(time.Minute).UnixMilli(), now.UnixMilli(), false)
|
||
return
|
||
}
|
||
options := giftSendOptions{BatchDisplay: strings.EqualFold(strings.TrimSpace(req.GetDisplayMode()), "batch")}
|
||
if _, err := s.sendGiftWithRecovery(operationCtx, &req, options, operation); err != nil {
|
||
// sendGiftWithRecovery 用同一 command_id 记录 retry；deadline 后下一轮只发现/重放首次 wallet/lucky 结果，绝不新建账单。
|
||
slog.WarnContext(operationCtx, "recover gift operation failed", "room_id", operation.RoomID, "command_id", operation.CommandID, "status", operation.Status, "error", err)
|
||
}
|
||
}
|