02117bca4c
机器人房写命令间隔常大于 lease TTL(10s)，双节点 orchestrator 无差别 起全量 runtime 导致互相抢占 ownership，跨节点命令全天 CONFLICT 刷屏， 礼物恢复 worker 在非 owner 节点把 pending saga 误标 compensated (~200/h)。 - startRobotRoomRuntime 启动前经 EnsureOwner 认领 lease，非 owner 跳过 - 每个 runtime 增加 lease keeper 按 TTL/3 主动续租，发现执行权被接管立即停止本地循环 - startActiveRobotRooms 改为 reconcile：lease 迁走或配置停用的房间停掉本地残留 runtime - 礼物恢复 worker 先查房间归属，归他人时释放 claim 让 owner 节点结算 - 跨节点 ownership 冲突不再进入 compensated 终态，改走 retry（pending 不保证钱包未扣款） Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>
252 lines
11 KiB
Go
252 lines
11 KiB
Go
package service
|
||
|
||
import (
|
||
"context"
|
||
"errors"
|
||
"testing"
|
||
"time"
|
||
|
||
"google.golang.org/grpc/codes"
|
||
"google.golang.org/grpc/status"
|
||
"google.golang.org/protobuf/proto"
|
||
roomv1 "hyapp.local/api/proto/room/v1"
|
||
"hyapp/pkg/appcode"
|
||
"hyapp/pkg/xerr"
|
||
"hyapp/services/room-service/internal/room/command"
|
||
"hyapp/services/room-service/internal/router"
|
||
)
|
||
|
||
type giftOperationOwnershipRepository struct {
|
||
Repository
|
||
retries []giftOperationRetryCall
|
||
compensated []giftOperationRetryCall
|
||
}
|
||
|
||
type giftOperationRetryCall struct {
|
||
roomID string
|
||
commandID string
|
||
lastError string
|
||
preserveLease bool
|
||
}
|
||
|
||
func (r *giftOperationOwnershipRepository) MarkGiftOperationRetry(_ context.Context, roomID string, commandID string, lastError string, _ int64, _ int64, preserveLease bool) error {
|
||
r.retries = append(r.retries, giftOperationRetryCall{roomID: roomID, commandID: commandID, lastError: lastError, preserveLease: preserveLease})
|
||
return nil
|
||
}
|
||
|
||
func (r *giftOperationOwnershipRepository) MarkGiftOperationCompensated(_ context.Context, roomID string, commandID string, lastError string, _ int64) error {
|
||
r.compensated = append(r.compensated, giftOperationRetryCall{roomID: roomID, commandID: commandID, lastError: lastError})
|
||
return nil
|
||
}
|
||
|
||
type compensatedGiftOperationRepository struct {
|
||
Repository
|
||
operation GiftOperation
|
||
reactivateCalls int
|
||
}
|
||
|
||
func (r *compensatedGiftOperationRepository) EnsureGiftOperation(_ context.Context, _ GiftOperation) (GiftOperation, bool, error) {
|
||
return r.operation, false, nil
|
||
}
|
||
|
||
func (r *compensatedGiftOperationRepository) ReactivateCompensatedGiftOperation(_ context.Context, _, _ string, updatedAtMS int64) (GiftOperation, bool, error) {
|
||
r.reactivateCalls++
|
||
if r.operation.Status == GiftOperationStatusCompensated {
|
||
r.operation.Status = GiftOperationStatusPending
|
||
r.operation.LastError = ""
|
||
r.operation.UpdatedAtMS = updatedAtMS
|
||
return r.operation, true, nil
|
||
}
|
||
return r.operation, false, nil
|
||
}
|
||
|
||
func TestGiftOperationRecoveryAttemptFitsInsideClaimLease(t *testing.T) {
|
||
// mutateRoom 在钱包可能已经扣费后允许一个独立的持久提交窗口，失败处理也会脱离过期请求落 retry；
|
||
// claim 租约必须覆盖三段最坏边界，否则第二实例会在第一实例仍收尾时重复施压。
|
||
maxBoundedWork := giftOperationAttemptTimeout + roomMutationCommitTimeout + giftOperationPersistTimeout
|
||
if maxBoundedWork >= giftOperationClaimTTL {
|
||
t.Fatalf("recovery bounded work %s must be shorter than claim lease %s", maxBoundedWork, giftOperationClaimTTL)
|
||
}
|
||
}
|
||
|
||
func TestGiftOperationRecoveryDeadlineKeepsClaimUntilCellCommitBudgetEnds(t *testing.T) {
|
||
for _, err := range []error{
|
||
context.DeadlineExceeded,
|
||
context.Canceled,
|
||
status.Error(codes.DeadlineExceeded, "wallet timed out"),
|
||
status.Error(codes.Canceled, "attempt canceled"),
|
||
} {
|
||
if !giftOperationFailureMustKeepClaim(true, err) {
|
||
t.Fatalf("recovery timeout must preserve its claim: %v", err)
|
||
}
|
||
}
|
||
if giftOperationFailureMustKeepClaim(false, context.DeadlineExceeded) {
|
||
t.Fatal("foreground request has no recovery claim to preserve")
|
||
}
|
||
if giftOperationFailureMustKeepClaim(true, errors.New("wallet unavailable")) {
|
||
t.Fatal("completed retryable failures may release the claim for the next attempt")
|
||
}
|
||
}
|
||
|
||
func TestGiftOperationFailureOwnershipConflictRetriesInsteadOfCompensating(t *testing.T) {
|
||
ctx := appcode.WithContext(context.Background(), appcode.Default)
|
||
now := time.Date(2026, 7, 12, 9, 0, 0, 0, time.UTC)
|
||
req := &roomv1.SendGiftRequest{
|
||
Meta: &roomv1.RequestMeta{AppCode: appcode.Default, RoomId: "room-owner-conflict", CommandId: "cmd-owner-conflict", ActorUserId: 101},
|
||
TargetType: "user",
|
||
TargetUserId: 202,
|
||
GiftId: "rose",
|
||
GiftCount: 1,
|
||
}
|
||
|
||
repository := &giftOperationOwnershipRepository{}
|
||
flow := newGiftFlow(&Service{repository: repository, clock: fixedRoomTestClock{now: now}}, ctx, req, giftSendOptions{})
|
||
flow.operationStarted = true
|
||
flow.recovering = true
|
||
flow.handleGiftOperationFailure(roomOwnedByOtherNodeError("room-node-b"))
|
||
if len(repository.compensated) != 0 {
|
||
t.Fatalf("cross-node ownership conflict must not compensate a pending saga: %+v", repository.compensated)
|
||
}
|
||
if len(repository.retries) != 1 || repository.retries[0].preserveLease {
|
||
t.Fatalf("ownership conflict should release the claim for the owner node to retry: %+v", repository.retries)
|
||
}
|
||
|
||
repository = &giftOperationOwnershipRepository{}
|
||
flow = newGiftFlow(&Service{repository: repository, clock: fixedRoomTestClock{now: now}}, ctx, req, giftSendOptions{})
|
||
flow.operationStarted = true
|
||
flow.handleGiftOperationFailure(xerr.New(xerr.Conflict, "command_id payload conflict"))
|
||
if len(repository.compensated) != 1 || len(repository.retries) != 0 {
|
||
t.Fatalf("business conflict before debit must stay compensated: compensated=%+v retries=%+v", repository.compensated, repository.retries)
|
||
}
|
||
}
|
||
|
||
func TestRecoverGiftOperationDefersToOwnerNode(t *testing.T) {
|
||
ctx := context.Background()
|
||
now := time.Date(2026, 7, 12, 9, 0, 0, 0, time.UTC)
|
||
directory := router.NewMemoryDirectory()
|
||
repository := &giftOperationOwnershipRepository{}
|
||
svc := &Service{
|
||
nodeID: "room-node-a",
|
||
leaseTTL: 10 * time.Second,
|
||
clock: fixedRoomTestClock{now: now},
|
||
directory: directory,
|
||
repository: repository,
|
||
}
|
||
operation := GiftOperation{
|
||
AppCode: appcode.Default,
|
||
RoomID: "room-recovery-owner",
|
||
CommandID: "cmd-recovery-owner",
|
||
Status: GiftOperationStatusPending,
|
||
}
|
||
if _, err := directory.EnsureOwner(ctx, runtimeRoomKey(operation.AppCode, operation.RoomID), "room-node-b", now, 10*time.Second); err != nil {
|
||
t.Fatalf("seed other-node lease: %v", err)
|
||
}
|
||
|
||
// RequestPayload 故意留空：非 owner 分支必须在解码存量请求、进入 Room Cell 之前把任务让给 owner。
|
||
svc.recoverGiftOperation(ctx, operation)
|
||
if len(repository.retries) != 1 {
|
||
t.Fatalf("non-owner recovery must defer exactly once, got %+v", repository.retries)
|
||
}
|
||
if repository.retries[0].preserveLease || repository.retries[0].lastError != roomOwnedByOtherNodeMsgPrefix+"room-node-b" {
|
||
t.Fatalf("defer must release the claim and record the current owner: %+v", repository.retries[0])
|
||
}
|
||
if len(repository.compensated) != 0 {
|
||
t.Fatalf("defer must not compensate the operation: %+v", repository.compensated)
|
||
}
|
||
}
|
||
|
||
func TestIsRoomOwnedByOtherNodeError(t *testing.T) {
|
||
if !isRoomOwnedByOtherNodeError(roomOwnedByOtherNodeError("room-node-b")) {
|
||
t.Fatal("ownership conflict error must be recognized")
|
||
}
|
||
for _, err := range []error{
|
||
xerr.New(xerr.Conflict, "command_id payload conflict"),
|
||
xerr.New(xerr.Unavailable, roomOwnedByOtherNodeMsgPrefix+"room-node-b"),
|
||
errors.New(roomOwnedByOtherNodeMsgPrefix + "room-node-b"),
|
||
} {
|
||
if isRoomOwnedByOtherNodeError(err) {
|
||
t.Fatalf("must not treat %v as cross-node ownership conflict", err)
|
||
}
|
||
}
|
||
}
|
||
|
||
func TestCompensatedGiftOperationReactivatesOnlyAfterPayloadMatch(t *testing.T) {
|
||
ctx := appcode.WithContext(context.Background(), appcode.Default)
|
||
originalRequest := &roomv1.SendGiftRequest{
|
||
Meta: &roomv1.RequestMeta{AppCode: appcode.Default, RoomId: "room-reactivate", CommandId: "cmd-reactivate", ActorUserId: 101},
|
||
TargetType: "user", TargetUserId: 202, GiftId: "rose", GiftCount: 1, SenderRegionId: 10,
|
||
TargetIsHost: true, TargetHostRegionId: 20, TargetAgencyOwnerUserId: 303,
|
||
SenderDisplayProfile: &roomv1.SendGiftDisplayProfile{UserId: 101, Username: "first sender"},
|
||
}
|
||
// Build the stored command through the same normalizer used by production so this test exercises
|
||
// the semantic idempotency comparison instead of depending on hand-written JSON field defaults.
|
||
originalFlow := newGiftFlow(&Service{}, ctx, originalRequest, giftSendOptions{})
|
||
storedPayload, err := command.Serialize(originalFlow.cmd)
|
||
if err != nil {
|
||
t.Fatalf("serialize stored gift command: %v", err)
|
||
}
|
||
storedRequestPayload, err := proto.Marshal(originalRequest)
|
||
if err != nil {
|
||
t.Fatalf("serialize stored gift request: %v", err)
|
||
}
|
||
|
||
t.Run("conflicting payload stays compensated", func(t *testing.T) {
|
||
repository := &compensatedGiftOperationRepository{operation: GiftOperation{
|
||
AppCode: appcode.Default, RoomID: "room-reactivate", CommandID: "cmd-reactivate",
|
||
CommandPayload: storedPayload, RequestPayload: storedRequestPayload,
|
||
Status: GiftOperationStatusCompensated, LastError: "insufficient balance",
|
||
}}
|
||
service := &Service{repository: repository}
|
||
conflictingRequest := &roomv1.SendGiftRequest{
|
||
Meta: &roomv1.RequestMeta{AppCode: appcode.Default, RoomId: "room-reactivate", CommandId: "cmd-reactivate", ActorUserId: 101},
|
||
TargetType: "user", TargetUserId: 202, GiftId: "different-gift", GiftCount: 1,
|
||
}
|
||
flow := newGiftFlow(service, ctx, conflictingRequest, giftSendOptions{})
|
||
if err := flow.ensureGiftOperation(time.UnixMilli(2_000)); !xerr.IsCode(err, xerr.Conflict) {
|
||
t.Fatalf("conflicting command must fail before reactivation: %v", err)
|
||
}
|
||
if repository.reactivateCalls != 0 || repository.operation.Status != GiftOperationStatusCompensated {
|
||
t.Fatalf("conflict exposed stored work to recovery: calls=%d operation=%+v", repository.reactivateCalls, repository.operation)
|
||
}
|
||
})
|
||
|
||
t.Run("identical payload CAS reactivates", func(t *testing.T) {
|
||
repository := &compensatedGiftOperationRepository{operation: GiftOperation{
|
||
AppCode: appcode.Default, RoomID: "room-reactivate", CommandID: "cmd-reactivate",
|
||
CommandPayload: storedPayload, RequestPayload: storedRequestPayload,
|
||
Status: GiftOperationStatusCompensated, LastError: "insufficient balance",
|
||
}}
|
||
service := &Service{repository: repository}
|
||
flow := newGiftFlow(service, ctx, originalRequest, giftSendOptions{})
|
||
if err := flow.ensureGiftOperation(time.UnixMilli(2_000)); err != nil {
|
||
t.Fatalf("identical command should reactivate: %v", err)
|
||
}
|
||
if repository.reactivateCalls != 1 || repository.operation.Status != GiftOperationStatusPending || !flow.operationStarted {
|
||
t.Fatalf("identical command did not enter pending exactly once: calls=%d operation=%+v", repository.reactivateCalls, repository.operation)
|
||
}
|
||
})
|
||
|
||
t.Run("load miss duplicate resumes first gateway snapshots", func(t *testing.T) {
|
||
repository := &compensatedGiftOperationRepository{operation: GiftOperation{
|
||
AppCode: appcode.Default, RoomID: "room-reactivate", CommandID: "cmd-reactivate",
|
||
CommandPayload: storedPayload, RequestPayload: storedRequestPayload, Status: GiftOperationStatusPending,
|
||
}}
|
||
service := &Service{repository: repository}
|
||
refreshedRequest := proto.Clone(originalRequest).(*roomv1.SendGiftRequest)
|
||
refreshedRequest.SenderRegionId = 99
|
||
refreshedRequest.TargetHostRegionId = 88
|
||
refreshedRequest.TargetAgencyOwnerUserId = 404
|
||
refreshedRequest.SenderDisplayProfile = &roomv1.SendGiftDisplayProfile{UserId: 101, Username: "renamed sender"}
|
||
flow := newGiftFlow(service, ctx, refreshedRequest, giftSendOptions{})
|
||
if err := flow.ensureGiftOperation(time.UnixMilli(2_000)); err != nil {
|
||
t.Fatalf("idempotency-equivalent duplicate should resume stored operation: %v", err)
|
||
}
|
||
if !flow.recovering || flow.cmd.SenderRegionID != 10 || flow.req.GetSenderRegionId() != 10 || flow.cmd.TargetHostRegionID != 20 || flow.req.GetTargetHostRegionId() != 20 || giftDisplayName(flow.cmd.SenderDisplayProfile) != "first sender" {
|
||
t.Fatalf("duplicate continued with refreshed gateway snapshots: command=%+v request=%+v", flow.cmd, flow.req)
|
||
}
|
||
if repository.reactivateCalls != 0 {
|
||
t.Fatalf("already-pending duplicate must not reactivate: %d", repository.reactivateCalls)
|
||
}
|
||
})
|
||
}
|