package service import ( "context" "errors" "testing" "time" "google.golang.org/grpc/codes" "google.golang.org/grpc/status" "google.golang.org/protobuf/proto" roomv1 "hyapp.local/api/proto/room/v1" "hyapp/pkg/appcode" "hyapp/pkg/xerr" "hyapp/services/room-service/internal/room/command" "hyapp/services/room-service/internal/router" ) type giftOperationOwnershipRepository struct { Repository retries []giftOperationRetryCall compensated []giftOperationRetryCall } type giftOperationRetryCall struct { roomID string commandID string lastError string preserveLease bool } func (r *giftOperationOwnershipRepository) MarkGiftOperationRetry(_ context.Context, roomID string, commandID string, lastError string, _ int64, _ int64, preserveLease bool) error { r.retries = append(r.retries, giftOperationRetryCall{roomID: roomID, commandID: commandID, lastError: lastError, preserveLease: preserveLease}) return nil } func (r *giftOperationOwnershipRepository) MarkGiftOperationCompensated(_ context.Context, roomID string, commandID string, lastError string, _ int64) error { r.compensated = append(r.compensated, giftOperationRetryCall{roomID: roomID, commandID: commandID, lastError: lastError}) return nil } type compensatedGiftOperationRepository struct { Repository operation GiftOperation reactivateCalls int } func (r *compensatedGiftOperationRepository) EnsureGiftOperation(_ context.Context, _ GiftOperation) (GiftOperation, bool, error) { return r.operation, false, nil } func (r *compensatedGiftOperationRepository) ReactivateCompensatedGiftOperation(_ context.Context, _, _ string, updatedAtMS int64) (GiftOperation, bool, error) { r.reactivateCalls++ if r.operation.Status == GiftOperationStatusCompensated { r.operation.Status = GiftOperationStatusPending r.operation.LastError = "" r.operation.UpdatedAtMS = updatedAtMS return r.operation, true, nil } return r.operation, false, nil } func TestGiftOperationRecoveryAttemptFitsInsideClaimLease(t *testing.T) { // mutateRoom 在钱包可能已经扣费后允许一个独立的持久提交窗口,失败处理也会脱离过期请求落 retry; // claim 租约必须覆盖三段最坏边界,否则第二实例会在第一实例仍收尾时重复施压。 maxBoundedWork := giftOperationAttemptTimeout + roomMutationCommitTimeout + giftOperationPersistTimeout if maxBoundedWork >= giftOperationClaimTTL { t.Fatalf("recovery bounded work %s must be shorter than claim lease %s", maxBoundedWork, giftOperationClaimTTL) } } func TestGiftOperationRecoveryDeadlineKeepsClaimUntilCellCommitBudgetEnds(t *testing.T) { for _, err := range []error{ context.DeadlineExceeded, context.Canceled, status.Error(codes.DeadlineExceeded, "wallet timed out"), status.Error(codes.Canceled, "attempt canceled"), } { if !giftOperationFailureMustKeepClaim(true, err) { t.Fatalf("recovery timeout must preserve its claim: %v", err) } } if giftOperationFailureMustKeepClaim(false, context.DeadlineExceeded) { t.Fatal("foreground request has no recovery claim to preserve") } if giftOperationFailureMustKeepClaim(true, errors.New("wallet unavailable")) { t.Fatal("completed retryable failures may release the claim for the next attempt") } } func TestGiftOperationFailureOwnershipConflictRetriesInsteadOfCompensating(t *testing.T) { ctx := appcode.WithContext(context.Background(), appcode.Default) now := time.Date(2026, 7, 12, 9, 0, 0, 0, time.UTC) req := &roomv1.SendGiftRequest{ Meta: &roomv1.RequestMeta{AppCode: appcode.Default, RoomId: "room-owner-conflict", CommandId: "cmd-owner-conflict", ActorUserId: 101}, TargetType: "user", TargetUserId: 202, GiftId: "rose", GiftCount: 1, } repository := &giftOperationOwnershipRepository{} flow := newGiftFlow(&Service{repository: repository, clock: fixedRoomTestClock{now: now}}, ctx, req, giftSendOptions{}) flow.operationStarted = true flow.recovering = true flow.handleGiftOperationFailure(roomOwnedByOtherNodeError("room-node-b")) if len(repository.compensated) != 0 { t.Fatalf("cross-node ownership conflict must not compensate a pending saga: %+v", repository.compensated) } if len(repository.retries) != 1 || repository.retries[0].preserveLease { t.Fatalf("ownership conflict should release the claim for the owner node to retry: %+v", repository.retries) } repository = &giftOperationOwnershipRepository{} flow = newGiftFlow(&Service{repository: repository, clock: fixedRoomTestClock{now: now}}, ctx, req, giftSendOptions{}) flow.operationStarted = true flow.handleGiftOperationFailure(xerr.New(xerr.Conflict, "command_id payload conflict")) if len(repository.compensated) != 1 || len(repository.retries) != 0 { t.Fatalf("business conflict before debit must stay compensated: compensated=%+v retries=%+v", repository.compensated, repository.retries) } } func TestRecoverGiftOperationDefersToOwnerNode(t *testing.T) { ctx := context.Background() now := time.Date(2026, 7, 12, 9, 0, 0, 0, time.UTC) directory := router.NewMemoryDirectory() repository := &giftOperationOwnershipRepository{} svc := &Service{ nodeID: "room-node-a", leaseTTL: 10 * time.Second, clock: fixedRoomTestClock{now: now}, directory: directory, repository: repository, } operation := GiftOperation{ AppCode: appcode.Default, RoomID: "room-recovery-owner", CommandID: "cmd-recovery-owner", Status: GiftOperationStatusPending, } if _, err := directory.EnsureOwner(ctx, runtimeRoomKey(operation.AppCode, operation.RoomID), "room-node-b", now, 10*time.Second); err != nil { t.Fatalf("seed other-node lease: %v", err) } // RequestPayload 故意留空:非 owner 分支必须在解码存量请求、进入 Room Cell 之前把任务让给 owner。 svc.recoverGiftOperation(ctx, operation) if len(repository.retries) != 1 { t.Fatalf("non-owner recovery must defer exactly once, got %+v", repository.retries) } if repository.retries[0].preserveLease || repository.retries[0].lastError != roomOwnedByOtherNodeMsgPrefix+"room-node-b" { t.Fatalf("defer must release the claim and record the current owner: %+v", repository.retries[0]) } if len(repository.compensated) != 0 { t.Fatalf("defer must not compensate the operation: %+v", repository.compensated) } } func TestIsRoomOwnedByOtherNodeError(t *testing.T) { if !isRoomOwnedByOtherNodeError(roomOwnedByOtherNodeError("room-node-b")) { t.Fatal("ownership conflict error must be recognized") } for _, err := range []error{ xerr.New(xerr.Conflict, "command_id payload conflict"), xerr.New(xerr.Unavailable, roomOwnedByOtherNodeMsgPrefix+"room-node-b"), errors.New(roomOwnedByOtherNodeMsgPrefix + "room-node-b"), } { if isRoomOwnedByOtherNodeError(err) { t.Fatalf("must not treat %v as cross-node ownership conflict", err) } } } func TestCompensatedGiftOperationReactivatesOnlyAfterPayloadMatch(t *testing.T) { ctx := appcode.WithContext(context.Background(), appcode.Default) originalRequest := &roomv1.SendGiftRequest{ Meta: &roomv1.RequestMeta{AppCode: appcode.Default, RoomId: "room-reactivate", CommandId: "cmd-reactivate", ActorUserId: 101}, TargetType: "user", TargetUserId: 202, GiftId: "rose", GiftCount: 1, SenderRegionId: 10, TargetIsHost: true, TargetHostRegionId: 20, TargetAgencyOwnerUserId: 303, SenderDisplayProfile: &roomv1.SendGiftDisplayProfile{UserId: 101, Username: "first sender"}, } // Build the stored command through the same normalizer used by production so this test exercises // the semantic idempotency comparison instead of depending on hand-written JSON field defaults. originalFlow := newGiftFlow(&Service{}, ctx, originalRequest, giftSendOptions{}) storedPayload, err := command.Serialize(originalFlow.cmd) if err != nil { t.Fatalf("serialize stored gift command: %v", err) } storedRequestPayload, err := proto.Marshal(originalRequest) if err != nil { t.Fatalf("serialize stored gift request: %v", err) } t.Run("conflicting payload stays compensated", func(t *testing.T) { repository := &compensatedGiftOperationRepository{operation: GiftOperation{ AppCode: appcode.Default, RoomID: "room-reactivate", CommandID: "cmd-reactivate", CommandPayload: storedPayload, RequestPayload: storedRequestPayload, Status: GiftOperationStatusCompensated, LastError: "insufficient balance", }} service := &Service{repository: repository} conflictingRequest := &roomv1.SendGiftRequest{ Meta: &roomv1.RequestMeta{AppCode: appcode.Default, RoomId: "room-reactivate", CommandId: "cmd-reactivate", ActorUserId: 101}, TargetType: "user", TargetUserId: 202, GiftId: "different-gift", GiftCount: 1, } flow := newGiftFlow(service, ctx, conflictingRequest, giftSendOptions{}) if err := flow.ensureGiftOperation(time.UnixMilli(2_000)); !xerr.IsCode(err, xerr.Conflict) { t.Fatalf("conflicting command must fail before reactivation: %v", err) } if repository.reactivateCalls != 0 || repository.operation.Status != GiftOperationStatusCompensated { t.Fatalf("conflict exposed stored work to recovery: calls=%d operation=%+v", repository.reactivateCalls, repository.operation) } }) t.Run("identical payload CAS reactivates", func(t *testing.T) { repository := &compensatedGiftOperationRepository{operation: GiftOperation{ AppCode: appcode.Default, RoomID: "room-reactivate", CommandID: "cmd-reactivate", CommandPayload: storedPayload, RequestPayload: storedRequestPayload, Status: GiftOperationStatusCompensated, LastError: "insufficient balance", }} service := &Service{repository: repository} flow := newGiftFlow(service, ctx, originalRequest, giftSendOptions{}) if err := flow.ensureGiftOperation(time.UnixMilli(2_000)); err != nil { t.Fatalf("identical command should reactivate: %v", err) } if repository.reactivateCalls != 1 || repository.operation.Status != GiftOperationStatusPending || !flow.operationStarted { t.Fatalf("identical command did not enter pending exactly once: calls=%d operation=%+v", repository.reactivateCalls, repository.operation) } }) t.Run("load miss duplicate resumes first gateway snapshots", func(t *testing.T) { repository := &compensatedGiftOperationRepository{operation: GiftOperation{ AppCode: appcode.Default, RoomID: "room-reactivate", CommandID: "cmd-reactivate", CommandPayload: storedPayload, RequestPayload: storedRequestPayload, Status: GiftOperationStatusPending, }} service := &Service{repository: repository} refreshedRequest := proto.Clone(originalRequest).(*roomv1.SendGiftRequest) refreshedRequest.SenderRegionId = 99 refreshedRequest.TargetHostRegionId = 88 refreshedRequest.TargetAgencyOwnerUserId = 404 refreshedRequest.SenderDisplayProfile = &roomv1.SendGiftDisplayProfile{UserId: 101, Username: "renamed sender"} flow := newGiftFlow(service, ctx, refreshedRequest, giftSendOptions{}) if err := flow.ensureGiftOperation(time.UnixMilli(2_000)); err != nil { t.Fatalf("idempotency-equivalent duplicate should resume stored operation: %v", err) } if !flow.recovering || flow.cmd.SenderRegionID != 10 || flow.req.GetSenderRegionId() != 10 || flow.cmd.TargetHostRegionID != 20 || flow.req.GetTargetHostRegionId() != 20 || giftDisplayName(flow.cmd.SenderDisplayProfile) != "first sender" { t.Fatalf("duplicate continued with refreshed gateway snapshots: command=%+v request=%+v", flow.cmd, flow.req) } if repository.reactivateCalls != 0 { t.Fatalf("already-pending duplicate must not reactivate: %d", repository.reactivateCalls) } }) }