441 lines
19 KiB
Go

package outboxarchiverestore
import (
"context"
"crypto/sha256"
"database/sql"
"encoding/hex"
"errors"
"fmt"
"reflect"
"strings"
"time"
_ "github.com/go-sql-driver/mysql"
)
const (
RehearsalTable = "wallet_outbox_restore_rehearsals_v1"
RowsTable = "wallet_outbox_restore_rows_v1"
restoreState = "RESTORING"
)
// RestoreResult is safe to print: it identifies the selected schema and fixed
// table contract but never includes the DSN or credentials used to reach it.
type RestoreResult struct {
State string `json:"state"`
RehearsalID string `json:"rehearsal_id"`
BatchID string `json:"batch_id"`
AppCode string `json:"app_code"`
RowCount int `json:"row_count"`
TargetDatabase string `json:"target_database"`
RehearsalTable string `json:"rehearsal_table"`
RowsTable string `json:"rows_table"`
DataObjectKey string `json:"data_object_key"`
FirstCursor Cursor `json:"first_cursor"`
LastCursor Cursor `json:"last_cursor"`
NDJSONSHA256 string `json:"ndjson_sha256"`
GzipSHA256 string `json:"gzip_sha256"`
DataCRC64 string `json:"data_crc64"`
ManifestSHA256 string `json:"manifest_sha256"`
ManifestCRC64 string `json:"manifest_crc64"`
SuccessSHA256 string `json:"success_sha256"`
SuccessCRC64 string `json:"success_crc64"`
AvailableTotal string `json:"available_delta_total"`
FrozenTotal string `json:"frozen_delta_total"`
VerifiedAtMS int64 `json:"verified_at_ms"`
IdempotentReadback bool `json:"idempotent_readback"`
}
// Restore persists a fully verified batch into versioned isolation tables. All
// DDL targets new/fixed table names; all DML is scoped by rehearsal PK. It never
// reads, locks, updates, or deletes wallet_outbox.
func Restore(ctx context.Context, dsn, expectedDatabase, rehearsalID string, verification Verification) (RestoreResult, error) {
dsn = strings.TrimSpace(dsn)
expectedDatabase = strings.TrimSpace(expectedDatabase)
rehearsalID = strings.TrimSpace(rehearsalID)
if dsn == "" || expectedDatabase == "" {
return RestoreResult{}, errors.New("restore DSN and exact expected database are required")
}
if !validRehearsalID(rehearsalID) {
return RestoreResult{}, errors.New("rehearsal_id must be 1..64 ASCII letters, digits, dot, underscore, or dash")
}
if verification.State != filesVerifiedState || len(verification.Records) != verification.Manifest.RowCount {
return RestoreResult{}, errors.New("local archive verification is incomplete")
}
db, err := sql.Open("mysql", dsn)
if err != nil {
return RestoreResult{}, err
}
defer db.Close()
// A rehearsal is intentionally single-connection and serial; this prevents a
// restore tool from consuming the production-sized connection pool by mistake.
db.SetMaxOpenConns(1)
db.SetMaxIdleConns(1)
db.SetConnMaxLifetime(10 * time.Minute)
conn, err := db.Conn(ctx)
if err != nil {
return RestoreResult{}, err
}
defer conn.Close()
var databaseName string
if err := conn.QueryRowContext(ctx, `SELECT DATABASE()`).Scan(&databaseName); err != nil {
return RestoreResult{}, err
}
if databaseName != expectedDatabase {
return RestoreResult{}, fmt.Errorf("restore database mismatch: connected=%q expected=%q", databaseName, expectedDatabase)
}
if err := ensureIsolationSchema(ctx, conn); err != nil {
return RestoreResult{}, err
}
release, acquired, err := acquireRestoreLock(ctx, conn, rehearsalID)
if err != nil {
return RestoreResult{}, err
}
if !acquired {
return RestoreResult{}, errors.New("another process holds the same restore rehearsal lock")
}
defer release()
if existing, found, err := readExistingRestore(ctx, conn, rehearsalID, verification); err != nil {
return RestoreResult{}, err
} else if found {
return existing, nil
}
tx, err := conn.BeginTx(ctx, &sql.TxOptions{Isolation: sql.LevelReadCommitted})
if err != nil {
return RestoreResult{}, err
}
defer tx.Rollback()
nowMS := time.Now().UTC().UnixMilli()
if err := insertRehearsal(ctx, tx, rehearsalID, databaseName, verification, nowMS); err != nil {
return RestoreResult{}, err
}
for offset := 0; offset < len(verification.Records); offset += 100 {
end := min(offset+100, len(verification.Records))
if err := insertRestoreRows(ctx, tx, rehearsalID, verification.Manifest.BatchID, nowMS, verification.Records[offset:end]); err != nil {
return RestoreResult{}, err
}
}
// These aggregates only scan one bounded rehearsal partition in the isolation
// table. They never aggregate the live outbox and are the database-level proof
// that JSON conversion and INSERT semantics preserved the archive controls.
if err := verifyRestoredControls(ctx, tx, rehearsalID, verification); err != nil {
return RestoreResult{}, err
}
if err := verifyRestoredRowsExact(ctx, tx, rehearsalID, verification.Records); err != nil {
return RestoreResult{}, err
}
result := newRestoreResult(rehearsalID, databaseName, verification, nowMS, false)
update, err := tx.ExecContext(ctx, `
UPDATE wallet_outbox_restore_rehearsals_v1
SET state = ?, verified_at_ms = ?, updated_at_ms = ?
WHERE rehearsal_id = ? AND state = ?`, restoreVerifiedState, nowMS, nowMS, rehearsalID, restoreState)
if err != nil {
return RestoreResult{}, err
}
if affected, err := update.RowsAffected(); err != nil || affected != 1 {
return RestoreResult{}, errors.New("restore rehearsal state transition did not affect exactly one row")
}
if err := tx.Commit(); err != nil {
return RestoreResult{}, err
}
return result, nil
}
func ensureIsolationSchema(ctx context.Context, conn *sql.Conn) error {
// CREATE IF NOT EXISTS takes metadata locks only on the dedicated table names;
// unlike ALTER wallet_outbox it cannot rebuild or block the hot owner table.
statements := []string{`
CREATE TABLE IF NOT EXISTS wallet_outbox_restore_rehearsals_v1 (
rehearsal_id VARCHAR(64) NOT NULL,
source_app_code VARCHAR(32) NOT NULL,
source_batch_id CHAR(64) NOT NULL,
state VARCHAR(32) NOT NULL,
schema_version INT NOT NULL,
data_object_key VARCHAR(512) NOT NULL,
row_count INT NOT NULL,
first_updated_at_ms BIGINT NOT NULL,
first_event_id VARCHAR(128) NOT NULL,
last_updated_at_ms BIGINT NOT NULL,
last_event_id VARCHAR(128) NOT NULL,
available_delta_total VARCHAR(80) NOT NULL,
frozen_delta_total VARCHAR(80) NOT NULL,
ndjson_sha256 CHAR(64) NOT NULL,
gzip_sha256 CHAR(64) NOT NULL,
data_crc64 VARCHAR(32) NOT NULL,
manifest_sha256 CHAR(64) NOT NULL,
manifest_crc64 VARCHAR(32) NOT NULL,
success_sha256 CHAR(64) NOT NULL,
success_crc64 VARCHAR(32) NOT NULL,
target_database VARCHAR(128) NOT NULL,
verified_at_ms BIGINT NOT NULL,
created_at_ms BIGINT NOT NULL,
updated_at_ms BIGINT NOT NULL,
PRIMARY KEY (rehearsal_id),
KEY idx_wallet_outbox_restore_batch (source_app_code, source_batch_id, state)
) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_unicode_ci COMMENT='wallet outbox archive restore rehearsal evidence v1'`, `
CREATE TABLE IF NOT EXISTS wallet_outbox_restore_rows_v1 (
rehearsal_id VARCHAR(64) NOT NULL,
source_batch_id CHAR(64) NOT NULL,
app_code VARCHAR(32) NOT NULL,
event_id VARCHAR(128) NOT NULL,
event_type VARCHAR(64) NOT NULL,
transaction_id VARCHAR(96) NOT NULL,
command_id VARCHAR(128) NOT NULL,
user_id BIGINT NOT NULL,
asset_type VARCHAR(32) NOT NULL,
available_delta BIGINT NOT NULL,
frozen_delta BIGINT NOT NULL,
payload JSON NOT NULL,
status VARCHAR(32) NOT NULL,
worker_id VARCHAR(128) NOT NULL,
lock_until_ms BIGINT NULL,
retry_count INT NOT NULL,
next_retry_at_ms BIGINT NULL,
last_error TEXT NULL,
created_at_ms BIGINT NOT NULL,
updated_at_ms BIGINT NOT NULL,
restored_at_ms BIGINT NOT NULL,
PRIMARY KEY (rehearsal_id, app_code, event_id),
KEY idx_wallet_outbox_restore_cursor (rehearsal_id, app_code, updated_at_ms, event_id),
KEY idx_wallet_outbox_restore_batch (source_batch_id, app_code, event_id)
) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_unicode_ci COMMENT='isolated wallet_outbox-compatible restored rows v1'`}
for _, statement := range statements {
if _, err := conn.ExecContext(ctx, statement); err != nil {
return err
}
}
return nil
}
func insertRehearsal(ctx context.Context, tx *sql.Tx, rehearsalID, databaseName string, verification Verification, nowMS int64) error {
manifest := verification.Manifest
_, err := tx.ExecContext(ctx, `
INSERT INTO wallet_outbox_restore_rehearsals_v1 (
rehearsal_id, source_app_code, source_batch_id, state, schema_version, data_object_key, row_count,
first_updated_at_ms, first_event_id, last_updated_at_ms, last_event_id,
available_delta_total, frozen_delta_total, ndjson_sha256, gzip_sha256, data_crc64,
manifest_sha256, manifest_crc64, success_sha256, success_crc64, target_database,
verified_at_ms, created_at_ms, updated_at_ms
) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, 0, ?, ?)`,
rehearsalID, manifest.AppCode, manifest.BatchID, restoreState, manifest.SchemaVersion, manifest.DataObjectKey, manifest.RowCount,
manifest.FirstCursor.UpdatedAtMS, manifest.FirstCursor.EventID, manifest.LastCursor.UpdatedAtMS, manifest.LastCursor.EventID,
manifest.AvailableDeltaTotal, manifest.FrozenDeltaTotal, manifest.NDJSONSHA256, manifest.GzipSHA256, manifest.DataCRC64,
verification.ManifestSHA256, verification.ManifestCRC64, verification.SuccessSHA256, verification.SuccessCRC64, databaseName,
nowMS, nowMS)
return err
}
func insertRestoreRows(ctx context.Context, tx *sql.Tx, rehearsalID, batchID string, restoredAtMS int64, records []Record) error {
if len(records) == 0 || len(records) > 100 {
return errors.New("restore insert batch must contain 1..100 rows")
}
const columns = `(rehearsal_id, source_batch_id, app_code, event_id, event_type, transaction_id, command_id,
user_id, asset_type, available_delta, frozen_delta, payload, status, worker_id, lock_until_ms,
retry_count, next_retry_at_ms, last_error, created_at_ms, updated_at_ms, restored_at_ms)`
placeholders := make([]string, 0, len(records))
args := make([]any, 0, len(records)*21)
for _, record := range records {
placeholders = append(placeholders, `(?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)`)
args = append(args,
rehearsalID, batchID, record.AppCode, record.EventID, record.EventType, record.TransactionID, record.CommandID,
record.UserID, record.AssetType, record.AvailableDelta, record.FrozenDelta, record.PayloadJSON, record.Status, record.WorkerID,
nullInt64(record.LockUntilMS), record.RetryCount, nullInt64(record.NextRetryAtMS), nullString(record.LastError),
record.CreatedAtMS, record.UpdatedAtMS, restoredAtMS,
)
}
_, err := tx.ExecContext(ctx, `INSERT INTO wallet_outbox_restore_rows_v1 `+columns+` VALUES `+strings.Join(placeholders, ","), args...)
return err
}
func verifyRestoredControls(ctx context.Context, query interface {
QueryRowContext(context.Context, string, ...any) *sql.Row
}, rehearsalID string, verification Verification) error {
var count int
var availableTotal, frozenTotal string
var minCreated, maxCreated, minUpdated, maxUpdated int64
if err := query.QueryRowContext(ctx, `
SELECT COUNT(*),
CAST(COALESCE(SUM(available_delta), 0) AS CHAR),
CAST(COALESCE(SUM(frozen_delta), 0) AS CHAR),
COALESCE(MIN(created_at_ms), 0), COALESCE(MAX(created_at_ms), 0),
COALESCE(MIN(updated_at_ms), 0), COALESCE(MAX(updated_at_ms), 0)
FROM wallet_outbox_restore_rows_v1 FORCE INDEX (PRIMARY)
WHERE rehearsal_id = ?`, rehearsalID).Scan(&count, &availableTotal, &frozenTotal, &minCreated, &maxCreated, &minUpdated, &maxUpdated); err != nil {
return err
}
manifest := verification.Manifest
if count != manifest.RowCount || availableTotal != manifest.AvailableDeltaTotal || frozenTotal != manifest.FrozenDeltaTotal || minCreated != manifest.MinCreatedAtMS || maxCreated != manifest.MaxCreatedAtMS || minUpdated != manifest.MinUpdatedAtMS || maxUpdated != manifest.MaxUpdatedAtMS {
return errors.New("isolation-table row count, time bounds, or control totals do not match manifest")
}
var firstUpdated, lastUpdated int64
var firstEvent, lastEvent string
if err := query.QueryRowContext(ctx, `
SELECT updated_at_ms, event_id FROM wallet_outbox_restore_rows_v1 FORCE INDEX (idx_wallet_outbox_restore_cursor)
WHERE rehearsal_id = ? AND app_code = ? ORDER BY updated_at_ms ASC, event_id ASC LIMIT 1`, rehearsalID, manifest.AppCode).Scan(&firstUpdated, &firstEvent); err != nil {
return err
}
if err := query.QueryRowContext(ctx, `
SELECT updated_at_ms, event_id FROM wallet_outbox_restore_rows_v1 FORCE INDEX (idx_wallet_outbox_restore_cursor)
WHERE rehearsal_id = ? AND app_code = ? ORDER BY updated_at_ms DESC, event_id DESC LIMIT 1`, rehearsalID, manifest.AppCode).Scan(&lastUpdated, &lastEvent); err != nil {
return err
}
if (Cursor{UpdatedAtMS: firstUpdated, EventID: firstEvent}) != manifest.FirstCursor || (Cursor{UpdatedAtMS: lastUpdated, EventID: lastEvent}) != manifest.LastCursor {
return errors.New("isolation-table cursor bounds do not match manifest")
}
return nil
}
func readExistingRestore(ctx context.Context, conn *sql.Conn, rehearsalID string, verification Verification) (RestoreResult, bool, error) {
var state, appCode, batchID, manifestSHA, manifestCRC, successCRC, availableTotal, frozenTotal string
var rowCount int
var verifiedAtMS int64
err := conn.QueryRowContext(ctx, `
SELECT state, source_app_code, source_batch_id, row_count, manifest_sha256, manifest_crc64, success_crc64,
available_delta_total, frozen_delta_total, verified_at_ms
FROM wallet_outbox_restore_rehearsals_v1 WHERE rehearsal_id = ?`, rehearsalID).
Scan(&state, &appCode, &batchID, &rowCount, &manifestSHA, &manifestCRC, &successCRC, &availableTotal, &frozenTotal, &verifiedAtMS)
if err == sql.ErrNoRows {
return RestoreResult{}, false, nil
}
if err != nil {
return RestoreResult{}, false, err
}
manifest := verification.Manifest
if state != restoreVerifiedState || appCode != manifest.AppCode || batchID != manifest.BatchID || rowCount != manifest.RowCount || manifestSHA != verification.ManifestSHA256 || manifestCRC != verification.ManifestCRC64 || successCRC != verification.SuccessCRC64 || availableTotal != manifest.AvailableDeltaTotal || frozenTotal != manifest.FrozenDeltaTotal {
return RestoreResult{}, false, errors.New("rehearsal_id already exists with non-matching or incomplete evidence")
}
if err := verifyRestoredControls(ctx, conn, rehearsalID, verification); err != nil {
return RestoreResult{}, false, fmt.Errorf("existing restore readback failed: %w", err)
}
if err := verifyRestoredRowsExact(ctx, conn, rehearsalID, verification.Records); err != nil {
return RestoreResult{}, false, fmt.Errorf("existing restore exact-row readback failed: %w", err)
}
result := newRestoreResult(rehearsalID, "", verification, verifiedAtMS, true)
if err := conn.QueryRowContext(ctx, `SELECT DATABASE()`).Scan(&result.TargetDatabase); err != nil {
return RestoreResult{}, false, err
}
return result, true, nil
}
func newRestoreResult(rehearsalID, databaseName string, verification Verification, verifiedAtMS int64, idempotent bool) RestoreResult {
manifest := verification.Manifest
return RestoreResult{
State: restoreVerifiedState, RehearsalID: rehearsalID, BatchID: manifest.BatchID, AppCode: manifest.AppCode,
RowCount: manifest.RowCount, TargetDatabase: databaseName, RehearsalTable: RehearsalTable, RowsTable: RowsTable,
DataObjectKey: manifest.DataObjectKey, FirstCursor: manifest.FirstCursor, LastCursor: manifest.LastCursor,
NDJSONSHA256: manifest.NDJSONSHA256, GzipSHA256: manifest.GzipSHA256, DataCRC64: manifest.DataCRC64,
ManifestSHA256: verification.ManifestSHA256, ManifestCRC64: verification.ManifestCRC64,
SuccessSHA256: verification.SuccessSHA256, SuccessCRC64: verification.SuccessCRC64,
AvailableTotal: manifest.AvailableDeltaTotal, FrozenTotal: manifest.FrozenDeltaTotal,
VerifiedAtMS: verifiedAtMS, IdempotentReadback: idempotent,
}
}
func verifyRestoredRowsExact(ctx context.Context, query interface {
QueryContext(context.Context, string, ...any) (*sql.Rows, error)
}, rehearsalID string, expected []Record) error {
rows, err := query.QueryContext(ctx, `
SELECT app_code, event_id, event_type, transaction_id, command_id, user_id, asset_type,
available_delta, frozen_delta, CAST(payload AS CHAR), status, worker_id,
lock_until_ms, retry_count, next_retry_at_ms, last_error, created_at_ms, updated_at_ms
FROM wallet_outbox_restore_rows_v1 FORCE INDEX (idx_wallet_outbox_restore_cursor)
WHERE rehearsal_id = ?
ORDER BY app_code ASC, updated_at_ms ASC, event_id ASC`, rehearsalID)
if err != nil {
return err
}
defer rows.Close()
index := 0
for rows.Next() {
if index >= len(expected) {
return errors.New("isolation table contains more rows than verified archive")
}
var actual Record
var lockUntilMS, nextRetryAtMS sql.NullInt64
var lastError sql.NullString
if err := rows.Scan(
&actual.AppCode, &actual.EventID, &actual.EventType, &actual.TransactionID, &actual.CommandID,
&actual.UserID, &actual.AssetType, &actual.AvailableDelta, &actual.FrozenDelta, &actual.PayloadJSON,
&actual.Status, &actual.WorkerID, &lockUntilMS, &actual.RetryCount, &nextRetryAtMS, &lastError,
&actual.CreatedAtMS, &actual.UpdatedAtMS,
); err != nil {
return err
}
if lockUntilMS.Valid {
value := lockUntilMS.Int64
actual.LockUntilMS = &value
}
if nextRetryAtMS.Valid {
value := nextRetryAtMS.Int64
actual.NextRetryAtMS = &value
}
if lastError.Valid {
value := lastError.String
actual.LastError = &value
}
// Source payload was archived from CAST(payload AS CHAR); casting the restored
// JSON back through MySQL proves JSON parsing did not alter the owner snapshot.
if !reflect.DeepEqual(actual, expected[index]) {
return fmt.Errorf("isolation table row %d does not exactly match archive", index+1)
}
index++
}
if err := rows.Err(); err != nil {
return err
}
if index != len(expected) {
return errors.New("isolation table contains fewer rows than verified archive")
}
return nil
}
func acquireRestoreLock(ctx context.Context, conn *sql.Conn, rehearsalID string) (func(), bool, error) {
hash := sha256.Sum256([]byte(rehearsalID))
lockName := "wallet_outbox_restore:" + hex.EncodeToString(hash[:16])
var acquired sql.NullInt64
if err := conn.QueryRowContext(ctx, `SELECT GET_LOCK(?, 0)`, lockName).Scan(&acquired); err != nil {
return nil, false, err
}
if !acquired.Valid || acquired.Int64 != 1 {
return func() {}, false, nil
}
return func() {
releaseCtx, cancel := context.WithTimeout(context.Background(), 3*time.Second)
defer cancel()
var released sql.NullInt64
_ = conn.QueryRowContext(releaseCtx, `SELECT RELEASE_LOCK(?)`, lockName).Scan(&released)
}, true, nil
}
func validRehearsalID(value string) bool {
if len(value) == 0 || len(value) > 64 {
return false
}
for _, char := range value {
if (char >= 'a' && char <= 'z') || (char >= 'A' && char <= 'Z') || (char >= '0' && char <= '9') || char == '.' || char == '_' || char == '-' {
continue
}
return false
}
return true
}
func nullInt64(value *int64) any {
if value == nil {
return nil
}
return *value
}
func nullString(value *string) any {
if value == nil {
return nil
}
return *value
}