432 lines
16 KiB
Go
432 lines
16 KiB
Go
// Package outboxarchiverestore verifies one immutable wallet_outbox archive batch
|
|
// and restores it into versioned isolation tables. It deliberately contains no
|
|
// purge API: a successful rehearsal is evidence for a later retention gate, not
|
|
// authorization to mutate the owner outbox.
|
|
package outboxarchiverestore
|
|
|
|
import (
|
|
"bufio"
|
|
"bytes"
|
|
"compress/gzip"
|
|
"crypto/sha256"
|
|
"encoding/hex"
|
|
"encoding/json"
|
|
"errors"
|
|
"fmt"
|
|
"hash/crc64"
|
|
"io"
|
|
"math/big"
|
|
"os"
|
|
"path"
|
|
"strconv"
|
|
"strings"
|
|
"time"
|
|
)
|
|
|
|
const (
|
|
archiveSchema = "hyapp.wallet_outbox.archive"
|
|
archiveSchemaVersion = 1
|
|
manifestState = "DATA_VERIFIED"
|
|
successState = "VERIFIED"
|
|
filesVerifiedState = "FILES_VERIFIED"
|
|
restoreVerifiedState = "RESTORE_VERIFIED"
|
|
|
|
maxManifestBytes = 1 << 20
|
|
maxSuccessBytes = 64 << 10
|
|
maxNDJSONLine = 64 << 20
|
|
)
|
|
|
|
// Limits bounds local CPU, memory and disk exposure before the operator-selected
|
|
// restore database is opened. Values are intentionally independent of worker
|
|
// configuration so an accidentally supplied object cannot bypass the CLI guard.
|
|
type Limits struct {
|
|
MaxRows int
|
|
MaxCompressedBytes int64
|
|
MaxUncompressedBytes int64
|
|
}
|
|
|
|
func DefaultLimits() Limits {
|
|
return Limits{MaxRows: 100000, MaxCompressedBytes: 512 << 20, MaxUncompressedBytes: 1 << 30}
|
|
}
|
|
|
|
type Cursor struct {
|
|
UpdatedAtMS int64 `json:"updated_at_ms"`
|
|
EventID string `json:"event_id"`
|
|
}
|
|
|
|
type Manifest struct {
|
|
Schema string `json:"schema"`
|
|
SchemaVersion int `json:"schema_version"`
|
|
State string `json:"state"`
|
|
AppCode string `json:"app_code"`
|
|
BatchID string `json:"batch_id"`
|
|
RowCount int `json:"row_count"`
|
|
FirstCursor Cursor `json:"first_cursor"`
|
|
LastCursor Cursor `json:"last_cursor"`
|
|
MinCreatedAtMS int64 `json:"min_created_at_ms"`
|
|
MaxCreatedAtMS int64 `json:"max_created_at_ms"`
|
|
MinUpdatedAtMS int64 `json:"min_updated_at_ms"`
|
|
MaxUpdatedAtMS int64 `json:"max_updated_at_ms"`
|
|
AvailableDeltaTotal string `json:"available_delta_total"`
|
|
FrozenDeltaTotal string `json:"frozen_delta_total"`
|
|
NDJSONSHA256 string `json:"ndjson_sha256"`
|
|
GzipSHA256 string `json:"gzip_sha256"`
|
|
DataCRC64 string `json:"data_crc64"`
|
|
UncompressedBytes int64 `json:"uncompressed_bytes"`
|
|
CompressedBytes int64 `json:"compressed_bytes"`
|
|
DataObjectKey string `json:"data_object_key"`
|
|
}
|
|
|
|
type Success struct {
|
|
Schema string `json:"schema"`
|
|
SchemaVersion int `json:"schema_version"`
|
|
State string `json:"state"`
|
|
BatchID string `json:"batch_id"`
|
|
ManifestSHA256 string `json:"manifest_sha256"`
|
|
ManifestCRC64 string `json:"manifest_crc64"`
|
|
}
|
|
|
|
// Record mirrors every wallet_outbox column. Restore tables add rehearsal and
|
|
// source-batch keys around this payload rather than changing owner-table fields,
|
|
// so later historical tools can compare one archived row without translation.
|
|
type Record struct {
|
|
AppCode string `json:"app_code"`
|
|
EventID string `json:"event_id"`
|
|
EventType string `json:"event_type"`
|
|
TransactionID string `json:"transaction_id"`
|
|
CommandID string `json:"command_id"`
|
|
UserID int64 `json:"user_id"`
|
|
AssetType string `json:"asset_type"`
|
|
AvailableDelta int64 `json:"available_delta"`
|
|
FrozenDelta int64 `json:"frozen_delta"`
|
|
PayloadJSON string `json:"payload_json"`
|
|
Status string `json:"status"`
|
|
WorkerID string `json:"worker_id"`
|
|
LockUntilMS *int64 `json:"lock_until_ms"`
|
|
RetryCount int `json:"retry_count"`
|
|
NextRetryAtMS *int64 `json:"next_retry_at_ms"`
|
|
LastError *string `json:"last_error"`
|
|
CreatedAtMS int64 `json:"created_at_ms"`
|
|
UpdatedAtMS int64 `json:"updated_at_ms"`
|
|
}
|
|
|
|
// Verification is the complete local evidence set. SuccessCRC64 is computed
|
|
// locally because _SUCCESS cannot contain its own checksum; a future purge gate
|
|
// must compare all three CRC values with wallet_outbox_archive_receipts.
|
|
type Verification struct {
|
|
State string `json:"state"`
|
|
Manifest Manifest `json:"manifest"`
|
|
Records []Record `json:"-"`
|
|
ManifestSHA256 string `json:"manifest_sha256"`
|
|
ManifestCRC64 string `json:"manifest_crc64"`
|
|
SuccessSHA256 string `json:"success_sha256"`
|
|
SuccessCRC64 string `json:"success_crc64"`
|
|
}
|
|
|
|
// VerifyLocalFiles establishes the _SUCCESS gate before any restore DSN is
|
|
// opened. The order is deliberate: an incomplete batch may have data/manifest
|
|
// objects, but without a valid marker it can never reach a database write path.
|
|
func VerifyLocalFiles(dataPath, manifestPath, successPath string, limits Limits) (Verification, error) {
|
|
limits = normalizeLimits(limits)
|
|
successBody, err := readBoundedFile(successPath, maxSuccessBytes)
|
|
if err != nil {
|
|
return Verification{}, fmt.Errorf("read _SUCCESS: %w", err)
|
|
}
|
|
var success Success
|
|
if err := decodeStrictJSON(successBody, &success); err != nil {
|
|
return Verification{}, fmt.Errorf("decode _SUCCESS: %w", err)
|
|
}
|
|
if success.Schema != archiveSchema || success.SchemaVersion != archiveSchemaVersion || success.State != successState || !validSHA256(success.ManifestSHA256) || !validCRC64(success.ManifestCRC64) || !validSHA256(success.BatchID) {
|
|
return Verification{}, errors.New("_SUCCESS schema, state, batch_id, or manifest checksum is invalid")
|
|
}
|
|
|
|
manifestBody, err := readBoundedFile(manifestPath, maxManifestBytes)
|
|
if err != nil {
|
|
return Verification{}, fmt.Errorf("read manifest: %w", err)
|
|
}
|
|
manifestSHA256, manifestCRC64 := checksumBytes(manifestBody)
|
|
if manifestSHA256 != success.ManifestSHA256 || manifestCRC64 != success.ManifestCRC64 {
|
|
return Verification{}, errors.New("manifest does not match _SUCCESS SHA256/CRC64")
|
|
}
|
|
var manifest Manifest
|
|
if err := decodeStrictJSON(manifestBody, &manifest); err != nil {
|
|
return Verification{}, fmt.Errorf("decode manifest: %w", err)
|
|
}
|
|
if err := validateManifest(manifest, success, limits); err != nil {
|
|
return Verification{}, err
|
|
}
|
|
|
|
records, err := verifyDataFile(dataPath, manifest, limits)
|
|
if err != nil {
|
|
return Verification{}, err
|
|
}
|
|
successSHA256, successCRC64 := checksumBytes(successBody)
|
|
return Verification{
|
|
State: filesVerifiedState, Manifest: manifest, Records: records,
|
|
ManifestSHA256: manifestSHA256, ManifestCRC64: manifestCRC64,
|
|
SuccessSHA256: successSHA256, SuccessCRC64: successCRC64,
|
|
}, nil
|
|
}
|
|
|
|
func normalizeLimits(limits Limits) Limits {
|
|
defaults := DefaultLimits()
|
|
if limits.MaxRows <= 0 {
|
|
limits.MaxRows = defaults.MaxRows
|
|
}
|
|
if limits.MaxCompressedBytes <= 0 {
|
|
limits.MaxCompressedBytes = defaults.MaxCompressedBytes
|
|
}
|
|
if limits.MaxUncompressedBytes <= 0 {
|
|
limits.MaxUncompressedBytes = defaults.MaxUncompressedBytes
|
|
}
|
|
return limits
|
|
}
|
|
|
|
func validateManifest(manifest Manifest, success Success, limits Limits) error {
|
|
if manifest.Schema != archiveSchema || manifest.SchemaVersion != archiveSchemaVersion || manifest.State != manifestState {
|
|
return errors.New("manifest schema, version, or state is invalid")
|
|
}
|
|
if manifest.BatchID != success.BatchID || !validSHA256(manifest.BatchID) || strings.TrimSpace(manifest.AppCode) == "" || strings.TrimSpace(manifest.DataObjectKey) == "" {
|
|
return errors.New("manifest identity does not match _SUCCESS")
|
|
}
|
|
if manifest.RowCount <= 0 || manifest.RowCount > limits.MaxRows {
|
|
return fmt.Errorf("manifest row_count must be within [1,%d]", limits.MaxRows)
|
|
}
|
|
if manifest.CompressedBytes <= 0 || manifest.CompressedBytes > limits.MaxCompressedBytes || manifest.UncompressedBytes <= 0 || manifest.UncompressedBytes > limits.MaxUncompressedBytes {
|
|
return errors.New("manifest byte counts exceed restore safety limits")
|
|
}
|
|
if !validSHA256(manifest.NDJSONSHA256) || !validSHA256(manifest.GzipSHA256) || !validCRC64(manifest.DataCRC64) {
|
|
return errors.New("manifest data checksums are invalid")
|
|
}
|
|
if _, ok := new(big.Int).SetString(manifest.AvailableDeltaTotal, 10); !ok {
|
|
return errors.New("manifest available_delta_total is invalid")
|
|
}
|
|
if _, ok := new(big.Int).SetString(manifest.FrozenDeltaTotal, 10); !ok {
|
|
return errors.New("manifest frozen_delta_total is invalid")
|
|
}
|
|
if manifest.FirstCursor.UpdatedAtMS <= 0 || manifest.LastCursor.UpdatedAtMS <= 0 || strings.TrimSpace(manifest.FirstCursor.EventID) == "" || strings.TrimSpace(manifest.LastCursor.EventID) == "" {
|
|
return errors.New("manifest cursor is incomplete")
|
|
}
|
|
deliveredDay := time.UnixMilli(manifest.FirstCursor.UpdatedAtMS).UTC().Format("2006-01-02")
|
|
if time.UnixMilli(manifest.LastCursor.UpdatedAtMS).UTC().Format("2006-01-02") != deliveredDay {
|
|
return errors.New("manifest crosses UTC delivered day")
|
|
}
|
|
expectedSuffix := path.Join("app_code="+manifest.AppCode, "delivered_day="+deliveredDay, "batch_id="+manifest.BatchID, "data.ndjson.gz")
|
|
cleanObjectKey := path.Clean(manifest.DataObjectKey)
|
|
if cleanObjectKey != expectedSuffix && !strings.HasSuffix(cleanObjectKey, "/"+expectedSuffix) {
|
|
return errors.New("manifest data_object_key does not match app/day/batch identity")
|
|
}
|
|
return nil
|
|
}
|
|
|
|
func verifyDataFile(path string, manifest Manifest, limits Limits) ([]Record, error) {
|
|
file, err := os.Open(strings.TrimSpace(path))
|
|
if err != nil {
|
|
return nil, fmt.Errorf("open data object: %w", err)
|
|
}
|
|
defer file.Close()
|
|
info, err := file.Stat()
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
if info.Size() != manifest.CompressedBytes || info.Size() > limits.MaxCompressedBytes {
|
|
return nil, errors.New("compressed data size does not match manifest")
|
|
}
|
|
|
|
gzipHash := sha256.New()
|
|
gzipCRC := crc64.New(crc64.MakeTable(crc64.ECMA))
|
|
compressedReader := io.TeeReader(file, io.MultiWriter(gzipHash, gzipCRC))
|
|
gzipReader, err := gzip.NewReader(compressedReader)
|
|
if err != nil {
|
|
return nil, fmt.Errorf("open archived gzip: %w", err)
|
|
}
|
|
records, ndjsonSHA256, uncompressedBytes, totals, err := readAndValidateNDJSON(gzipReader, manifest, limits)
|
|
closeErr := gzipReader.Close()
|
|
_, drainErr := io.Copy(io.Discard, compressedReader)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
if closeErr != nil {
|
|
return nil, fmt.Errorf("close archived gzip: %w", closeErr)
|
|
}
|
|
if drainErr != nil {
|
|
return nil, fmt.Errorf("read archived gzip tail: %w", drainErr)
|
|
}
|
|
if hex.EncodeToString(gzipHash.Sum(nil)) != manifest.GzipSHA256 || strconv.FormatUint(gzipCRC.Sum64(), 10) != manifest.DataCRC64 {
|
|
return nil, errors.New("data object SHA256/CRC64 does not match manifest")
|
|
}
|
|
if ndjsonSHA256 != manifest.NDJSONSHA256 || uncompressedBytes != manifest.UncompressedBytes {
|
|
return nil, errors.New("NDJSON SHA256/byte count does not match manifest")
|
|
}
|
|
if totals.available.String() != manifest.AvailableDeltaTotal || totals.frozen.String() != manifest.FrozenDeltaTotal {
|
|
return nil, errors.New("NDJSON control totals do not match manifest")
|
|
}
|
|
return records, nil
|
|
}
|
|
|
|
type controlTotals struct {
|
|
available *big.Int
|
|
frozen *big.Int
|
|
}
|
|
|
|
func readAndValidateNDJSON(reader io.Reader, manifest Manifest, limits Limits) ([]Record, string, int64, controlTotals, error) {
|
|
buffered := bufio.NewReaderSize(reader, 64<<10)
|
|
hash := sha256.New()
|
|
records := make([]Record, 0, manifest.RowCount)
|
|
seen := make(map[string]struct{}, manifest.RowCount)
|
|
totals := controlTotals{available: new(big.Int), frozen: new(big.Int)}
|
|
var byteCount int64
|
|
var minCreated, maxCreated int64
|
|
for {
|
|
line, err := readRawLine(buffered, maxNDJSONLine)
|
|
if err == io.EOF {
|
|
break
|
|
}
|
|
if err != nil {
|
|
return nil, "", 0, totals, err
|
|
}
|
|
byteCount += int64(len(line))
|
|
if byteCount > limits.MaxUncompressedBytes {
|
|
return nil, "", 0, totals, errors.New("NDJSON exceeds uncompressed byte limit")
|
|
}
|
|
_, _ = hash.Write(line)
|
|
var record Record
|
|
if err := decodeStrictJSON(bytes.TrimSuffix(line, []byte{'\n'}), &record); err != nil {
|
|
return nil, "", 0, totals, fmt.Errorf("decode NDJSON row %d: %w", len(records)+1, err)
|
|
}
|
|
if err := validateRecord(record, manifest, records, seen); err != nil {
|
|
return nil, "", 0, totals, fmt.Errorf("validate NDJSON row %d: %w", len(records)+1, err)
|
|
}
|
|
if len(records) == 0 {
|
|
minCreated, maxCreated = record.CreatedAtMS, record.CreatedAtMS
|
|
} else {
|
|
if record.CreatedAtMS < minCreated {
|
|
minCreated = record.CreatedAtMS
|
|
}
|
|
if record.CreatedAtMS > maxCreated {
|
|
maxCreated = record.CreatedAtMS
|
|
}
|
|
}
|
|
totals.available.Add(totals.available, big.NewInt(record.AvailableDelta))
|
|
totals.frozen.Add(totals.frozen, big.NewInt(record.FrozenDelta))
|
|
records = append(records, record)
|
|
if len(records) > limits.MaxRows || len(records) > manifest.RowCount {
|
|
return nil, "", 0, totals, errors.New("NDJSON row count exceeds manifest or safety limit")
|
|
}
|
|
}
|
|
if len(records) != manifest.RowCount {
|
|
return nil, "", 0, totals, errors.New("NDJSON row count does not match manifest")
|
|
}
|
|
first, last := records[0], records[len(records)-1]
|
|
if manifest.FirstCursor != (Cursor{UpdatedAtMS: first.UpdatedAtMS, EventID: first.EventID}) || manifest.LastCursor != (Cursor{UpdatedAtMS: last.UpdatedAtMS, EventID: last.EventID}) || manifest.MinUpdatedAtMS != first.UpdatedAtMS || manifest.MaxUpdatedAtMS != last.UpdatedAtMS {
|
|
return nil, "", 0, totals, errors.New("NDJSON cursor bounds do not match manifest")
|
|
}
|
|
if manifest.MinCreatedAtMS != minCreated || manifest.MaxCreatedAtMS != maxCreated {
|
|
return nil, "", 0, totals, errors.New("NDJSON created_at bounds do not match manifest")
|
|
}
|
|
ndjsonSHA256 := hex.EncodeToString(hash.Sum(nil))
|
|
batchHash := sha256.Sum256([]byte(strings.Join([]string{
|
|
manifest.AppCode, strconv.Itoa(len(records)), strconv.FormatInt(first.UpdatedAtMS, 10), first.EventID,
|
|
strconv.FormatInt(last.UpdatedAtMS, 10), last.EventID, ndjsonSHA256,
|
|
}, "\n")))
|
|
if hex.EncodeToString(batchHash[:]) != manifest.BatchID {
|
|
return nil, "", 0, totals, errors.New("recomputed batch_id does not match manifest")
|
|
}
|
|
return records, ndjsonSHA256, byteCount, totals, nil
|
|
}
|
|
|
|
func validateRecord(record Record, manifest Manifest, prior []Record, seen map[string]struct{}) error {
|
|
if record.AppCode != manifest.AppCode || record.Status != "delivered" || strings.TrimSpace(record.EventID) == "" || strings.TrimSpace(record.EventType) == "" || record.UpdatedAtMS <= 0 || record.CreatedAtMS <= 0 {
|
|
return errors.New("row identity, state, or timestamp is invalid")
|
|
}
|
|
if !json.Valid([]byte(record.PayloadJSON)) {
|
|
return errors.New("payload_json is not valid JSON")
|
|
}
|
|
if _, exists := seen[record.EventID]; exists {
|
|
return errors.New("duplicate (app_code,event_id) archive PK")
|
|
}
|
|
seen[record.EventID] = struct{}{}
|
|
if len(prior) > 0 {
|
|
previous := prior[len(prior)-1]
|
|
if record.UpdatedAtMS < previous.UpdatedAtMS || (record.UpdatedAtMS == previous.UpdatedAtMS && record.EventID <= previous.EventID) {
|
|
return errors.New("rows are not strictly ordered by (updated_at_ms,event_id)")
|
|
}
|
|
}
|
|
return nil
|
|
}
|
|
|
|
func readRawLine(reader *bufio.Reader, maxBytes int) ([]byte, error) {
|
|
line := make([]byte, 0, 4096)
|
|
for {
|
|
fragment, err := reader.ReadSlice('\n')
|
|
if len(line)+len(fragment) > maxBytes {
|
|
return nil, fmt.Errorf("NDJSON line exceeds %d bytes", maxBytes)
|
|
}
|
|
line = append(line, fragment...)
|
|
switch err {
|
|
case nil:
|
|
return line, nil
|
|
case bufio.ErrBufferFull:
|
|
continue
|
|
case io.EOF:
|
|
if len(line) == 0 {
|
|
return nil, io.EOF
|
|
}
|
|
return nil, errors.New("NDJSON final row is missing newline terminator")
|
|
default:
|
|
return nil, err
|
|
}
|
|
}
|
|
}
|
|
|
|
func readBoundedFile(path string, maxBytes int64) ([]byte, error) {
|
|
file, err := os.Open(strings.TrimSpace(path))
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
defer file.Close()
|
|
body, err := io.ReadAll(io.LimitReader(file, maxBytes+1))
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
if int64(len(body)) > maxBytes || len(body) == 0 {
|
|
return nil, errors.New("file is empty or exceeds safety limit")
|
|
}
|
|
return body, nil
|
|
}
|
|
|
|
func decodeStrictJSON(body []byte, destination any) error {
|
|
decoder := json.NewDecoder(bytes.NewReader(body))
|
|
decoder.DisallowUnknownFields()
|
|
if err := decoder.Decode(destination); err != nil {
|
|
return err
|
|
}
|
|
var trailing any
|
|
if err := decoder.Decode(&trailing); err != io.EOF {
|
|
if err == nil {
|
|
return errors.New("JSON contains more than one value")
|
|
}
|
|
return err
|
|
}
|
|
return nil
|
|
}
|
|
|
|
func checksumBytes(body []byte) (string, string) {
|
|
sha := sha256.Sum256(body)
|
|
crc := crc64.Update(0, crc64.MakeTable(crc64.ECMA), body)
|
|
return hex.EncodeToString(sha[:]), strconv.FormatUint(crc, 10)
|
|
}
|
|
|
|
func validSHA256(value string) bool {
|
|
if len(value) != sha256.Size*2 {
|
|
return false
|
|
}
|
|
_, err := hex.DecodeString(value)
|
|
return err == nil
|
|
}
|
|
|
|
func validCRC64(value string) bool {
|
|
_, err := strconv.ParseUint(strings.TrimSpace(value), 10, 64)
|
|
return err == nil
|
|
}
|