193 lines
7.2 KiB
Go
193 lines
7.2 KiB
Go
package app
|
||
|
||
import (
|
||
"compress/gzip"
|
||
"context"
|
||
"crypto/sha256"
|
||
"encoding/hex"
|
||
"errors"
|
||
"fmt"
|
||
"io"
|
||
"net/http"
|
||
"net/url"
|
||
"os"
|
||
"strings"
|
||
|
||
cos "github.com/tencentyun/cos-go-sdk-v5"
|
||
"hyapp/services/wallet-service/internal/config"
|
||
)
|
||
|
||
var errWalletOutboxArchiveObjectExists = errors.New("wallet outbox archive object already exists")
|
||
|
||
// walletOutboxArchiveCOSStore 只使用私有 bucket API，不生成公网 URL，也不在对象上设置 public-read ACL。
|
||
type walletOutboxArchiveCOSStore struct {
|
||
client *cos.Client
|
||
}
|
||
|
||
func newWalletOutboxArchiveCOSStore(cfg config.OutboxArchiveCOSConfig, transport http.RoundTripper) (*walletOutboxArchiveCOSStore, error) {
|
||
bucketURL, err := url.Parse(fmt.Sprintf("https://%s.cos.%s.myqcloud.com", strings.TrimSpace(cfg.Bucket), strings.TrimSpace(cfg.Region)))
|
||
if err != nil {
|
||
return nil, fmt.Errorf("build wallet outbox archive COS url: %w", err)
|
||
}
|
||
if transport == nil {
|
||
transport = http.DefaultTransport
|
||
}
|
||
var authorizationTransport http.RoundTripper
|
||
if cfg.CredentialSource == "cvm_role" {
|
||
authorizationTransport = &cvmRoleAuthorizationTransport{provider: newCVMCAMCredentialProvider(cfg.CAMRoleName), transport: transport}
|
||
} else {
|
||
authorizationTransport = &cos.AuthorizationTransport{
|
||
SecretID: cfg.SecretID,
|
||
SecretKey: cfg.SecretKey,
|
||
SessionToken: cfg.SessionToken,
|
||
Transport: transport,
|
||
}
|
||
}
|
||
return &walletOutboxArchiveCOSStore{client: cos.NewClient(&cos.BaseURL{BucketURL: bucketURL}, &http.Client{
|
||
Timeout: cfg.HTTPTimeout,
|
||
Transport: authorizationTransport,
|
||
})}, nil
|
||
}
|
||
|
||
func (s *walletOutboxArchiveCOSStore) putGzipAndVerify(ctx context.Context, key string, filePath string, sizeBytes int64, gzipSHA256 string, ndjsonSHA256 string) (string, error) {
|
||
file, err := os.Open(filePath)
|
||
if err != nil {
|
||
return "", err
|
||
}
|
||
defer file.Close()
|
||
// 对象本体已经是 gzip 文件，不设置 Content-Encoding。Go HTTP Transport 会自动解压带
|
||
// Content-Encoding:gzip 的响应，导致后续无法同时核对“压缩对象 hash”和“解压 NDJSON hash”。
|
||
putCRC64, err := s.put(ctx, key, file, sizeBytes, "application/gzip", "", gzipSHA256)
|
||
if err != nil && !errors.Is(err, errWalletOutboxArchiveObjectExists) {
|
||
return "", err
|
||
}
|
||
crc64, err := s.verifyHead(ctx, key, sizeBytes, gzipSHA256, putCRC64)
|
||
if err != nil {
|
||
return "", err
|
||
}
|
||
if err := s.verifyGzipReadback(ctx, key, gzipSHA256, ndjsonSHA256); err != nil {
|
||
return "", err
|
||
}
|
||
return crc64, nil
|
||
}
|
||
|
||
func (s *walletOutboxArchiveCOSStore) putBytesAndVerify(ctx context.Context, key string, body []byte, contentType string, expectedSHA256 string) (string, error) {
|
||
putCRC64, err := s.put(ctx, key, strings.NewReader(string(body)), int64(len(body)), contentType, "", expectedSHA256)
|
||
if err != nil && !errors.Is(err, errWalletOutboxArchiveObjectExists) {
|
||
return "", err
|
||
}
|
||
crc64, err := s.verifyHead(ctx, key, int64(len(body)), expectedSHA256, putCRC64)
|
||
if err != nil {
|
||
return "", err
|
||
}
|
||
response, err := s.client.Object.Get(ctx, key, nil)
|
||
if err != nil {
|
||
return "", err
|
||
}
|
||
defer response.Body.Close()
|
||
hash := sha256.New()
|
||
if _, err := io.Copy(hash, response.Body); err != nil {
|
||
return "", err
|
||
}
|
||
if actual := hex.EncodeToString(hash.Sum(nil)); actual != expectedSHA256 {
|
||
return "", fmt.Errorf("wallet outbox archive COS readback hash mismatch for %s", key)
|
||
}
|
||
return crc64, nil
|
||
}
|
||
|
||
func (s *walletOutboxArchiveCOSStore) put(ctx context.Context, key string, reader io.Reader, sizeBytes int64, contentType string, contentEncoding string, contentSHA256 string) (string, error) {
|
||
if s == nil || s.client == nil || strings.TrimSpace(key) == "" || reader == nil || sizeBytes <= 0 {
|
||
return "", fmt.Errorf("wallet outbox archive COS put input is invalid")
|
||
}
|
||
metadata := make(http.Header)
|
||
metadata.Set("x-cos-meta-sha256", contentSHA256)
|
||
optionHeaders := make(http.Header)
|
||
// 归档键由内容指纹决定；禁止覆盖可避免 bucket versioning 在并发/重试时生成重复版本。
|
||
optionHeaders.Set("x-cos-forbid-overwrite", "true")
|
||
options := &cos.ObjectPutOptions{ObjectPutHeaderOptions: &cos.ObjectPutHeaderOptions{
|
||
ContentLength: sizeBytes,
|
||
ContentType: contentType,
|
||
ContentEncoding: contentEncoding,
|
||
XCosMetaXXX: &metadata,
|
||
XOptionHeader: &optionHeaders,
|
||
XCosStorageClass: "STANDARD",
|
||
XCosServerSideEncryption: "AES256",
|
||
}}
|
||
response, err := s.client.Object.Put(ctx, strings.TrimLeft(key, "/"), reader, options)
|
||
crc64 := ""
|
||
if response != nil {
|
||
crc64 = strings.TrimSpace(response.Header.Get("x-cos-hash-crc64ecma"))
|
||
}
|
||
if response != nil && response.Body != nil {
|
||
_ = response.Body.Close()
|
||
}
|
||
if err != nil {
|
||
var cosErr *cos.ErrorResponse
|
||
if errors.As(err, &cosErr) && cosErr.Response != nil && (cosErr.Response.StatusCode == http.StatusConflict || cosErr.Response.StatusCode == http.StatusPreconditionFailed) {
|
||
return "", errWalletOutboxArchiveObjectExists
|
||
}
|
||
return "", err
|
||
}
|
||
if crc64 == "" {
|
||
return "", fmt.Errorf("wallet outbox archive COS PUT response has no CRC64 for %s", key)
|
||
}
|
||
return crc64, nil
|
||
}
|
||
|
||
func (s *walletOutboxArchiveCOSStore) verifyHead(ctx context.Context, key string, expectedSize int64, expectedSHA256 string, expectedCRC64 string) (string, error) {
|
||
response, err := s.client.Object.Head(ctx, key, nil)
|
||
if err != nil {
|
||
return "", err
|
||
}
|
||
if response.Body != nil {
|
||
defer response.Body.Close()
|
||
}
|
||
if response.ContentLength != expectedSize || response.Header.Get("x-cos-meta-sha256") != expectedSHA256 {
|
||
return "", fmt.Errorf("wallet outbox archive COS HEAD mismatch for %s", key)
|
||
}
|
||
crc64 := strings.TrimSpace(response.Header.Get("x-cos-hash-crc64ecma"))
|
||
if crc64 == "" || (expectedCRC64 != "" && crc64 != expectedCRC64) {
|
||
return "", fmt.Errorf("wallet outbox archive COS HEAD CRC64 mismatch for %s", key)
|
||
}
|
||
// SSE-COS 是归档对象的强制约束；响应缺失加密标记时不能写 verified receipt。
|
||
if !strings.EqualFold(response.Header.Get("x-cos-server-side-encryption"), "AES256") {
|
||
return "", fmt.Errorf("wallet outbox archive COS object is not SSE-COS encrypted: %s", key)
|
||
}
|
||
return crc64, nil
|
||
}
|
||
|
||
func (s *walletOutboxArchiveCOSStore) verifyGzipReadback(ctx context.Context, key string, expectedGzipSHA256 string, expectedNDJSONSHA256 string) error {
|
||
response, err := s.client.Object.Get(ctx, key, nil)
|
||
if err != nil {
|
||
return err
|
||
}
|
||
defer response.Body.Close()
|
||
gzipHash := sha256.New()
|
||
compressedReader := io.TeeReader(response.Body, gzipHash)
|
||
gzipReader, err := gzip.NewReader(compressedReader)
|
||
if err != nil {
|
||
return err
|
||
}
|
||
ndjsonHash := sha256.New()
|
||
_, copyErr := io.Copy(ndjsonHash, gzipReader)
|
||
closeErr := gzipReader.Close()
|
||
// gzip.Reader 可能在内部缓冲区之后留下响应尾部；继续读完才能验证整个压缩对象的 hash。
|
||
_, drainErr := io.Copy(io.Discard, compressedReader)
|
||
if copyErr != nil {
|
||
return copyErr
|
||
}
|
||
if closeErr != nil {
|
||
return closeErr
|
||
}
|
||
if drainErr != nil {
|
||
return drainErr
|
||
}
|
||
if actual := hex.EncodeToString(gzipHash.Sum(nil)); actual != expectedGzipSHA256 {
|
||
return fmt.Errorf("wallet outbox archive COS gzip hash mismatch for %s", key)
|
||
}
|
||
if actual := hex.EncodeToString(ndjsonHash.Sum(nil)); actual != expectedNDJSONSHA256 {
|
||
return fmt.Errorf("wallet outbox archive COS ndjson hash mismatch for %s", key)
|
||
}
|
||
return nil
|
||
}
|