yumi-golang/internal/service/taskcenter/archive_object.go

package taskcenter

import (
	"bytes"
	"compress/gzip"
	"context"
	"crypto/md5"
	"crypto/sha256"
	"encoding/base64"
	"encoding/hex"
	"fmt"
	"io"
	"net/http"
	"strconv"
	"strings"

	"github.com/tencentyun/cos-go-sdk-v5"
)

const (
	maxArchiveCOSObjectBytes       = 128 << 20
	maxArchiveCOSUncompressedBytes = 256 << 20
)

type verifiedArchiveObject struct {
	COSKey         string
	SHA256         string
	ETag           string
	CompressedSize int64
	Header         http.Header
}

// putAndHeadArchiveObject 将本地摘要同时写入 COS metadata，并在 PUT 后重新 HEAD。
// 只有服务端返回的长度和摘要都与本地一致才继续写 manifest/outbox，防止网关成功响应、
// 错误对象 key 或被并发覆盖造成“数据库显示已归档但对象不可恢复”。
func putAndHeadArchiveObject(
	ctx context.Context,
	client *cos.Client,
	cosKey string,
	body []byte,
	metadata map[string]string,
) (verifiedArchiveObject, error) {
	sha := archiveSHA256(body)
	headers := make(http.Header)
	headers.Set("x-cos-meta-sha256", sha)
	for key, value := range metadata {
		headers.Set("x-cos-meta-"+strings.ToLower(strings.TrimSpace(key)), strings.TrimSpace(value))
	}
	response, err := client.Object.Put(ctx, cosKey, bytes.NewReader(body), &cos.ObjectPutOptions{
		ObjectPutHeaderOptions: &cos.ObjectPutHeaderOptions{
			ContentType:   "application/gzip",
			ContentMD5:    archiveContentMD5(body),
			ContentLength: int64(len(body)),
			XCosMetaXXX:   &headers,
		},
	})
	if response != nil && response.Body != nil {
		_ = response.Body.Close()
	}
	if err != nil {
		return verifiedArchiveObject{}, err
	}
	verified, err := headAndVerifyArchiveObject(ctx, client, cosKey, sha, int64(len(body)), true)
	if err != nil {
		return verifiedArchiveObject{}, err
	}
	for key, expected := range metadata {
		headerName := "x-cos-meta-" + strings.ToLower(strings.TrimSpace(key))
		if actual := strings.TrimSpace(verified.Header.Get(headerName)); actual != strings.TrimSpace(expected) {
			return verifiedArchiveObject{}, fmt.Errorf("COS HEAD %s metadata %s does not match", cosKey, headerName)
		}
	}
	return verified, nil
}

func headAndVerifyArchiveObject(
	ctx context.Context,
	client *cos.Client,
	cosKey, expectedSHA string,
	expectedSize int64,
	requireSHAMetadata bool,
) (verifiedArchiveObject, error) {
	response, err := client.Object.Head(ctx, cosKey, nil)
	if response != nil && response.Body != nil {
		defer response.Body.Close()
	}
	if err != nil {
		return verifiedArchiveObject{}, err
	}
	if response == nil || response.Response == nil {
		return verifiedArchiveObject{}, fmt.Errorf("COS HEAD %s returned no response", cosKey)
	}
	size, err := strconv.ParseInt(strings.TrimSpace(response.Header.Get("Content-Length")), 10, 64)
	if err != nil || size < 0 {
		return verifiedArchiveObject{}, fmt.Errorf("COS HEAD %s has invalid content length", cosKey)
	}
	if expectedSize >= 0 && size != expectedSize {
		return verifiedArchiveObject{}, fmt.Errorf("COS HEAD %s size %d does not match %d", cosKey, size, expectedSize)
	}
	remoteSHA := strings.TrimSpace(response.Header.Get("x-cos-meta-sha256"))
	if requireSHAMetadata && remoteSHA == "" {
		return verifiedArchiveObject{}, fmt.Errorf("COS HEAD %s is missing required sha256 metadata", cosKey)
	}
	if remoteSHA != "" && expectedSHA != "" && !strings.EqualFold(remoteSHA, expectedSHA) {
		return verifiedArchiveObject{}, fmt.Errorf("COS HEAD %s sha256 metadata mismatch", cosKey)
	}
	return verifiedArchiveObject{
		COSKey:         cosKey,
		SHA256:         expectedSHA,
		ETag:           strings.Trim(strings.TrimSpace(response.Header.Get("ETag")), `"`),
		CompressedSize: size,
		Header:         response.Header.Clone(),
	}, nil
}

// downloadArchiveObject 强制 identity 传输并限制压缩对象大小。清理路径必须拿到对象原始
// gzip 字节才能同时验证 SHA256 和 JSONL，不能只相信 HEAD/ETag。
func downloadArchiveObject(ctx context.Context, client *cos.Client, cosKey string) ([]byte, http.Header, error) {
	headers := make(http.Header)
	headers.Set("Accept-Encoding", "identity")
	response, err := client.Object.Get(ctx, cosKey, &cos.ObjectGetOptions{XOptionHeader: &headers})
	if err != nil {
		return nil, nil, err
	}
	if response == nil || response.Response == nil || response.Body == nil {
		return nil, nil, fmt.Errorf("COS GET %s returned no body", cosKey)
	}
	defer response.Body.Close()
	body, err := io.ReadAll(io.LimitReader(response.Body, maxArchiveCOSObjectBytes+1))
	if err != nil {
		return nil, response.Header, err
	}
	if len(body) > maxArchiveCOSObjectBytes {
		return nil, response.Header, fmt.Errorf("COS object %s exceeds %d bytes", cosKey, maxArchiveCOSObjectBytes)
	}
	return body, response.Header, nil
}

// archiveObjectLines 解压并返回规范化 JSONL 行。历史对象与新对象都由同一个编码器写入；
// 空行被忽略，但单行上限和总解压大小都有硬门槛，避免损坏对象造成内存放大。
func archiveObjectLines(body []byte) ([]string, error) {
	if len(body) < 2 || body[0] != 0x1f || body[1] != 0x8b {
		return nil, fmt.Errorf("archive object is not gzip encoded")
	}
	reader, err := gzip.NewReader(bytes.NewReader(body))
	if err != nil {
		return nil, err
	}
	defer reader.Close()
	uncompressed, err := io.ReadAll(io.LimitReader(reader, maxArchiveCOSUncompressedBytes+1))
	if err != nil {
		return nil, err
	}
	if len(uncompressed) > maxArchiveCOSUncompressedBytes {
		return nil, fmt.Errorf("archive object expands beyond %d bytes", maxArchiveCOSUncompressedBytes)
	}
	parts := bytes.Split(uncompressed, []byte{'\n'})
	lines := make([]string, 0, len(parts))
	for _, part := range parts {
		line := strings.TrimSpace(string(part))
		if line != "" {
			lines = append(lines, line)
		}
	}
	return lines, nil
}

func archiveSHA256(value []byte) string {
	sum := sha256.Sum256(value)
	return hex.EncodeToString(sum[:])
}

func archiveContentMD5(value []byte) string {
	sum := md5.Sum(value)
	return base64.StdEncoding.EncodeToString(sum[:])
}

func archiveLineSHA256(value string) string {
	return archiveSHA256([]byte(strings.TrimSpace(value)))
}