package taskcenter import ( "bytes" "compress/gzip" "context" "crypto/md5" "crypto/sha256" "encoding/base64" "encoding/hex" "fmt" "io" "net/http" "strconv" "strings" "github.com/tencentyun/cos-go-sdk-v5" ) const ( maxArchiveCOSObjectBytes = 128 << 20 maxArchiveCOSUncompressedBytes = 256 << 20 ) type verifiedArchiveObject struct { COSKey string SHA256 string ETag string CompressedSize int64 Header http.Header } // putAndHeadArchiveObject 将本地摘要同时写入 COS metadata,并在 PUT 后重新 HEAD。 // 只有服务端返回的长度和摘要都与本地一致才继续写 manifest/outbox,防止网关成功响应、 // 错误对象 key 或被并发覆盖造成“数据库显示已归档但对象不可恢复”。 func putAndHeadArchiveObject( ctx context.Context, client *cos.Client, cosKey string, body []byte, metadata map[string]string, ) (verifiedArchiveObject, error) { sha := archiveSHA256(body) headers := make(http.Header) headers.Set("x-cos-meta-sha256", sha) for key, value := range metadata { headers.Set("x-cos-meta-"+strings.ToLower(strings.TrimSpace(key)), strings.TrimSpace(value)) } response, err := client.Object.Put(ctx, cosKey, bytes.NewReader(body), &cos.ObjectPutOptions{ ObjectPutHeaderOptions: &cos.ObjectPutHeaderOptions{ ContentType: "application/gzip", ContentMD5: archiveContentMD5(body), ContentLength: int64(len(body)), XCosMetaXXX: &headers, }, }) if response != nil && response.Body != nil { _ = response.Body.Close() } if err != nil { return verifiedArchiveObject{}, err } verified, err := headAndVerifyArchiveObject(ctx, client, cosKey, sha, int64(len(body)), true) if err != nil { return verifiedArchiveObject{}, err } for key, expected := range metadata { headerName := "x-cos-meta-" + strings.ToLower(strings.TrimSpace(key)) if actual := strings.TrimSpace(verified.Header.Get(headerName)); actual != strings.TrimSpace(expected) { return verifiedArchiveObject{}, fmt.Errorf("COS HEAD %s metadata %s does not match", cosKey, headerName) } } return verified, nil } func headAndVerifyArchiveObject( ctx context.Context, client *cos.Client, cosKey, expectedSHA string, expectedSize int64, requireSHAMetadata bool, ) (verifiedArchiveObject, error) { response, err := client.Object.Head(ctx, cosKey, nil) if response != nil && response.Body != nil { defer response.Body.Close() } if err != nil { return verifiedArchiveObject{}, err } if response == nil || response.Response == nil { return verifiedArchiveObject{}, fmt.Errorf("COS HEAD %s returned no response", cosKey) } size, err := strconv.ParseInt(strings.TrimSpace(response.Header.Get("Content-Length")), 10, 64) if err != nil || size < 0 { return verifiedArchiveObject{}, fmt.Errorf("COS HEAD %s has invalid content length", cosKey) } if expectedSize >= 0 && size != expectedSize { return verifiedArchiveObject{}, fmt.Errorf("COS HEAD %s size %d does not match %d", cosKey, size, expectedSize) } remoteSHA := strings.TrimSpace(response.Header.Get("x-cos-meta-sha256")) if requireSHAMetadata && remoteSHA == "" { return verifiedArchiveObject{}, fmt.Errorf("COS HEAD %s is missing required sha256 metadata", cosKey) } if remoteSHA != "" && expectedSHA != "" && !strings.EqualFold(remoteSHA, expectedSHA) { return verifiedArchiveObject{}, fmt.Errorf("COS HEAD %s sha256 metadata mismatch", cosKey) } return verifiedArchiveObject{ COSKey: cosKey, SHA256: expectedSHA, ETag: strings.Trim(strings.TrimSpace(response.Header.Get("ETag")), `"`), CompressedSize: size, Header: response.Header.Clone(), }, nil } // downloadArchiveObject 强制 identity 传输并限制压缩对象大小。清理路径必须拿到对象原始 // gzip 字节才能同时验证 SHA256 和 JSONL,不能只相信 HEAD/ETag。 func downloadArchiveObject(ctx context.Context, client *cos.Client, cosKey string) ([]byte, http.Header, error) { headers := make(http.Header) headers.Set("Accept-Encoding", "identity") response, err := client.Object.Get(ctx, cosKey, &cos.ObjectGetOptions{XOptionHeader: &headers}) if err != nil { return nil, nil, err } if response == nil || response.Response == nil || response.Body == nil { return nil, nil, fmt.Errorf("COS GET %s returned no body", cosKey) } defer response.Body.Close() body, err := io.ReadAll(io.LimitReader(response.Body, maxArchiveCOSObjectBytes+1)) if err != nil { return nil, response.Header, err } if len(body) > maxArchiveCOSObjectBytes { return nil, response.Header, fmt.Errorf("COS object %s exceeds %d bytes", cosKey, maxArchiveCOSObjectBytes) } return body, response.Header, nil } // archiveObjectLines 解压并返回规范化 JSONL 行。历史对象与新对象都由同一个编码器写入; // 空行被忽略,但单行上限和总解压大小都有硬门槛,避免损坏对象造成内存放大。 func archiveObjectLines(body []byte) ([]string, error) { if len(body) < 2 || body[0] != 0x1f || body[1] != 0x8b { return nil, fmt.Errorf("archive object is not gzip encoded") } reader, err := gzip.NewReader(bytes.NewReader(body)) if err != nil { return nil, err } defer reader.Close() uncompressed, err := io.ReadAll(io.LimitReader(reader, maxArchiveCOSUncompressedBytes+1)) if err != nil { return nil, err } if len(uncompressed) > maxArchiveCOSUncompressedBytes { return nil, fmt.Errorf("archive object expands beyond %d bytes", maxArchiveCOSUncompressedBytes) } parts := bytes.Split(uncompressed, []byte{'\n'}) lines := make([]string, 0, len(parts)) for _, part := range parts { line := strings.TrimSpace(string(part)) if line != "" { lines = append(lines, line) } } return lines, nil } func archiveSHA256(value []byte) string { sum := sha256.Sum256(value) return hex.EncodeToString(sum[:]) } func archiveContentMD5(value []byte) string { sum := md5.Sum(value) return base64.StdEncoding.EncodeToString(sum[:]) } func archiveLineSHA256(value string) string { return archiveSHA256([]byte(strings.TrimSpace(value))) }