fix: preserve internal gRPC error causes

This commit is contained in:
zhx 2026-07-17 12:17:04 +08:00
parent b7378c98db
commit e705d8cc44
5 changed files with 113 additions and 5 deletions

View File

@ -260,6 +260,10 @@ func UnaryServerInterceptor(service string) grpc.UnaryServerInterceptor {
if err != nil {
// gRPC access 日志是跨服务排障入口reason 只给稳定错误码message 保留服务端校验分支,便于定位 400/409 的真实拒绝原因。
attrs = append(attrs, slog.String("error_message", grpcErrorMessage(err)))
if cause := xerr.CauseFromGRPC(err); cause != nil {
// Internal/Unknown 对客户端固定脱敏成 "internal error"error_cause 是服务端唯一能看到原始错误链的地方。
attrs = append(attrs, slog.String("error_cause", cause.Error()))
}
}
attrs = appendServiceAttr(attrs, service)
attrs = append(attrs, commonAttrsFromPayload(request)...)

View File

@ -5,13 +5,17 @@ import (
"context"
"encoding/json"
"errors"
"fmt"
"net/http"
"net/http/httptest"
"strings"
"testing"
userv1 "hyapp.local/api/proto/user/v1"
walletv1 "hyapp.local/api/proto/wallet/v1"
"hyapp/pkg/xerr"
"google.golang.org/grpc"
"google.golang.org/grpc/codes"
"google.golang.org/grpc/status"
@ -127,6 +131,39 @@ func TestUnaryServerInterceptorLevelsAndRequestMeta(t *testing.T) {
}
}
// TestUnaryServerInterceptorLogsOriginalCauseForInternal 锁定 2026-07-12 排障改进:
// Internal 类错误客户端只见 "internal error",但 grpc_access ERROR 日志必须带脱敏前的原始错误链。
func TestUnaryServerInterceptorLogsOriginalCauseForInternal(t *testing.T) {
var output bytes.Buffer
if err := Init(Config{Service: "room-service", Env: "test", Format: "json", Output: &output}); err != nil {
t.Fatalf("Init failed: %v", err)
}
interceptor := UnaryServerInterceptor("room-service")
request := &userv1.GetUserRequest{Meta: &userv1.RequestMeta{RequestId: "req-grpc-2", AppCode: "lalu"}, UserId: 1001}
_, err := interceptor(context.Background(), request, &grpc.UnaryServerInfo{FullMethod: "/hyapp.room.v1.RoomCommandService/MicHeartbeat"}, func(ctx context.Context, req any) (any, error) {
return nil, xerr.ToGRPCError(fmt.Errorf("refresh mic presence: %w", errors.New("redis: connection refused")))
})
if err == nil {
t.Fatal("interceptor should return handler error")
}
if st, ok := status.FromError(err); !ok || st.Message() != "internal error" {
t.Fatalf("client-facing error must stay sanitized: %v", err)
}
entry := decodeOne(t, output.Bytes())
if entry["level"] != "ERROR" || entry["msg"] != "grpc_access" || entry["request_id"] != "req-grpc-2" {
t.Fatalf("unexpected grpc log entry: %+v", entry)
}
if entry["error_message"] != "internal error" {
t.Fatalf("error_message=%v, want sanitized message", entry["error_message"])
}
cause, _ := entry["error_cause"].(string)
if !strings.Contains(cause, "refresh mic presence") || !strings.Contains(cause, "redis: connection refused") {
t.Fatalf("error_cause=%q, want original error chain", cause)
}
}
func TestUnaryServerInterceptorExtractsTopLevelTraceFieldsWithoutBody(t *testing.T) {
var output bytes.Buffer
if err := Init(Config{Service: "wallet-service", Env: "test", Format: "json", Output: &output}); err != nil {

View File

@ -49,12 +49,43 @@ func ToGRPCError(err error) error {
Domain: errorDomain,
}
withDetails, detailErr := st.WithDetails(detail)
if detailErr != nil {
return st.Err()
if withDetails, detailErr := st.WithDetails(detail); detailErr == nil {
st = withDetails
}
return withDetails.Err()
return &grpcError{status: st, cause: err}
}
// grpcError 对 grpc-go 暴露脱敏后的 status同时保留脱敏前的原始错误链。
// 客户端从 GRPCStatus/Error 只能看到 catalog 文案;原始链只有服务端进程内
// 通过 CauseFromGRPC 可达,不会进 wire。
type grpcError struct {
status *status.Status
cause error
}
func (e *grpcError) Error() string {
return e.status.Err().Error()
}
// GRPCStatus 让 status.FromError 和 grpc-go server 识别脱敏 status而不是降级成 Unknown。
func (e *grpcError) GRPCStatus() *status.Status {
return e.status
}
// Unwrap 保留原始错误链,供服务端 errors.Is/As 与 access 日志定位根因。
func (e *grpcError) Unwrap() error {
return e.cause
}
// CauseFromGRPC 取回 ToGRPCError 脱敏前的原始错误链,仅用于服务端日志;
// 对已经是 gRPC status 的透传错误返回 nil。禁止把返回值写回给客户端。
func CauseFromGRPC(err error) error {
var typed *grpcError
if errors.As(err, &typed) {
return typed.cause
}
return nil
}
// ReasonFromGRPC 供 gateway 或测试从 gRPC error detail 中恢复稳定 reason。

View File

@ -2,6 +2,7 @@ package xerr
import (
"context"
"errors"
"fmt"
"go/ast"
"go/parser"
@ -54,6 +55,37 @@ func TestToGRPCErrorPreservesContextTermination(t *testing.T) {
}
}
// TestToGRPCErrorSanitizesButPreservesCause 锁定 2026-07-12 排障改进:
// 非 xerr 错误对客户端仍固定脱敏成 internal error但服务端能通过 CauseFromGRPC 取回原始错误链。
func TestToGRPCErrorSanitizesButPreservesCause(t *testing.T) {
rootCause := errors.New("redis: connection refused")
err := ToGRPCError(fmt.Errorf("acquire room lease: %w", rootCause))
st, ok := status.FromError(err)
if !ok {
t.Fatalf("expected grpc status")
}
if st.Code() != codes.Internal || st.Message() != "internal error" {
t.Fatalf("client-facing status must stay sanitized: code=%s message=%q", st.Code(), st.Message())
}
if err.Error() != st.Err().Error() {
t.Fatalf("Error() must not leak cause: %q", err.Error())
}
cause := CauseFromGRPC(err)
if cause == nil || !errors.Is(cause, rootCause) {
t.Fatalf("cause chain lost: got %v", cause)
}
// 透传已有 status 的错误没有被脱敏,不应报告 cause。
if got := CauseFromGRPC(status.Error(codes.NotFound, "room not found")); got != nil {
t.Fatalf("passthrough status should have no cause, got %v", got)
}
if got := ToGRPCError(err); got != err {
t.Fatalf("ToGRPCError must be idempotent on wrapped errors")
}
}
func TestCatalogCoversDeclaredCodes(t *testing.T) {
declared := declaredCodesFromSource(t)

View File

@ -12,6 +12,7 @@ import (
"hyapp/pkg/appcode"
"hyapp/pkg/apptracking"
"hyapp/pkg/logx"
mysqlstorage "hyapp/services/statistics-service/internal/storage/mysql"
)
@ -44,7 +45,10 @@ func newQueryHTTPServer(addr string, repo *mysqlstorage.Repository) (*queryHTTPS
mux.HandleFunc("/internal/v1/statistics/self-game/events", s.selfGameEvents)
mux.HandleFunc("/internal/v1/statistics/self-games/overview", s.selfGameOverview)
mux.HandleFunc("/internal/v1/statistics/activity-templates/data", s.activityTemplateData)
s.server = &http.Server{Handler: mux, ReadHeaderTimeout: 2 * time.Second}
// 内部查询/ingest API 之前完全没有 access 日志,线上只能靠调用方日志反推请求是否到达。
// statistics 没有自己的 request_id 链路,日志里 request_id 留空即可。
handler := logx.HTTPMiddleware("statistics-service", func(context.Context) string { return "" }, mux)
s.server = &http.Server{Handler: handler, ReadHeaderTimeout: 2 * time.Second}
return s, nil
}