fix: preserve internal gRPC error causes
This commit is contained in:
parent
b7378c98db
commit
e705d8cc44
@ -260,6 +260,10 @@ func UnaryServerInterceptor(service string) grpc.UnaryServerInterceptor {
|
|||||||
if err != nil {
|
if err != nil {
|
||||||
// gRPC access 日志是跨服务排障入口;reason 只给稳定错误码,message 保留服务端校验分支,便于定位 400/409 的真实拒绝原因。
|
// gRPC access 日志是跨服务排障入口;reason 只给稳定错误码,message 保留服务端校验分支,便于定位 400/409 的真实拒绝原因。
|
||||||
attrs = append(attrs, slog.String("error_message", grpcErrorMessage(err)))
|
attrs = append(attrs, slog.String("error_message", grpcErrorMessage(err)))
|
||||||
|
if cause := xerr.CauseFromGRPC(err); cause != nil {
|
||||||
|
// Internal/Unknown 对客户端固定脱敏成 "internal error",error_cause 是服务端唯一能看到原始错误链的地方。
|
||||||
|
attrs = append(attrs, slog.String("error_cause", cause.Error()))
|
||||||
|
}
|
||||||
}
|
}
|
||||||
attrs = appendServiceAttr(attrs, service)
|
attrs = appendServiceAttr(attrs, service)
|
||||||
attrs = append(attrs, commonAttrsFromPayload(request)...)
|
attrs = append(attrs, commonAttrsFromPayload(request)...)
|
||||||
|
|||||||
@ -5,13 +5,17 @@ import (
|
|||||||
"context"
|
"context"
|
||||||
"encoding/json"
|
"encoding/json"
|
||||||
"errors"
|
"errors"
|
||||||
|
"fmt"
|
||||||
"net/http"
|
"net/http"
|
||||||
"net/http/httptest"
|
"net/http/httptest"
|
||||||
|
"strings"
|
||||||
"testing"
|
"testing"
|
||||||
|
|
||||||
userv1 "hyapp.local/api/proto/user/v1"
|
userv1 "hyapp.local/api/proto/user/v1"
|
||||||
walletv1 "hyapp.local/api/proto/wallet/v1"
|
walletv1 "hyapp.local/api/proto/wallet/v1"
|
||||||
|
|
||||||
|
"hyapp/pkg/xerr"
|
||||||
|
|
||||||
"google.golang.org/grpc"
|
"google.golang.org/grpc"
|
||||||
"google.golang.org/grpc/codes"
|
"google.golang.org/grpc/codes"
|
||||||
"google.golang.org/grpc/status"
|
"google.golang.org/grpc/status"
|
||||||
@ -127,6 +131,39 @@ func TestUnaryServerInterceptorLevelsAndRequestMeta(t *testing.T) {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// TestUnaryServerInterceptorLogsOriginalCauseForInternal 锁定 2026-07-12 排障改进:
|
||||||
|
// Internal 类错误客户端只见 "internal error",但 grpc_access ERROR 日志必须带脱敏前的原始错误链。
|
||||||
|
func TestUnaryServerInterceptorLogsOriginalCauseForInternal(t *testing.T) {
|
||||||
|
var output bytes.Buffer
|
||||||
|
if err := Init(Config{Service: "room-service", Env: "test", Format: "json", Output: &output}); err != nil {
|
||||||
|
t.Fatalf("Init failed: %v", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
interceptor := UnaryServerInterceptor("room-service")
|
||||||
|
request := &userv1.GetUserRequest{Meta: &userv1.RequestMeta{RequestId: "req-grpc-2", AppCode: "lalu"}, UserId: 1001}
|
||||||
|
_, err := interceptor(context.Background(), request, &grpc.UnaryServerInfo{FullMethod: "/hyapp.room.v1.RoomCommandService/MicHeartbeat"}, func(ctx context.Context, req any) (any, error) {
|
||||||
|
return nil, xerr.ToGRPCError(fmt.Errorf("refresh mic presence: %w", errors.New("redis: connection refused")))
|
||||||
|
})
|
||||||
|
if err == nil {
|
||||||
|
t.Fatal("interceptor should return handler error")
|
||||||
|
}
|
||||||
|
if st, ok := status.FromError(err); !ok || st.Message() != "internal error" {
|
||||||
|
t.Fatalf("client-facing error must stay sanitized: %v", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
entry := decodeOne(t, output.Bytes())
|
||||||
|
if entry["level"] != "ERROR" || entry["msg"] != "grpc_access" || entry["request_id"] != "req-grpc-2" {
|
||||||
|
t.Fatalf("unexpected grpc log entry: %+v", entry)
|
||||||
|
}
|
||||||
|
if entry["error_message"] != "internal error" {
|
||||||
|
t.Fatalf("error_message=%v, want sanitized message", entry["error_message"])
|
||||||
|
}
|
||||||
|
cause, _ := entry["error_cause"].(string)
|
||||||
|
if !strings.Contains(cause, "refresh mic presence") || !strings.Contains(cause, "redis: connection refused") {
|
||||||
|
t.Fatalf("error_cause=%q, want original error chain", cause)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
func TestUnaryServerInterceptorExtractsTopLevelTraceFieldsWithoutBody(t *testing.T) {
|
func TestUnaryServerInterceptorExtractsTopLevelTraceFieldsWithoutBody(t *testing.T) {
|
||||||
var output bytes.Buffer
|
var output bytes.Buffer
|
||||||
if err := Init(Config{Service: "wallet-service", Env: "test", Format: "json", Output: &output}); err != nil {
|
if err := Init(Config{Service: "wallet-service", Env: "test", Format: "json", Output: &output}); err != nil {
|
||||||
|
|||||||
@ -49,12 +49,43 @@ func ToGRPCError(err error) error {
|
|||||||
Domain: errorDomain,
|
Domain: errorDomain,
|
||||||
}
|
}
|
||||||
|
|
||||||
withDetails, detailErr := st.WithDetails(detail)
|
if withDetails, detailErr := st.WithDetails(detail); detailErr == nil {
|
||||||
if detailErr != nil {
|
st = withDetails
|
||||||
return st.Err()
|
|
||||||
}
|
}
|
||||||
|
|
||||||
return withDetails.Err()
|
return &grpcError{status: st, cause: err}
|
||||||
|
}
|
||||||
|
|
||||||
|
// grpcError 对 grpc-go 暴露脱敏后的 status,同时保留脱敏前的原始错误链。
|
||||||
|
// 客户端从 GRPCStatus/Error 只能看到 catalog 文案;原始链只有服务端进程内
|
||||||
|
// 通过 CauseFromGRPC 可达,不会进 wire。
|
||||||
|
type grpcError struct {
|
||||||
|
status *status.Status
|
||||||
|
cause error
|
||||||
|
}
|
||||||
|
|
||||||
|
func (e *grpcError) Error() string {
|
||||||
|
return e.status.Err().Error()
|
||||||
|
}
|
||||||
|
|
||||||
|
// GRPCStatus 让 status.FromError 和 grpc-go server 识别脱敏 status,而不是降级成 Unknown。
|
||||||
|
func (e *grpcError) GRPCStatus() *status.Status {
|
||||||
|
return e.status
|
||||||
|
}
|
||||||
|
|
||||||
|
// Unwrap 保留原始错误链,供服务端 errors.Is/As 与 access 日志定位根因。
|
||||||
|
func (e *grpcError) Unwrap() error {
|
||||||
|
return e.cause
|
||||||
|
}
|
||||||
|
|
||||||
|
// CauseFromGRPC 取回 ToGRPCError 脱敏前的原始错误链,仅用于服务端日志;
|
||||||
|
// 对已经是 gRPC status 的透传错误返回 nil。禁止把返回值写回给客户端。
|
||||||
|
func CauseFromGRPC(err error) error {
|
||||||
|
var typed *grpcError
|
||||||
|
if errors.As(err, &typed) {
|
||||||
|
return typed.cause
|
||||||
|
}
|
||||||
|
return nil
|
||||||
}
|
}
|
||||||
|
|
||||||
// ReasonFromGRPC 供 gateway 或测试从 gRPC error detail 中恢复稳定 reason。
|
// ReasonFromGRPC 供 gateway 或测试从 gRPC error detail 中恢复稳定 reason。
|
||||||
|
|||||||
@ -2,6 +2,7 @@ package xerr
|
|||||||
|
|
||||||
import (
|
import (
|
||||||
"context"
|
"context"
|
||||||
|
"errors"
|
||||||
"fmt"
|
"fmt"
|
||||||
"go/ast"
|
"go/ast"
|
||||||
"go/parser"
|
"go/parser"
|
||||||
@ -54,6 +55,37 @@ func TestToGRPCErrorPreservesContextTermination(t *testing.T) {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// TestToGRPCErrorSanitizesButPreservesCause 锁定 2026-07-12 排障改进:
|
||||||
|
// 非 xerr 错误对客户端仍固定脱敏成 internal error,但服务端能通过 CauseFromGRPC 取回原始错误链。
|
||||||
|
func TestToGRPCErrorSanitizesButPreservesCause(t *testing.T) {
|
||||||
|
rootCause := errors.New("redis: connection refused")
|
||||||
|
err := ToGRPCError(fmt.Errorf("acquire room lease: %w", rootCause))
|
||||||
|
|
||||||
|
st, ok := status.FromError(err)
|
||||||
|
if !ok {
|
||||||
|
t.Fatalf("expected grpc status")
|
||||||
|
}
|
||||||
|
if st.Code() != codes.Internal || st.Message() != "internal error" {
|
||||||
|
t.Fatalf("client-facing status must stay sanitized: code=%s message=%q", st.Code(), st.Message())
|
||||||
|
}
|
||||||
|
if err.Error() != st.Err().Error() {
|
||||||
|
t.Fatalf("Error() must not leak cause: %q", err.Error())
|
||||||
|
}
|
||||||
|
|
||||||
|
cause := CauseFromGRPC(err)
|
||||||
|
if cause == nil || !errors.Is(cause, rootCause) {
|
||||||
|
t.Fatalf("cause chain lost: got %v", cause)
|
||||||
|
}
|
||||||
|
|
||||||
|
// 透传已有 status 的错误没有被脱敏,不应报告 cause。
|
||||||
|
if got := CauseFromGRPC(status.Error(codes.NotFound, "room not found")); got != nil {
|
||||||
|
t.Fatalf("passthrough status should have no cause, got %v", got)
|
||||||
|
}
|
||||||
|
if got := ToGRPCError(err); got != err {
|
||||||
|
t.Fatalf("ToGRPCError must be idempotent on wrapped errors")
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
func TestCatalogCoversDeclaredCodes(t *testing.T) {
|
func TestCatalogCoversDeclaredCodes(t *testing.T) {
|
||||||
declared := declaredCodesFromSource(t)
|
declared := declaredCodesFromSource(t)
|
||||||
|
|
||||||
|
|||||||
@ -12,6 +12,7 @@ import (
|
|||||||
|
|
||||||
"hyapp/pkg/appcode"
|
"hyapp/pkg/appcode"
|
||||||
"hyapp/pkg/apptracking"
|
"hyapp/pkg/apptracking"
|
||||||
|
"hyapp/pkg/logx"
|
||||||
mysqlstorage "hyapp/services/statistics-service/internal/storage/mysql"
|
mysqlstorage "hyapp/services/statistics-service/internal/storage/mysql"
|
||||||
)
|
)
|
||||||
|
|
||||||
@ -44,7 +45,10 @@ func newQueryHTTPServer(addr string, repo *mysqlstorage.Repository) (*queryHTTPS
|
|||||||
mux.HandleFunc("/internal/v1/statistics/self-game/events", s.selfGameEvents)
|
mux.HandleFunc("/internal/v1/statistics/self-game/events", s.selfGameEvents)
|
||||||
mux.HandleFunc("/internal/v1/statistics/self-games/overview", s.selfGameOverview)
|
mux.HandleFunc("/internal/v1/statistics/self-games/overview", s.selfGameOverview)
|
||||||
mux.HandleFunc("/internal/v1/statistics/activity-templates/data", s.activityTemplateData)
|
mux.HandleFunc("/internal/v1/statistics/activity-templates/data", s.activityTemplateData)
|
||||||
s.server = &http.Server{Handler: mux, ReadHeaderTimeout: 2 * time.Second}
|
// 内部查询/ingest API 之前完全没有 access 日志,线上只能靠调用方日志反推请求是否到达。
|
||||||
|
// statistics 没有自己的 request_id 链路,日志里 request_id 留空即可。
|
||||||
|
handler := logx.HTTPMiddleware("statistics-service", func(context.Context) string { return "" }, mux)
|
||||||
|
s.server = &http.Server{Handler: handler, ReadHeaderTimeout: 2 * time.Second}
|
||||||
return s, nil
|
return s, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|||||||
Loading…
x
Reference in New Issue
Block a user