111 lines
2.5 KiB
Go
111 lines
2.5 KiB
Go
package upstream
|
|
|
|
import (
|
|
"context"
|
|
"strings"
|
|
"testing"
|
|
"time"
|
|
|
|
"google.golang.org/grpc/codes"
|
|
"google.golang.org/grpc/status"
|
|
)
|
|
|
|
func TestCallRetriesToSecondEndpoint(t *testing.T) {
|
|
t.Parallel()
|
|
|
|
pool := &Pool{
|
|
name: "user",
|
|
timeout: time.Second,
|
|
maxAttempts: 2,
|
|
retryBackoff: 0,
|
|
failureThreshold: 3,
|
|
openTimeout: 5 * time.Second,
|
|
healthCacheTTL: 2 * time.Second,
|
|
endpoints: []*endpoint{
|
|
{handle: Handle{Target: "10.0.11.17:9001"}},
|
|
{handle: Handle{Target: "10.0.12.6:9001"}},
|
|
},
|
|
}
|
|
|
|
var attempts []string
|
|
result, err := Call(context.Background(), pool, func(_ context.Context, handle Handle) (string, error) {
|
|
attempts = append(attempts, handle.Target)
|
|
if handle.Target == "10.0.11.17:9001" {
|
|
return "", status.Error(codes.Unavailable, "first endpoint down")
|
|
}
|
|
return "ok", nil
|
|
})
|
|
if err != nil {
|
|
t.Fatalf("Call returned error: %v", err)
|
|
}
|
|
if result != "ok" {
|
|
t.Fatalf("unexpected result: %s", result)
|
|
}
|
|
if len(attempts) != 2 {
|
|
t.Fatalf("unexpected attempts: %#v", attempts)
|
|
}
|
|
if attempts[0] != "10.0.11.17:9001" || attempts[1] != "10.0.12.6:9001" {
|
|
t.Fatalf("unexpected attempts order: %#v", attempts)
|
|
}
|
|
}
|
|
|
|
func TestCallOpensCircuitAfterRepeatedFailures(t *testing.T) {
|
|
t.Parallel()
|
|
|
|
pool := &Pool{
|
|
name: "pay",
|
|
timeout: time.Second,
|
|
maxAttempts: 1,
|
|
retryBackoff: 0,
|
|
failureThreshold: 2,
|
|
openTimeout: time.Minute,
|
|
healthCacheTTL: 2 * time.Second,
|
|
endpoints: []*endpoint{
|
|
{handle: Handle{Target: "10.0.22.13:9002"}},
|
|
},
|
|
}
|
|
|
|
invocations := 0
|
|
fail := func(_ context.Context, _ Handle) (string, error) {
|
|
invocations++
|
|
return "", status.Error(codes.Unavailable, "down")
|
|
}
|
|
|
|
for i := 0; i < 2; i++ {
|
|
_, err := Call(context.Background(), pool, fail)
|
|
if err == nil {
|
|
t.Fatal("Call returned nil error")
|
|
}
|
|
}
|
|
|
|
_, err := Call(context.Background(), pool, fail)
|
|
if err == nil {
|
|
t.Fatal("expected circuit-open error")
|
|
}
|
|
if !strings.Contains(err.Error(), "circuit open") {
|
|
t.Fatalf("unexpected error: %v", err)
|
|
}
|
|
if invocations != 2 {
|
|
t.Fatalf("unexpected invocation count: %d", invocations)
|
|
}
|
|
}
|
|
|
|
func TestReadyUsesCachedHealthyState(t *testing.T) {
|
|
t.Parallel()
|
|
|
|
pool := &Pool{
|
|
name: "user",
|
|
timeout: time.Second,
|
|
maxAttempts: 2,
|
|
healthCacheTTL: 5 * time.Second,
|
|
endpoints: []*endpoint{
|
|
{handle: Handle{Target: "10.0.11.17:9001"}},
|
|
},
|
|
}
|
|
|
|
pool.endpoints[0].recordSuccess(time.Now())
|
|
if err := pool.Ready(context.Background()); err != nil {
|
|
t.Fatalf("Ready returned error: %v", err)
|
|
}
|
|
}
|