package gateway import ( "context" "io" "net/http" "strings" "testing" "github.com/enterpilot/gomodel/internal/core" "github.com/stretchr/testify/require" ) // stubFailoverResolver returns a fixed selector list regardless of input. type stubFailoverResolver struct { selectors []core.ModelSelector } func (s stubFailoverResolver) ResolveFailovers(_ *core.RequestModelResolution, _ core.Operation) []core.ModelSelector { return s.selectors } func failoverTestFixture() (*InferenceOrchestrator, *core.Workflow) { o := &InferenceOrchestrator{ failoverResolver: stubFailoverResolver{ selectors: []core.ModelSelector{{Provider: "openai", Model: "gpt-5"}}, }, } workflow := &core.Workflow{ Endpoint: core.EndpointDescriptor{Operation: core.OperationChatCompletions}, Resolution: &core.RequestModelResolution{}, } return o, workflow } // A canceled context means the client is gone; failover must not sweep providers // (doing so wastes attempts and trips healthy providers' circuit breakers). func TestTryFailoverResponseSkipsWhenContextCanceled(t *testing.T) { o, workflow := failoverTestFixture() ctx, cancel := context.WithCancel(context.Background()) cancel() primaryErr := core.NewProviderError("openai", http.StatusBadGateway, "context canceled", context.Canceled) called := false call := func(core.ModelSelector, string, string) (string, string, error) { called = true return "", "", core.NewProviderError("openai", http.StatusBadGateway, "unexpected failover call", nil) } _, meta, err := tryFailoverResponse(ctx, o, workflow, "openai/gpt-4o", "openai", primaryErr, nil, call) require.False(t, called) require.False(t, meta.UsedFailover) require.ErrorIs(t, err, context.Canceled) } // The guard is scoped to a done context: a live request still attempts failover. func TestTryFailoverResponseAttemptsWhenContextLive(t *testing.T) { o, workflow := failoverTestFixture() primaryErr := core.NewProviderError("openai", http.StatusInternalServerError, "primary boom", nil) called := false call := func(core.ModelSelector, string, string) (string, string, error) { called = true return "ok", "openai", nil } resp, meta, err := tryFailoverResponse(context.Background(), o, workflow, "openai/gpt-4o", "openai", primaryErr, nil, call) require.True(t, called) require.True(t, meta.UsedFailover) require.NoError(t, err) require.Equal(t, "ok", resp) } // blockingRouteGate refuses the listed qualified models. type blockingRouteGate struct { blocked map[string]bool } func (g blockingRouteGate) RouteAvailable(_, model string) bool { return !g.blocked[model] } // A failover target whose provider or model is rate-saturated is skipped, so // the sweep moves on to the next candidate instead of burning its attempt. func TestTryFailoverResponseSkipsRateLimitedTargets(t *testing.T) { o, workflow := failoverTestFixture() o.failoverResolver = stubFailoverResolver{selectors: []core.ModelSelector{ {Provider: "openai", Model: "gpt-5"}, {Provider: "anthropic", Model: "claude"}, }} o.routeGate = blockingRouteGate{blocked: map[string]bool{"openai/gpt-5": true}} primaryErr := core.NewProviderError("openai", http.StatusInternalServerError, "primary boom", nil) var attempted []string call := func(selector core.ModelSelector, _, _ string) (string, string, error) { attempted = append(attempted, selector.QualifiedModel()) return "ok", "anthropic", nil } resp, meta, err := tryFailoverResponse(context.Background(), o, workflow, "openai/gpt-4o", "openai", primaryErr, nil, call) require.Len(t, attempted, 1) require.Equal(t, "anthropic/claude", attempted[0]) require.True(t, meta.UsedFailover) require.NoError(t, err) require.Equal(t, "ok", resp) require.Equal(t, "anthropic/claude", meta.FailoverModel) } // The stream sweep shares the route-gate skip with the response sweep. func TestTryFailoverStreamSkipsRateLimitedTargets(t *testing.T) { o, workflow := failoverTestFixture() o.failoverResolver = stubFailoverResolver{selectors: []core.ModelSelector{ {Provider: "openai", Model: "gpt-5"}, {Provider: "anthropic", Model: "claude"}, }} o.routeGate = blockingRouteGate{blocked: map[string]bool{"openai/gpt-5": true}} primaryErr := core.NewProviderError("openai", http.StatusInternalServerError, "primary boom", nil) var attempted []string call := func(selector core.ModelSelector, _, _ string) (io.ReadCloser, string, string, error) { attempted = append(attempted, selector.QualifiedModel()) return io.NopCloser(strings.NewReader("data")), "anthropic", "claude", nil } stream, meta, err := tryFailoverStream(context.Background(), o, workflow, "openai/gpt-4o", "openai", primaryErr, call) require.Len(t, attempted, 1) require.Equal(t, "anthropic/claude", attempted[0]) require.NoError(t, err) require.NotNil(t, stream) require.Equal(t, "anthropic/claude", meta.FailoverModel) stream.Close() } // namedRouteProvider resolves each model's provider type and instance name // from its qualified prefix, the way a configured router names instances. type namedRouteProvider struct { core.RoutableProvider } func (namedRouteProvider) GetProviderType(model string) string { provider, _, _ := strings.Cut(model, "/") return provider } func (namedRouteProvider) GetProviderName(model string) string { provider, _, _ := strings.Cut(model, "/") return provider } // overBudgetGate refuses the listed provider names. type overBudgetGate struct { exhausted map[string]bool } func (g overBudgetGate) ProviderWithinBudget(_ context.Context, providerName string) bool { return !g.exhausted[providerName] } func (g overBudgetGate) FailoverWithinBudget(ctx context.Context, _, target string) bool { return g.ProviderWithinBudget(ctx, target) } // A failover target whose provider budget is spent is skipped on both sweeps, // so failover can never route around an exhausted provider budget. func TestTryFailoverSkipsOverBudgetProviders(t *testing.T) { newFixture := func() (*InferenceOrchestrator, *core.Workflow) { o, workflow := failoverTestFixture() o.failoverResolver = stubFailoverResolver{selectors: []core.ModelSelector{ {Provider: "openai", Model: "gpt-5"}, {Provider: "anthropic", Model: "claude"}, }} o.provider = namedRouteProvider{} o.budgetGate = overBudgetGate{exhausted: map[string]bool{"openai": true}} return o, workflow } primaryErr := core.NewProviderError("openai", http.StatusInternalServerError, "primary boom", nil) t.Run("response", func(t *testing.T) { o, workflow := newFixture() var attempted []string call := func(selector core.ModelSelector, _, _ string) (string, string, error) { attempted = append(attempted, selector.QualifiedModel()) return "ok", "anthropic", nil } _, meta, err := tryFailoverResponse(context.Background(), o, workflow, "openai/gpt-4o", "openai", primaryErr, nil, call) require.NoError(t, err) require.Equal(t, []string{"anthropic/claude"}, attempted) require.Equal(t, "anthropic/claude", meta.FailoverModel) }) t.Run("stream", func(t *testing.T) { o, workflow := newFixture() var attempted []string call := func(selector core.ModelSelector, _, _ string) (io.ReadCloser, string, string, error) { attempted = append(attempted, selector.QualifiedModel()) return io.NopCloser(strings.NewReader("data")), "anthropic", "claude", nil } stream, meta, err := tryFailoverStream(context.Background(), o, workflow, "openai/gpt-4o", "openai", primaryErr, call) require.NoError(t, err) require.NotNil(t, stream) defer stream.Close() require.Equal(t, []string{"anthropic/claude"}, attempted) require.Equal(t, "anthropic/claude", meta.FailoverModel) }) } // consumerBudgetGate refuses a billable target after a non-billable primary, // the way the budget service does once the user-path budget is spent. type consumerBudgetGate struct { nonBillable map[string]bool asked [][2]string } func (g *consumerBudgetGate) ProviderWithinBudget(context.Context, string) bool { return true } func (g *consumerBudgetGate) FailoverWithinBudget(_ context.Context, primary, target string) bool { g.asked = append(g.asked, [2]string{primary, target}) return !g.nonBillable[primary] || g.nonBillable[target] } // The failover sweep hands the gate the primary provider, so the consumer // budgets admission skipped for a non-billable primary gate the billable // targets, while another non-billable target stays open. func TestTryFailoverGatesTargetsOnConsumerBudgetsAfterNonBillablePrimary(t *testing.T) { o, workflow := failoverTestFixture() o.failoverResolver = stubFailoverResolver{selectors: []core.ModelSelector{ {Provider: "openai", Model: "gpt-5"}, {Provider: "byok-backup", Model: "gpt-5"}, }} o.provider = namedRouteProvider{} gate := &consumerBudgetGate{nonBillable: map[string]bool{"byok": true, "byok-backup": true}} o.budgetGate = gate workflow.Resolution.ProviderName = "byok" primaryErr := core.NewProviderError("byok", http.StatusInternalServerError, "primary boom", nil) var attempted []string call := func(selector core.ModelSelector, _, _ string) (string, string, error) { attempted = append(attempted, selector.QualifiedModel()) return "ok", "openai", nil } _, meta, err := tryFailoverResponse(context.Background(), o, workflow, "byok/gpt-5", "byok", primaryErr, nil, call) require.NoError(t, err) require.Equal(t, []string{"byok-backup/gpt-5"}, attempted) require.Equal(t, "byok-backup/gpt-5", meta.FailoverModel) require.Equal(t, [][2]string{{"byok", "openai"}, {"byok", "byok-backup"}}, gate.asked) } // A workflow with budgets disabled skips budget checks at admission, so // failover must not gate targets on provider budgets either. func TestTryFailoverIgnoresBudgetsWhenWorkflowDisablesThem(t *testing.T) { o, workflow := failoverTestFixture() o.provider = namedRouteProvider{} o.budgetGate = overBudgetGate{exhausted: map[string]bool{"openai": true}} workflow.Policy = &core.ResolvedWorkflowPolicy{ VersionID: "workflow-v1", Features: core.WorkflowFeatures{Failover: true, Budget: false}, } primaryErr := core.NewProviderError("openai", http.StatusInternalServerError, "primary boom", nil) var attempted []string call := func(selector core.ModelSelector, _, _ string) (string, string, error) { attempted = append(attempted, selector.QualifiedModel()) return "ok", "openai", nil } _, meta, err := tryFailoverResponse(context.Background(), o, workflow, "openai/gpt-4o", "openai", primaryErr, nil, call) require.NoError(t, err) require.Equal(t, []string{"openai/gpt-5"}, attempted) require.True(t, meta.UsedFailover) } func TestTryFailoverStreamSkipsWhenContextCanceled(t *testing.T) { o, workflow := failoverTestFixture() ctx, cancel := context.WithCancel(context.Background()) cancel() primaryErr := core.NewProviderError("openai", http.StatusBadGateway, "context canceled", context.Canceled) called := false call := func(core.ModelSelector, string, string) (io.ReadCloser, string, string, error) { called = true return nil, "", "", core.NewProviderError("openai", http.StatusBadGateway, "unexpected failover call", nil) } stream, _, err := tryFailoverStream(ctx, o, workflow, "openai/gpt-4o", "openai", primaryErr, call) require.False(t, called) require.Nil(t, stream) require.ErrorIs(t, err, context.Canceled) } func TestShouldAttemptFailover(t *testing.T) { tests := []struct { name string status int message string want bool }{ // Server-side and rate-limit failures always fall back. {"500 server error", http.StatusInternalServerError, "internal error", true}, {"429 rate limited", http.StatusTooManyRequests, "slow down", true}, // Model-availability phrasing falls back regardless of status code. {"model not found message", http.StatusBadRequest, "model gpt-9 does not exist", true}, // 404 with availability phrasing (no literal "model") still falls back: // providers report retired/unavailable models this way. {"availability 404", http.StatusNotFound, "Claude Fable 5 is not available. Please use Opus 4.8.", true}, {"deprecated 404", http.StatusNotFound, "this checkpoint is deprecated", true}, // 404s without availability phrasing must NOT fall back — they are // genuine routing/endpoint misses, not model failures. {"generic endpoint 404", http.StatusNotFound, "endpoint not found", false}, {"route 404", http.StatusNotFound, "404 page not found", false}, {"unknown path 404", http.StatusNotFound, "no route for /v1/foo", false}, // Aggregator providers relay transient failures of their own upstream // as 4xx client errors; those must fall back (issue #605). {"opencode upstream 400", http.StatusBadRequest, "Error from provider (Console Go): Upstream request failed", true}, {"upstream timeout 400", http.StatusBadRequest, "upstream timed out", true}, {"upstream unavailable 400", http.StatusBadRequest, "upstream provider is currently unavailable", true}, // Mentioning an upstream without failure phrasing is a genuine // validation error and must NOT fall back. {"upstream capability 400", http.StatusBadRequest, "parameter tools is not supported by the upstream provider", false}, // A plain client error without availability phrasing is not retried. {"plain 400", http.StatusBadRequest, "invalid request", false}, } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { err := core.NewProviderError("anthropic", tt.status, tt.message, nil) got := ShouldAttemptFailover(err) require.Equal(t, tt.want, got) }) } } // A saturated primary route must never reach the provider (the upstream would // serve it and defeat the gateway's limit); its stored 429 seeds the sweep. func TestExecuteTranslatedSkipsSaturatedPrimaryAndFailsOver(t *testing.T) { o, workflow := failoverTestFixture() saturated := core.NewRateLimitError("ratelimit", "rate limit exceeded for provider openai").WithCode("rate_limit_exceeded") ctx := core.WithPrimaryRouteSaturated(context.Background(), saturated) var calls []string resp, meta, err := executeTranslatedWithFailover( ctx, o, workflow, "req", "openai/gpt-4o", "openai", func(req string, selector core.ModelSelector) string { return selector.QualifiedModel() }, func(_ context.Context, req string, _ string) (string, string, error) { calls = append(calls, req) require.NotEqual(t, "req", req) return "ok", "openai", nil }, ) require.Len(t, calls, 1) require.Equal(t, "openai/gpt-5", calls[0]) require.True(t, meta.UsedFailover) require.NoError(t, err) require.Equal(t, "ok", resp) require.Equal(t, "openai/gpt-5", meta.FailoverModel) } // When every failover target is also unavailable, the client receives the // original rate-limit rejection, not a provider error. func TestExecuteTranslatedSaturatedPrimarySurfaces429WhenNoTargetRemains(t *testing.T) { o, workflow := failoverTestFixture() o.routeGate = blockingRouteGate{blocked: map[string]bool{"openai/gpt-5": true}} saturated := core.NewRateLimitError("ratelimit", "rate limit exceeded for provider openai").WithCode("rate_limit_exceeded") ctx := core.WithPrimaryRouteSaturated(context.Background(), saturated) _, meta, err := executeTranslatedWithFailover( ctx, o, workflow, "req", "openai/gpt-4o", "openai", func(req string, selector core.ModelSelector) string { return selector.QualifiedModel() }, func(_ context.Context, _ string, _ string) (string, string, error) { t.Fatal("no provider call expected: primary saturated, failover gated") return "", "", nil }, ) require.False(t, meta.UsedFailover) var gatewayErr *core.GatewayError require.ErrorAs(t, err, &gatewayErr) require.Equal(t, http.StatusTooManyRequests, gatewayErr.HTTPStatusCode()) } // The stream path shares the skip. func TestStreamTranslatedSkipsSaturatedPrimaryAndFailsOver(t *testing.T) { o, workflow := failoverTestFixture() saturated := core.NewRateLimitError("ratelimit", "rate limit exceeded for model gpt-4o").WithCode("rate_limit_exceeded") ctx := core.WithPrimaryRouteSaturated(context.Background(), saturated) var calls []string stream, meta, err := streamTranslatedProviderRequest( o, ctx, workflow, "req", "openai/gpt-4o", "openai", "openai", "openai", "gpt-4o", func(req string, selector core.ModelSelector) string { return selector.QualifiedModel() }, func(_ context.Context, req string, _ string) (io.ReadCloser, error) { calls = append(calls, req) require.NotEqual(t, "req", req) return io.NopCloser(strings.NewReader("data")), nil }, ) require.Len(t, calls, 1) require.Equal(t, "openai/gpt-5", calls[0]) require.NoError(t, err) require.NotNil(t, stream) require.True(t, meta.UsedFailover) require.Equal(t, "openai/gpt-5", meta.FailoverModel) stream.Close() }