/* * Copyright (c) 2026, WSO2 LLC. (https://www.wso2.com). * * WSO2 LLC. licenses this file to you under the Apache License, * Version 2.0 (the "License"); you may not use this file except * in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, * software distributed under the License is distributed on an * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY * KIND, either express or implied. See the License for the * specific language governing permissions and limitations * under the License. */ // Package platformgateway contains platform-gateway-specific integration steps. package platformgateway import ( "context" _ "embed" "encoding/json" "errors" "fmt" "math" "net" "net/http" "net/url" "reflect" "slices" "sort" "strconv" "strings" "syscall" "time" "github.com/cucumber/godog" "gopkg.in/yaml.v3" "github.com/wso2/api-platform/tests/framework/core/cleanup" frameworkruntime "github.com/wso2/api-platform/tests/framework/core/runtime" "github.com/wso2/api-platform/tests/framework/core/util/httpx" "github.com/wso2/api-platform/tests/framework/core/util/retry" "github.com/wso2/api-platform/tests/framework/core/util/tcontext" stepscommon "github.com/wso2/gateway-controllers/tests/it/common/steps/common" ) // API base paths, kept in sync with the gateway's OpenAPI documents. const ( ManagementBasePath = "/api/management/v1" managementBasePathV11 = "/api/management/v0.9" adminBasePath = "/api/admin/v1" adminBasePathV11 = "/api/admin/v0.9" gatewaySpecVersion = "gateway.api-platform.wso2.com/v1" gatewaySpecVersionV11 = "gateway.api-platform.wso2.com/v1alpha1" // backendSpecVersion is the apiVersion of the Backend kind, which is new and uses the // Choreo Labs API group from its first release. backendSpecVersion = "gateway.choreolabs.com/v1" ) // The headers MCP 2026-07-28 requires a client to mirror its request into. A server that declares // that revision rejects a request whose headers and body disagree, so a test that alters one of // these is describing a request the upstream itself will refuse. const ( mcpProtocolVersionHeader = "MCP-Protocol-Version" mcpMethodHeader = "Mcp-Method" mcpNameHeader = "Mcp-Name" ) // ManagementBasePathForVersion returns the management API base path for a Gateway release. func ManagementBasePathForVersion(version string) string { if usesLegacyGatewayContract(version) { return managementBasePathV11 } return ManagementBasePath } // AwaitDeletionApplied waits, on a Gateway release with the xDS snapshot race, until the // controller has applied the deletion of handle, so the next change reaches it in a later poll. // // Releases before 1.2.0 apply each change from an event the controller polls for every three // seconds, and rebuild the route snapshot on a goroutine per event, without a lock (fixed in // 1.2.0 by "fix(xds): add mutex to SnapshotManager.UpdateSnapshot"). A deletion and a creation // applied in the same poll race, and the deletion's rebuild can publish last, leaving the new // resource without a route. The controller's config dump drops the handle when its poll applies // the deletion, so once the handle is gone a later creation is applied in a later poll. A kind // the dump does not list never rebuilds the route snapshot, so it passes at once. Later releases // are not polled. func AwaitDeletionApplied( ctx context.Context, client *httpx.Client, adminURL, version, authorization, handle string, ) error { if !usesLegacyGatewayContract(version) { return nil } if client == nil || strings.TrimSpace(adminURL) == "" || strings.TrimSpace(handle) == "" { return errors.New("awaiting a deletion: client, admin URL and handle are required") } url := adminURL + adminBasePathForVersion(version) + "/config_dump" quoted := `"` + handle + `"` accept := func(r *httpx.Response) bool { return r != nil && r.Succeeded() && !strings.Contains(r.Text(), quoted) } last, err := retry.Until(ctx, retry.Options{Interval: 100 * time.Millisecond}, func(ctx context.Context) (*httpx.Response, error) { return client.Do(ctx, httpx.Request{ Method: http.MethodGet, URL: url, Headers: map[string]string{"Authorization": authorization}, }, 0, 0) }, accept, ) return awaited(last, err, accept, fmt.Sprintf("the config dump still lists the deleted %q", handle)) } func adminBasePathForVersion(version string) string { if usesLegacyGatewayContract(version) { return adminBasePathV11 } return adminBasePath } func gatewaySpecVersionForVersion(version string) string { if usesLegacyGatewayContract(version) { return gatewaySpecVersionV11 } return gatewaySpecVersion } // gatewayMCPUpstreamPathForVersion returns the path configured on the MCP testbench upstream. // Gateway 1.1 and older append the requested MCP path themselves; later releases forward to the configured path. func gatewayMCPUpstreamPathForVersion(version string) string { version = strings.TrimPrefix(strings.TrimSpace(version), "v") version, _, _ = strings.Cut(version, "-") parts := strings.Split(version, ".") if len(parts) != 3 { return "/mcp" } major, majorErr := strconv.Atoi(parts[0]) minor, minorErr := strconv.Atoi(parts[1]) patch, patchErr := strconv.Atoi(parts[2]) if majorErr != nil || minorErr != nil || patchErr != nil || major < 0 || minor < 0 || patch < 0 { return "/mcp" } if major < 1 || (major == 1 && (minor < 1 || (minor == 1 && patch == 0))) { return "" } return "/mcp" } func gatewayVersion(topo *frameworkruntime.Topology) string { version, err := topo.ComponentVersion("platform-gateway") if err != nil { return "" } return version } // requireGatewayVersion skips the scenario when the gateway under test does not satisfy the // constraint, so one feature file can hold a scenario that needs a newer gateway than the // rest of its runner. A runner tag gates a whole runner; this gates one scenario. func (g *Gateway) requireGatewayVersion(_ context.Context, operator, version string) error { ok, err := gatewaySatisfies(g.topo, operator, version) if err != nil { return err } if !ok { return godog.ErrSkip } return nil } // gatewaySatisfies reports whether the gateway in topo satisfies "", with // the same rules as a runner's gateway-version tag: a release compares on major.minor.patch, // and the latest or a source-built gateway is newer than every release, so it satisfies only // ">" and ">=". func gatewaySatisfies(topo *frameworkruntime.Topology, operator, version string) (bool, error) { want, err := parseReleaseVersion(version) if err != nil { return false, err } if topo == nil || topo.Block == nil { return false, errors.New("gateway version: the topology has no block") } for _, component := range topo.Block.Components { if component.Def == nil || component.Def.Name != "platform-gateway" { continue } if component.BuildFromSource || strings.EqualFold(strings.TrimSpace(component.Version), "latest") { return operator == ">" || operator == ">=", nil } have, err := parseReleaseVersion(component.Version) if err != nil { return false, fmt.Errorf("gateway version: %w", err) } cmp := 0 for i := range have { if have[i] != want[i] { cmp = 1 if have[i] < want[i] { cmp = -1 } break } } switch operator { case ">": return cmp > 0, nil case ">=": return cmp >= 0, nil case "<": return cmp < 0, nil case "<=": return cmp <= 0, nil case "=", "==": return cmp == 0, nil default: return false, fmt.Errorf("gateway version: unsupported operator %q", operator) } } return false, errors.New("gateway version: the block declares no platform-gateway") } // parseReleaseVersion parses major.minor.patch, or a four-part released tag such as 1.2.0.6, // which compares on its first three parts. func parseReleaseVersion(raw string) ([3]uint64, error) { var out [3]uint64 parts := strings.Split(strings.TrimPrefix(strings.TrimSpace(raw), "v"), ".") if len(parts) != 3 && len(parts) != 4 { return out, fmt.Errorf("%q is not a release version (major.minor.patch)", raw) } for i, part := range parts[:3] { value, err := strconv.ParseUint(part, 10, 64) if err != nil { return out, fmt.Errorf("%q is not a release version (major.minor.patch)", raw) } out[i] = value } return out, nil } func gatewayAdminBasePath(topo *frameworkruntime.Topology) string { return adminBasePathForVersion(gatewayVersion(topo)) } func gatewayManagementBasePath(topo *frameworkruntime.Topology) string { return ManagementBasePathForVersion(gatewayVersion(topo)) } // Context keys this suite publishes. const ( keyLastAPIName = "lastApiName" // keyGatewaySpecVersion is the resource apiVersion for the selected Gateway release. // Feature tables use ${CTX:gatewaySpecVersion} rather than hard-coding a release contract. keyGatewaySpecVersion = "gatewaySpecVersion" // keyBackendSpecVersion is the Backend kind's apiVersion; feature tables use // ${CTX:backendSpecVersion}. keyBackendSpecVersion = "backendSpecVersion" // keyGatewayMCPUpstreamPath is the path configured on the MCP testbench upstream. // Gateway 1.1 and older append /mcp to that URL; later releases do not. keyGatewayMCPUpstreamPath = "gatewayMCPUpstreamPath" // keyTestbenchPartition is this block's partition segment. A stateful testbench service is // addressed as //… so that concurrent blocks cannot see each other's state, and an // upstream URL written in a feature table has no other way to name it. keyTestbenchPartition = "testbenchPartition" // Request-shaping state set by one step and read by the next. It lives in the SCENARIO // scope rather than on the Gateway struct because one Gateway serves a whole block, and // its runners run in parallel — a struct field here would be shared mutable state // across concurrent runners, which is the exact defect this framework exists to remove. // What this scenario expects the config dump to show: handle -> should-be-present. // A deploy sets true, a delete sets false. Both are waits — see awaitDumpConsistent. keyDumpExpectations = "dumpExpectations" ) // headerRateLimitRemaining is how every rate-limit policy reports what is left of a quota. // // ONE header, whatever the policy counts: advanced-ratelimit counts requests and // token-based-ratelimit counts tokens, but both report through this name. Lookup is // case-insensitive per RFC 7230, which is the only reason features spelling it // "X-Ratelimit-Remaining" also pass — the casing here is the product's. const headerRateLimitRemaining = "X-RateLimit-Remaining" // sendUntilQuotaRemaining waits for the configured rate-limit remainder. func (g *Gateway) sendUntilQuotaRemaining(ctx context.Context, method, path string, want int) error { return g.sendUntilHeader(ctx, method, path, headerRateLimitRemaining, strconv.Itoa(want)) } // Base provides the suite-wide request and scenario mechanics used by gateway steps. type Base interface { FeatureRoot() string GatewayURL(path string) (string, error) GatewayURLAt(ordinalWord, path string) (string, error) ScenarioHeaders(ctx context.Context) map[string]string InvokeWith(ctx context.Context, method, path string, headers map[string]string, body []byte) error RequestHost(ctx context.Context) string ResetRequest(ctx context.Context) error SendUntilHeader(ctx context.Context, method, path, name, want string) error } type Gateway struct { base Base topo *frameworkruntime.Topology funnel *httpx.Funnel featureRoot string } func (g *Gateway) gatewayURL(path string) (string, error) { return g.base.GatewayURL(path) } func (g *Gateway) gatewayURLAt(ordinal, path string) (string, error) { return g.base.GatewayURLAt(ordinal, path) } func (g *Gateway) scenarioHeaders(ctx context.Context) map[string]string { return g.base.ScenarioHeaders(ctx) } func (g *Gateway) invokeWith(ctx context.Context, method, path string, headers map[string]string, body []byte) error { return g.base.InvokeWith(ctx, method, path, headers, body) } func (g *Gateway) requestHost(ctx context.Context) string { return g.base.RequestHost(ctx) } func (g *Gateway) resetRequest(ctx context.Context) error { return g.base.ResetRequest(ctx) } func (g *Gateway) sendUntilHeader(ctx context.Context, method, path, name, want string) error { return g.base.SendUntilHeader(ctx, method, path, name, want) } type lazyResource struct { ID string `json:"id"` Type string `json:"resource_type"` Resource map[string]any `json:"resource"` } type lazyDump struct { ResourcesByType map[string][]lazyResource `json:"resources_by_type"` } type configDump struct { Lazy lazyDump `json:"lazy_resources"` RouteMetadata routeMetadataDump `json:"route_metadata"` PolicyChains policyChainsDump `json:"policy_chains"` } type configDumpSchema uint8 const ( configDumpCurrent configDumpSchema = iota configDumpLegacyRouteKey ) // configDumpSchemaFor selects the policy-engine config-dump relationship used by a Gateway // release. Legacy Gateway releases and the released Gateway 1.2.0 store each policy chain // under its route key; the checked-out 1.2.0-SNAPSHOT and later builds link route metadata to // a chain through chain_key. func configDumpSchemaFor(version string) configDumpSchema { version = strings.TrimPrefix(strings.TrimSpace(version), "v") if usesLegacyGatewayContract(version) || version == "1.2.0" { return configDumpLegacyRouteKey } return configDumpCurrent } type routeMetadataDump struct { Routes []routeMetadataEntry `json:"routes"` } type routeMetadataEntry struct { // Context is the API's resolved gateway-facing base path (e.g. "/admin-test/v1") - // the policy engine has no field literally named "basePath". Context string `json:"context"` // RouteKey is "METHOD|fullPath|vhost" (see GenerateRouteNameWithDiscriminator). RouteKey string `json:"route_key"` // ChainKey names the policy_chains entry this route binds. Routes sharing an // identical chain can report a chain owned by another route, so a chain must be // reached through the route rather than matched on the route's own path. Gateway 1.2.x // does not expose this field because its policy chains are keyed by RouteKey directly. ChainKey string `json:"chain_key"` } type policyChainsDump struct { PolicyChains []policyChain `json:"policy_chains"` } type policyChain struct { // RouteKey is populated by Gateway 1.2.x, whose chains are keyed directly by route. RouteKey string `json:"route_key"` // ChainKey is populated by newer Gateway releases. ChainKey string `json:"chain_key"` Policies []policySpec `json:"policies"` } type policySpec struct { Name string `json:"name"` Parameters map[string]any `json:"parameters"` } // policyMatcher reports whether a policy instance in a chain satisfies an assertion. type policyMatcher func(policySpec) bool // policyNamed matches a policy instance by name. func policyNamed(name string) policyMatcher { return func(policy policySpec) bool { return policy.Name == name } } // policyNamedWithParameter matches a policy instance by name whose parameter at the dotted // path has the scalar value want. An empty path matches by name alone. func policyNamedWithParameter(name, path, want string) policyMatcher { return func(policy policySpec) bool { if policy.Name != name { return false } if path == "" { return true } actual, found := policyParameterValue(policy.Parameters, path) return found && actual == want } } // policyParameterValue resolves a dotted path, such as "request.mode" or // "request.headers.0", against policy parameters. Numeric segments index arrays. It reports // false when the path is absent or does not end at a scalar value. func policyParameterValue(parameters map[string]any, path string) (string, bool) { var current any = parameters for _, segment := range strings.Split(path, ".") { if segment == "" { return "", false } switch node := current.(type) { case map[string]any: value, ok := node[segment] if !ok { return "", false } current = value case []any: index, err := strconv.Atoi(segment) if err != nil || index < 0 || index >= len(node) { return "", false } current = node[index] default: return "", false } } switch current.(type) { case nil, map[string]any, []any: return "", false default: return fmt.Sprint(current), true } } func (c policyChain) containsPolicy(match policyMatcher) bool { for _, policy := range c.Policies { if match(policy) { return true } } return false } func (d configDump) containsPolicy(schema configDumpSchema, routePath string, match policyMatcher) bool { switch schema { case configDumpLegacyRouteKey: for _, chain := range d.PolicyChains.PolicyChains { if routeKeyPath(chain.RouteKey) == routePath && chain.containsPolicy(match) { return true } } return false default: chainKeys := make(map[string]struct{}) for _, route := range d.RouteMetadata.Routes { if routeKeyPath(route.RouteKey) == routePath && route.ChainKey != "" { chainKeys[route.ChainKey] = struct{}{} } } if len(chainKeys) == 0 { return false } for _, chain := range d.PolicyChains.PolicyChains { if _, found := chainKeys[chain.ChainKey]; found && chain.containsPolicy(match) { return true } } return false } } // routeKeyPath extracts the fullPath segment from a "METHOD|fullPath|vhost" route key. func routeKeyPath(routeKey string) string { parts := strings.SplitN(routeKey, "|", 3) if len(parts) < 2 { return "" } return parts[1] } func (g *Gateway) lazyResources(ctx context.Context) ([]lazyResource, error) { resp, err := httpx.Published(ctx) if err != nil { return nil, err } var dump configDump if err := json.Unmarshal(resp.Body, &dump); err != nil { return nil, fmt.Errorf("parse policy-engine config dump: %w", err) } var resources []lazyResource for _, group := range dump.Lazy.ResourcesByType { resources = append(resources, group...) } return resources, nil } func (g *Gateway) lazyTemplatePresent(ctx context.Context, id, kind string) error { return g.lazyResourcePresentByType(ctx, id, kind, true) } func (g *Gateway) lazyResourcePresent(ctx context.Context, id, kind string) error { return g.lazyResourcePresentByType(ctx, id, kind, true) } func (g *Gateway) lazyResourcePresentByType(ctx context.Context, id, kind string, want bool) error { id, err := stepscommon.Expand(ctx, id) if err != nil { return err } resources, err := g.lazyResources(ctx) if err != nil { return err } for _, resource := range resources { if resource.ID == id && resource.Type == kind { if want { return nil } return fmt.Errorf("lazy resource %q of type %q is present", id, kind) } } if want { return fmt.Errorf("lazy resource %q of type %q is absent", id, kind) } return nil } func (g *Gateway) lazyTemplateAbsent(ctx context.Context, id string) error { return g.lazyResourceAbsent(ctx, id) } func (g *Gateway) lazyResourceAbsent(ctx context.Context, id string) error { id, err := stepscommon.Expand(ctx, id) if err != nil { return err } resources, err := g.lazyResources(ctx) if err != nil { return err } for _, resource := range resources { if resource.ID == id { return fmt.Errorf("lazy resource %q is present as %q", id, resource.Type) } } return nil } func (g *Gateway) lazyTypedResourceAbsent(ctx context.Context, id, kind string) error { return g.lazyResourcePresentByType(ctx, id, kind, false) } func (g *Gateway) lazyDisplayName(ctx context.Context, id, want string) error { id, err := stepscommon.Expand(ctx, id) if err != nil { return err } resources, err := g.lazyResources(ctx) if err != nil { return err } for _, resource := range resources { if resource.ID != id { continue } spec, ok := resource.Resource["spec"].(map[string]any) if !ok { return fmt.Errorf("lazy resource %q has no spec", id) } if spec["displayName"] != want { return fmt.Errorf("lazy resource %q has display name %v, want %q", id, spec["displayName"], want) } return nil } return fmt.Errorf("lazy resource %q is absent", id) } func (g *Gateway) providerTemplateMapping(ctx context.Context, provider, template string) error { provider, err := stepscommon.Expand(ctx, provider) if err != nil { return err } template, err = stepscommon.Expand(ctx, template) if err != nil { return err } resources, err := g.lazyResources(ctx) if err != nil { return err } for _, resource := range resources { if resource.ID == provider && resource.Type == "ProviderTemplateMapping" { if resource.Resource["template_handle"] != template { return fmt.Errorf("provider %q maps to %v, want %q", provider, resource.Resource["template_handle"], template) } return nil } } return fmt.Errorf("provider template mapping %q is absent", provider) } // serviceRequestUntilProviderTemplateMapping polls a policy-engine config dump until the // requested provider mapping points to the expected template. func (g *Gateway) serviceRequestUntilProviderTemplateMapping( ctx context.Context, method, service, path, provider, template string, ) error { if strings.ToUpper(method) != http.MethodGet || service != "policy-engine" { return fmt.Errorf("provider-mapping readiness requires a GET request to the policy-engine") } url, err := g.serviceURL(ctx, service, path) if err != nil { return err } provider, err = stepscommon.Expand(ctx, provider) if err != nil { return err } template, err = stepscommon.Expand(ctx, template) if err != nil { return err } accept := func(resp *httpx.Response) bool { return resp != nil && resp.Succeeded() && providerTemplateMappingMatches(resp.Body, provider, template) } last, err := retry.Until(ctx, retry.Options{Interval: 200 * time.Millisecond}, func(ctx context.Context) (*httpx.Response, error) { resp, requestErr := g.funnel.Client().Do(ctx, httpx.Request{ Method: http.MethodGet, URL: url, Headers: g.scenarioHeaders(ctx), }, 0, 0) if requestErr != nil { return nil, retry.Transient(requestErr) } return resp, nil }, accept) if err != nil { return fmt.Errorf("waiting for provider %q to map to template %q: %w", provider, template, err) } if !accept(last) { return fmt.Errorf("provider %q did not map to template %q", provider, template) } return g.funnel.Publish(ctx, last) } func providerTemplateMappingMatches(body []byte, provider, template string) bool { var dump configDump if err := json.Unmarshal(body, &dump); err != nil { return false } for _, resource := range dump.Lazy.ResourcesByType["ProviderTemplateMapping"] { if resource.ID == provider { return resource.Resource["template_handle"] == template } } return false } // serviceRequestUntilLazyResourceAbsent polls a policy-engine config dump until the specified // resource type no longer contains the requested resource ID. func (g *Gateway) serviceRequestUntilLazyResourceAbsent( ctx context.Context, method, service, path, resourceID, resourceType string, ) error { if strings.ToUpper(method) != http.MethodGet || service != "policy-engine" { return fmt.Errorf("lazy-resource deletion readiness requires a GET request to the policy-engine") } url, err := g.serviceURL(ctx, service, path) if err != nil { return err } resourceID, err = stepscommon.Expand(ctx, resourceID) if err != nil { return err } resourceType, err = stepscommon.Expand(ctx, resourceType) if err != nil { return err } accept := func(resp *httpx.Response) bool { return resp != nil && resp.Succeeded() && lazyResourceAbsent(resp.Body, resourceID, resourceType) } last, err := retry.Until(ctx, retry.Options{Interval: 200 * time.Millisecond}, func(ctx context.Context) (*httpx.Response, error) { resp, requestErr := g.funnel.Client().Do(ctx, httpx.Request{ Method: http.MethodGet, URL: url, Headers: g.scenarioHeaders(ctx), }, 0, 0) if requestErr != nil { return nil, retry.Transient(requestErr) } return resp, nil }, accept) if err != nil { return fmt.Errorf("waiting for lazy resource %q of type %q to be removed: %w", resourceID, resourceType, err) } if !accept(last) { return fmt.Errorf("lazy resource %q of type %q was not removed", resourceID, resourceType) } return g.funnel.Publish(ctx, last) } func lazyResourceAbsent(body []byte, resourceID, resourceType string) bool { var dump configDump if err := json.Unmarshal(body, &dump); err != nil { return false } for _, resource := range dump.Lazy.ResourcesByType[resourceType] { if resource.ID == resourceID { return false } } return true } func (g *Gateway) routeMetadataProvider(ctx context.Context, provider string) error { provider, err := stepscommon.Expand(ctx, provider) if err != nil { return err } resp, err := httpx.Published(ctx) if err != nil { return err } if strings.Contains(string(resp.Body), provider) { return nil } return fmt.Errorf("route metadata does not contain provider %q", provider) } // awaitConfigDumpRouteAbsent polls the policy engine's own config dump until no route has the // given base path. The generic funnel's dump-consistency wait (awaitDumpConsistent) is capped // at retry.PropagationCeiling (60s); the policy engine's own route_metadata rebuild after an // API deletion has been measured taking well beyond that and varies widely between runs // (observed ~70s in one run, ~153s in another - consistent with a periodic reconciliation // cycle rather than a fixed propagation delay), so this polls with its own much longer timeout // instead of relying on that fixed ceiling. func (g *Gateway) awaitConfigDumpRouteAbsent(ctx context.Context, basePath string) error { resolved, err := stepscommon.Expand(ctx, basePath) if err != nil { return err } url, err := g.serviceURL(ctx, "policy-engine", "/config_dump") if err != nil { return err } accept := func(resp *httpx.Response) bool { if resp == nil || !resp.Succeeded() { return false } var dump configDump if err := json.Unmarshal(resp.Body, &dump); err != nil { return false } for _, route := range dump.RouteMetadata.Routes { if route.Context == resolved { return false } } return true } last, err := retry.Until(ctx, retry.Options{Timeout: 240 * time.Second, Interval: 2 * time.Second}, func(ctx context.Context) (*httpx.Response, error) { resp, requestErr := g.funnel.Client().Do(ctx, httpx.Request{ Method: http.MethodGet, URL: url, Headers: g.scenarioHeaders(ctx), }, 0, 0) if requestErr != nil { return nil, retry.Transient(requestErr) } return resp, nil }, accept) if err := awaited(last, err, accept, fmt.Sprintf("waiting for the config dump to stop containing a route with base path %q", resolved)); err != nil { return err } return g.funnel.Publish(ctx, last) } // configDumpRouteBasePath asserts whether the policy engine's config dump has a route whose // resolved context (its gateway-facing base path) matches basePath. func (g *Gateway) configDumpRouteBasePath(ctx context.Context, basePath string, want bool) error { resolved, err := stepscommon.Expand(ctx, basePath) if err != nil { return err } resp, err := httpx.Published(ctx) if err != nil { return err } var dump configDump if err := json.Unmarshal(resp.Body, &dump); err != nil { return fmt.Errorf("parsing policy-engine config dump: %w", err) } found := false for _, route := range dump.RouteMetadata.Routes { if route.Context == resolved { found = true break } } if found == want { return nil } if want { return fmt.Errorf("config dump does not contain a route with base path %q", resolved) } return fmt.Errorf("config dump still contains a route with base path %q", resolved) } // configDumpPolicyForRoute waits until the policy engine's config dump shows policyName attached // to the operation whose full path is routePath and, when parameterPath is set, carrying the // scalar parameterValue at that dotted parameter path. The route can become visible before its // policy chain is populated, and an updated policy keeps its name, so checking one // already-published dump is not sufficient. func (g *Gateway) configDumpPolicyForRoute( ctx context.Context, policyName, routePath, parameterPath, parameterValue string, ) error { resolvedPath, err := stepscommon.Expand(ctx, routePath) if err != nil { return err } resolvedValue, err := stepscommon.Expand(ctx, parameterValue) if err != nil { return err } match := policyNamedWithParameter(policyName, parameterPath, resolvedValue) what := fmt.Sprintf("waiting for policy %q on route %q in the config dump", policyName, resolvedPath) if parameterPath != "" { what = fmt.Sprintf("waiting for policy %q on route %q with parameter %q set to %q in the config dump", policyName, resolvedPath, parameterPath, resolvedValue) } version, err := g.topo.ComponentVersion("platform-gateway") if err != nil { return err } schema := configDumpSchemaFor(version) containsPolicy := func(resp *httpx.Response) bool { if resp == nil || !resp.Succeeded() { return false } var dump configDump if err := json.Unmarshal(resp.Body, &dump); err != nil { return false } return dump.containsPolicy(schema, resolvedPath, match) } // Preserve the response from the preceding config-dump request when it already satisfies // the assertion. This is the normal path and keeps the assertion tied to the response the // scenario explicitly requested; polling below is only needed when that snapshot is stale. published, err := httpx.Published(ctx) if err != nil { return err } if containsPolicy(published) { return nil } url, err := g.serviceURL(ctx, "policy-engine", "/config_dump") if err != nil { return err } last, err := retry.Until(ctx, retry.Options{}, func(ctx context.Context) (*httpx.Response, error) { resp, requestErr := g.funnel.Client().Do(ctx, httpx.Request{ Method: http.MethodGet, URL: url, Headers: g.scenarioHeaders(ctx), }, 0, 0) if requestErr != nil { return nil, retry.Transient(requestErr) } return resp, nil }, containsPolicy) if err := awaited(last, err, containsPolicy, what); err != nil { return err } return g.funnel.Publish(ctx, last) } func (g *Gateway) lazyResourceCount(ctx context.Context, want int, id string) error { id, err := stepscommon.Expand(ctx, id) if err != nil { return err } resources, err := g.lazyResources(ctx) if err != nil { return err } count := 0 for _, resource := range resources { if resource.ID == id { count++ } } if count < want { return fmt.Errorf("found %d lazy resources with id %q, want at least %d", count, id, want) } return nil } func (g *Gateway) mcpInitialize(ctx context.Context, path string) error { return g.sendMCPRequest(ctx, path, mcpInitializeBody()) } func (g *Gateway) mcpInitializeUntilSuccessful(ctx context.Context, path string) error { response, err := retry.Until(ctx, retry.Options{}, func(ctx context.Context) (*httpx.Response, error) { response, requestErr := g.mcpRequest(ctx, path, mcpInitializeBody()) if requestErr != nil { return nil, retry.Transient(requestErr) } return response, nil }, func(response *httpx.Response) bool { return response.Succeeded() }, ) if err != nil { return err } if !response.Succeeded() { return fmt.Errorf("waiting for MCP initialize to succeed: the condition never held; the last response was %s", response.Describe()) } return g.funnel.Publish(ctx, response) } func mcpInitializeBody() string { return `{"jsonrpc":"2.0","id":1,"method":"initialize","params":{"protocolVersion":"2025-06-18","capabilities":{"roots":{"listChanged":true}},"clientInfo":{"name":"framework-client","version":"1.0.0"}}}` } // mcpToolCall calls the named testbench MCP tool with arguments matching that tool's own // input schema - the testbench "add" tool takes numeric operands, "echo" takes a message. func (g *Gateway) mcpToolCall(ctx context.Context, tool, path string) error { body := fmt.Sprintf(`{"jsonrpc":"2.0","id":2,"method":"tools/call","params":{"name":%q,"arguments":%s}}`, tool, mcpToolArguments(tool)) return g.sendMCPRequest(ctx, path, body) } func (g *Gateway) mcpToolsList(ctx context.Context, path string) error { body := `{"jsonrpc":"2.0","id":3,"method":"tools/list","params":{}}` return g.sendMCPRequest(ctx, path, body) } // mcpNotificationInitialized sends the notification a client is expected to send once, right // after a successful initialize response, before issuing any further request. func (g *Gateway) mcpNotificationInitialized(ctx context.Context, path string) error { body := `{"jsonrpc":"2.0","method":"notifications/initialized"}` return g.sendMCPRequest(ctx, path, body) } // mcpRawRequest sends a body the scenario wrote out in full. The MCP client steps above cover // the well-formed cases; this one exists for the bodies a client should never send - a duplicated // member, truncated JSON - which still have to travel the MCP transport and come back unwrapped // from their event-stream framing. func (g *Gateway) mcpRawRequest(ctx context.Context, path string, body *godog.DocString) error { resolved, err := stepscommon.Expand(ctx, body.Content) if err != nil { return err } return g.sendMCPRequest(ctx, path, resolved) } // mcpOversizedRequest sends a syntactically valid tools/call whose arguments are padded to the // requested size. The padding is inside a string value, so the body stays parseable and the only // thing under test is its length. func (g *Gateway) mcpOversizedRequest(ctx context.Context, method string, kib int, path string) error { body := fmt.Sprintf( `{"jsonrpc":"2.0","id":5,"method":%q,"params":{"name":"echo","arguments":{"message":%q}}}`, method, strings.Repeat("x", kib*1024)) return g.sendMCPRequest(ctx, path, body) } // mcpModernRequest sends a request in the 2026-07-28 shape: the protocol version, client identity // and client capabilities in params._meta, and the method mirrored into a header. func (g *Gateway) mcpModernRequest(ctx context.Context, method, path, version string) error { return g.sendModernMCPRequest(ctx, method, "", path, version) } // mcpModernNamedRequest is mcpModernRequest for a method that targets a named capability, which // 2026-07-28 requires be mirrored into Mcp-Name as well. func (g *Gateway) mcpModernNamedRequest(ctx context.Context, method, name, path, version string) error { return g.sendModernMCPRequest(ctx, method, name, path, version) } func (g *Gateway) sendModernMCPRequest(ctx context.Context, method, name, path, version string) error { headers := map[string]string{ mcpProtocolVersionHeader: version, mcpMethodHeader: method, } if name != "" { headers[mcpNameHeader] = name } resp, err := g.mcpRequestWithHeaders(ctx, path, mcpModernBody(method, name, version), headers) if err != nil { return err } return g.funnel.Publish(ctx, resp) } // mcpModernBody builds the 2026-07-28 request envelope. All three _meta members are required - // a conformant server answers -32602 when any of them is missing - and the protocol version in // _meta must equal the one in the header, which is the agreement the era is built on. func mcpModernBody(method, name, version string) string { meta := fmt.Sprintf( `"_meta":{"io.modelcontextprotocol/protocolVersion":%q,`+ `"io.modelcontextprotocol/clientInfo":{"name":"framework-client","version":"1.0.0"},`+ `"io.modelcontextprotocol/clientCapabilities":{}}`, version) switch { case name == "": return fmt.Sprintf(`{"jsonrpc":"2.0","id":6,"method":%q,"params":{%s}}`, method, meta) case method == "resources/read": // MCP identifies a resource by uri and defines no name for it, so that is the member // the mirrored Mcp-Name is compared against. return fmt.Sprintf(`{"jsonrpc":"2.0","id":6,"method":%q,"params":{"uri":%q,%s}}`, method, name, meta) case method == "prompts/get": return fmt.Sprintf(`{"jsonrpc":"2.0","id":6,"method":%q,"params":{"name":%q,%s}}`, method, name, meta) default: return fmt.Sprintf(`{"jsonrpc":"2.0","id":6,"method":%q,"params":{"name":%q,"arguments":%s,%s}}`, method, name, mcpToolArguments(name), meta) } } // mcpToolArguments returns arguments matching the named testbench tool's own input schema. func mcpToolArguments(tool string) string { switch tool { case "echo": return `{"message":"Hello, World!"}` case "headers": return `{"name":"Authorization"}` default: return `{"a":40,"b":60}` } } // mcpToolCallInvalidParams omits the required "name" field, which every JSON-RPC tools/call // request must carry. func (g *Gateway) mcpToolCallInvalidParams(ctx context.Context, path string) error { body := `{"jsonrpc":"2.0","id":4,"method":"tools/call","params":{"arguments":{"a":40,"b":60}}}` return g.sendMCPRequest(ctx, path, body) } // jwtToken mints a JWT from the testbench jwks mock and stores it in runner context under key, // for a later "I set header" step to attach as a bearer token. issuer becomes the token's iss // claim and must match a configured keymanager's issuer exactly; scope and claims are optional // (empty skips them) and claims is a comma-separated list of key=value pairs. func (g *Gateway) jwtToken(ctx context.Context, issuer, scope, claims, key string) error { resolvedIssuer, err := stepscommon.Expand(ctx, issuer) if err != nil { return err } base, err := g.topo.URL("testbench", "jwks") if err != nil { return err } q := url.Values{} q.Set("issuer", resolvedIssuer) if scope != "" { resolvedScope, err := stepscommon.Expand(ctx, scope) if err != nil { return err } q.Set("scope", resolvedScope) } for _, pair := range strings.Split(claims, ",") { pair = strings.TrimSpace(pair) if pair == "" { continue } kv := strings.SplitN(pair, "=", 2) if len(kv) != 2 { return fmt.Errorf("invalid claim %q: expected key=value", pair) } claimValue, err := stepscommon.Expand(ctx, strings.TrimSpace(kv[1])) if err != nil { return err } q.Set("claim_"+strings.TrimSpace(kv[0]), claimValue) } resp, err := retry.Until(ctx, retry.Options{}, func(ctx context.Context) (*httpx.Response, error) { r, requestErr := g.funnel.Client().Do(ctx, httpx.Request{ Method: http.MethodGet, URL: base + "/token?" + q.Encode(), }, 0, 0) if requestErr != nil { return nil, retry.Transient(requestErr) } return r, nil }, func(resp *httpx.Response) bool { return resp != nil && resp.Succeeded() }) if err != nil { return fmt.Errorf("minting a JWT token: %w", err) } if !resp.Succeeded() { return fmt.Errorf("minting a JWT token failed: %s", resp.Describe()) } local, ok := tcontext.LocalOf(ctx) if !ok || local == nil { return fmt.Errorf("cannot store a JWT token without runner context") } local.Set(key, strings.TrimSpace(string(resp.Body))) return nil } // sendMCPRequest sends a streamable HTTP MCP request and publishes its JSON-RPC payload. func (g *Gateway) sendMCPRequest(ctx context.Context, path, body string) error { resp, err := g.mcpRequest(ctx, path, body) if err != nil { return err } return g.funnel.Publish(ctx, resp) } // mcpRequest issues one MCP call and returns its response without publishing it, so a caller // polling for readiness can inspect each attempt. func (g *Gateway) mcpRequest(ctx context.Context, path, body string) (*httpx.Response, error) { return g.mcpRequestWithHeaders(ctx, path, body, nil) } // mcpRequestWithHeaders layers per-call headers under the scenario's own, so a scenario that has // already run "I set header" wins. That ordering is what lets a test make a mirrored header // disagree with the body it was built from. // // Both sides are canonicalised first, or "wins" would be decided by map iteration order: "I set // header" keeps whatever spelling the feature wrote, and the client applies every map entry through // Header.Set, which folds the key. Two spellings of one header would survive this merge as two // entries and then collapse into one header of undefined value. MCP-Protocol-Version makes that // easy to hit, because Go's canonical form of it is Mcp-Protocol-Version - so the spelling a // feature would naturally write differs from the one used here. // mergeCanonicalHeaders folds every name to its canonical form as it merges, so a later layer // always overwrites an earlier one whatever spelling either used. func mergeCanonicalHeaders(layers ...map[string]string) map[string]string { merged := map[string]string{} for _, layer := range layers { for name, value := range layer { merged[http.CanonicalHeaderKey(name)] = value } } return merged } func (g *Gateway) mcpRequestWithHeaders( ctx context.Context, path, body string, extra map[string]string, ) (*httpx.Response, error) { resolved, err := stepscommon.Expand(ctx, path) if err != nil { return nil, err } url, err := g.gatewayURL(resolved) if err != nil { return nil, err } resp, err := g.funnel.Send(ctx, httpx.Request{ Method: http.MethodPost, URL: url, Headers: mergeCanonicalHeaders(extra, g.scenarioHeaders(ctx), map[string]string{ "Content-Type": "application/json", "Accept": "application/json, text/event-stream", }), Body: []byte(body), Host: g.requestHost(ctx), }) if err != nil { return nil, err } resp.Body = mcpJSONPayload(resp.Body) return resp, nil } // mcpJSONPayload extracts the JSON-RPC document from a streamable HTTP SSE response. func mcpJSONPayload(body []byte) []byte { text := string(body) marker := "data:" start := strings.Index(text, marker) if start < 0 { return body } start += len(marker) end := strings.IndexByte(text[start:], '\n') if end >= 0 { end += start } else { end = len(text) } payload := strings.TrimSpace(text[start:end]) if json.Valid([]byte(payload)) { return []byte(payload) } return body } // awaited turns the result of a retry.Until poll into a pass or a failure. func awaited( last *httpx.Response, err error, accept func(*httpx.Response) bool, what string, ) error { if err != nil { return fmt.Errorf("%s: %w", what, err) } if accept(last) { return nil } if last == nil { return fmt.Errorf("%s: the condition never held, and no response was ever received", what) } return fmt.Errorf("%s: the condition never held; the last response was %s", what, last.Describe()) } // headerWith returns the scenario headers with one header set, overriding any sticky value. func (g *Gateway) headerWith(ctx context.Context, name, value string) map[string]string { h := g.scenarioHeaders(ctx) h[name] = value return h } // managementURL builds a management API URL from the running topology. func (g *Gateway) managementURL(path string) (string, error) { base, err := g.topo.URL("platform-gateway", "rest") if err != nil { return "", err } return base + gatewayManagementBasePath(g.topo) + path, nil } func (g *Gateway) adminURL(path string) (string, error) { base, err := g.topo.URL("platform-gateway", "admin") if err != nil { return "", err } return base + gatewayAdminBasePath(g.topo) + path, nil } func (g *Gateway) adminURLAt(ordinalWord, path string) (string, error) { ordinal, err := gatewayOrdinal(ordinalWord) if err != nil { return "", err } base, err := g.topo.URLAt("platform-gateway", ordinal, "admin") if err != nil { return "", err } return base + gatewayAdminBasePath(g.topo) + path, nil } // apiNameFrom extracts metadata.name from an API definition. func apiNameFrom(definition string) string { metadata := resourceMetadata{} if err := yaml.Unmarshal([]byte(definition), &metadata); err != nil { return "" } return metadata.Metadata.Name } type resourceMetadata struct { Kind string `yaml:"kind"` Metadata struct { Name string `yaml:"name"` } `yaml:"metadata"` } // register wires gateway, health, and timeout steps. func (g *Gateway) register(sc *godog.ScenarioContext) { sc.Step(`^the gateway version satisfies "(>=|>|<=|<|==|=)([0-9.]+)"$`, g.requireGatewayVersion) g.registerRawHTTPSteps(sc) g.registerMTLSSteps(sc) g.registerMTLSPoolSteps(sc) g.registerOutboundMTLSSteps(sc) g.registerMTLSHostnameSteps(sc) g.registerWaitSteps(sc) g.registerMTLSListenerSteps(sc) g.registerMTLSHeaderBypassSteps(sc) g.registerMTLSRelaySteps(sc) g.registerMTLSObservabilitySteps(sc) g.registerA2AClientSteps(sc) g.registerAgentSteps(sc) // Request state is runner-scoped, so clear it before each scenario. sc.Before(func(ctx context.Context, _ *godog.Scenario) (context.Context, error) { if err := tcontext.Set(ctx, keyGatewaySpecVersion, gatewaySpecVersionForVersion(gatewayVersion(g.topo))); err != nil { return ctx, err } if err := tcontext.Set(ctx, keyBackendSpecVersion, backendSpecVersion); err != nil { return ctx, err } if err := tcontext.Set(ctx, keyGatewayMCPUpstreamPath, gatewayMCPUpstreamPathForVersion(gatewayVersion(g.topo))); err != nil { return ctx, err } if g.topo.Block != nil { if err := tcontext.Set(ctx, keyTestbenchPartition, g.topo.Block.PartitionKey()); err != nil { return ctx, err } } if err := g.resetRequest(ctx); err != nil { return ctx, err } // Dump expectations belong to one scenario. Keeping them in the runner-local // context would make a later scenario wait for resources created by an earlier one. return ctx, tcontext.Set(ctx, keyDumpExpectations, map[string]bool{}) }) sc.Step(`^the gateway services are running$`, g.gatewayIsRunning) sc.Step(`^the (first|second) gateway is running$`, g.gatewayIsRunningAt) sc.Step(`^I authenticate using basic auth as "([^"]*)"$`, g.authenticateAs) sc.Step( `^I create (API|LLM provider|LLM provider template|MCP proxy|LLM proxy|Agent|Backend) with configuration:$`, g.createResource) sc.Step(`^I create API with JSON configuration:$`, g.createJSONAPI) g.registerResourceTemplateSteps(sc) sc.Step(`^I get the (API|LLM provider|LLM provider template|MCP proxy|LLM proxy|Agent|GraphQL API|Backend) "([^"]*)"$`, g.getResource) sc.Step(`^I list all (LLM providers|LLM provider templates|MCP proxies|LLM proxies)$`, g.listResources) sc.Step(`^I update the (API|LLM provider|LLM provider template|MCP proxy|LLM proxy|Agent|Backend) "([^"]*)" with configuration:$`, g.updateResource) sc.Step(`^I delete the (API|LLM provider|LLM provider template|MCP proxy|LLM proxy|Agent|GraphQL API|Backend) "([^"]*)"$`, g.deleteResource) sc.Step(`^I send a "([^"]*)" request to the "([^"]*)" service at "([^"]*)"$`, g.serviceRequest) sc.Step(`^I send a "([^"]*)" request to the "([^"]*)" service at "([^"]*)" with body:$`, g.serviceRequestWithBody) sc.Step(`^I send a "([^"]*)" request to the "([^"]*)" service at "([^"]*)" until status (\d+)$`, g.serviceRequestUntilStatus) sc.Step(`^I send a "([^"]*)" request to the "([^"]*)" service at "([^"]*)" until the response body does not contain "([^"]*)"$`, g.serviceRequestUntilBodyNotContains) sc.Step(`^I resolve the "([^"]*)" service URL at "([^"]*)" and store it as "([^"]*)"$`, g.resolveServiceURLAndStore) sc.Step(`^I register the "([^"]*)" "([^"]*)" for cleanup$`, g.registerAdminResourceForCleanup) sc.Step(`^the "([^"]*)" service logs should contain "([^"]*)"$`, g.serviceLogsContain) sc.Step(`^the "([^"]*)" service logs should not contain "([^"]*)"$`, g.serviceLogsNotContain) sc.Step(`^the "([^"]*)" service logs should have exactly (\d+) lines? containing:$`, g.serviceLogLinesCount) sc.Step(`^the "([^"]*)" service log event containing "([^"]*)" should( not)? contain "([^"]*)"$`, g.serviceLogEvent) sc.Step(`^I send a "([^"]*)" request to the "([^"]*)" service at "([^"]*)" until lazy resource "([^"]*)" has display name "([^"]*)"$`, g.serviceRequestUntilLazyDisplayName) sc.Step(`^I send a "([^"]*)" request to the "([^"]*)" service at "([^"]*)" until provider template mapping "([^"]*)" maps to template "([^"]*)"$`, g.serviceRequestUntilProviderTemplateMapping) sc.Step(`^I send a "([^"]*)" request to the "([^"]*)" service at "([^"]*)" until lazy resource "([^"]*)" of type "([^"]*)" is absent$`, g.serviceRequestUntilLazyResourceAbsent) sc.Step(`^the latest analytics event for path "([^"]*)" should (contain|not contain) (request|response) header "([^"]*)"(?: with value "([^"]*)")?$`, g.analyticsHeader) sc.Step(`^the latest analytics event for path "([^"]*)" should contain (request|response) header "([^"]*)" with values "([^"]*)"$`, g.analyticsHeaderValues) sc.Step(`^the analytics events for path "([^"]*)" should record (\d+) requests identified by request header "([^"]*)" with prefix "([^"]*)" and headers:$`, g.analyticsPerRequestHeaders) sc.Step(`^I reset the analytics collector$`, g.resetAnalyticsCollector) sc.Step(`^I wait for the analytics collector to settle$`, g.waitForAnalyticsToSettle) sc.Step(`^the analytics collector should have received at least (\d+) events?$`, g.analyticsEventCountAtLeast) sc.Step(`^the analytics collector should have received (\d+) events?$`, g.analyticsEventCountExactly) sc.Step(`^the latest analytics event for path "([^"]*)" should have request method "([^"]*)"$`, g.analyticsRequestMethod) sc.Step(`^the latest analytics event for path "([^"]*)" should have response status (\d+)$`, g.analyticsResponseStatus) sc.Step(`^the latest analytics event for path "([^"]*)" should have metadata field "([^"]*)" with value "([^"]*)"$`, g.analyticsMetadataField) sc.Step(`^the latest analytics event for path "([^"]*)" should have numeric metadata field "([^"]*)" (equal to|at least) "([^"]*)"$`, g.analyticsNumericMetadataField) sc.Step(`^the response should be an oob-template list$`, g.oobTemplateList) sc.Step(`^the lazy resources should contain template "([^"]*)" of type "([^"]*)"$`, g.lazyTemplatePresent) sc.Step(`^the lazy resources should not contain template "([^"]*)"$`, g.lazyTemplateAbsent) sc.Step(`^the lazy resource "([^"]*)" should have display name "([^"]*)"$`, g.lazyDisplayName) sc.Step(`^the lazy resources should contain resource "([^"]*)" of type "([^"]*)"$`, g.lazyResourcePresent) sc.Step(`^the lazy resources should not contain resource "([^"]*)"$`, g.lazyResourceAbsent) sc.Step(`^the lazy resources should not contain resource "([^"]*)" of type "([^"]*)"$`, g.lazyTypedResourceAbsent) sc.Step(`^the provider template mapping "([^"]*)" should map to template "([^"]*)"$`, g.providerTemplateMapping) sc.Step(`^the policy engine route metadata should contain provider_name "([^"]*)"$`, g.routeMetadataProvider) sc.Step(`^the config dump should contain route with base path "([^"]*)"$`, func(ctx context.Context, basePath string) error { return g.configDumpRouteBasePath(ctx, basePath, true) }) sc.Step(`^the config dump should not contain route with base path "([^"]*)"$`, func(ctx context.Context, basePath string) error { return g.configDumpRouteBasePath(ctx, basePath, false) }) sc.Step(`^the config dump should contain policy "([^"]*)" for route "([^"]*)"(?: with parameter "([^"]*)" set to "([^"]*)")?$`, g.configDumpPolicyForRoute) sc.Step(`^I wait for the config dump to stop containing a route with base path "([^"]*)"$`, g.awaitConfigDumpRouteAbsent) sc.Step(`^the lazy resources should have at least (\d+) resources with id "([^"]*)"$`, g.lazyResourceCount) sc.Step(`^I use the MCP Client to send an initialize request to "([^"]*)"$`, g.mcpInitialize) sc.Step(`^I use the MCP Client to send an initialize request to "([^"]*)" until successful$`, g.mcpInitializeUntilSuccessful) sc.Step(`^I use the MCP Client to send "([^"]*)" tools/call request to "([^"]*)"$`, g.mcpToolCall) sc.Step(`^I use the MCP Client to send a tools/list request to "([^"]*)"$`, g.mcpToolsList) sc.Step(`^I use the MCP Client to send a notifications/initialized notification to "([^"]*)"$`, g.mcpNotificationInitialized) sc.Step(`^I use the MCP Client to send this request to "([^"]*)":$`, g.mcpRawRequest) sc.Step(`^I use the MCP Client to send a "([^"]*)" request of (\d+) KiB to "([^"]*)"$`, g.mcpOversizedRequest) sc.Step(`^I use the MCP Client to send a "([^"]*)" request to "([^"]*)" declaring MCP version "([^"]*)"$`, g.mcpModernRequest) sc.Step(`^I use the MCP Client to send a "([^"]*)" request for "([^"]*)" to "([^"]*)" declaring MCP version "([^"]*)"$`, g.mcpModernNamedRequest) sc.Step(`^I use the MCP Client to send a tools/call request with invalid params to "([^"]*)"$`, g.mcpToolCallInvalidParams) sc.Step(`^I get a JWT token from the mock JWKS server with issuer "([^"]*)" and store it as "([^"]*)"$`, func(ctx context.Context, issuer, key string) error { return g.jwtToken(ctx, issuer, "", "", key) }) sc.Step(`^I get a JWT token from the mock JWKS server with issuer "([^"]*)" and scope "([^"]*)" and store it as "([^"]*)"$`, func(ctx context.Context, issuer, scope, key string) error { return g.jwtToken(ctx, issuer, scope, "", key) }) sc.Step(`^I get a JWT token from the mock JWKS server with issuer "([^"]*)" and claims "([^"]*)" and store it as "([^"]*)"$`, func(ctx context.Context, issuer, claims, key string) error { return g.jwtToken(ctx, issuer, "", claims, key) }) sc.Step(`^I get a JWT token from the mock JWKS server with issuer "([^"]*)", scope "([^"]*)" and claims "([^"]*)" and store it as "([^"]*)"$`, g.jwtToken) sc.Step(`^I send a "([^"]*)" request to "([^"]*)" until the rate limit reports (\d+) remaining$`, g.sendUntilQuotaRemaining) sc.Step(`^the response body should contain template literal:$`, g.responseBodyContainsTemplateLiteral) sc.Step(`^the LLM provider response should handle the configured secret safely:$`, g.llmProviderResponseHandlesSecret) sc.Step(`^the stored (RestApi|LlmProvider|LlmProxy|Mcp) configuration for "([^"]*)" should contain:$`, func(ctx context.Context, kind, handle string, literal *godog.DocString) error { return g.assertStoredConfiguration(ctx, kind, handle, literal, true) }) sc.Step(`^the stored (RestApi|LlmProvider|LlmProxy|Mcp) configuration for "([^"]*)" should not contain:$`, func(ctx context.Context, kind, handle string, literal *godog.DocString) error { return g.assertStoredConfiguration(ctx, kind, handle, literal, false) }) g.registerTimeoutSteps(sc) } const elapsedTolerance = 0.05 func (g *Gateway) registerTimeoutSteps(sc *godog.ScenarioContext) { sc.Step(`^the gateway should have timed out after "([^"]*)" seconds with status (\d+)$`, g.timedOutAfter) sc.Step(`^the gateway should have responded within "([^"]*)" seconds$`, g.respondedWithin) } func (g *Gateway) timedOutAfter(ctx context.Context, wantSeconds string, wantStatus int) error { resp, err := httpx.Published(ctx) if err != nil { return err } want, err := parseSeconds(wantSeconds) if err != nil { return err } floor := time.Duration(want * (1 - elapsedTolerance) * float64(time.Second)) if resp.Elapsed < floor { return fmt.Errorf("expected the gateway to take at least %ss before timing out, but %s returned after %s", wantSeconds, resp.Describe(), resp.Elapsed.Round(time.Millisecond)) } if resp.StatusCode != wantStatus { return fmt.Errorf("expected the timeout to surface as status %d, got %s", wantStatus, resp.Describe()) } return nil } func (g *Gateway) respondedWithin(ctx context.Context, wantSeconds string) error { resp, err := httpx.Published(ctx) if err != nil { return err } want, err := parseSeconds(wantSeconds) if err != nil { return err } ceiling := time.Duration(want * (1 + elapsedTolerance) * float64(time.Second)) if resp.Elapsed > ceiling { return fmt.Errorf("expected a response within %ss, but %s took %s", wantSeconds, resp.Describe(), resp.Elapsed.Round(time.Millisecond)) } return nil } func parseSeconds(value string) (float64, error) { seconds, err := strconv.ParseFloat(strings.TrimSpace(value), 64) if err != nil { return 0, fmt.Errorf("parsing expected seconds %q: %w", value, err) } if math.IsNaN(seconds) || math.IsInf(seconds, 0) || seconds < 0 { return 0, fmt.Errorf("expected seconds must be finite and non-negative, got %q", value) } return seconds, nil } // gatewayIsRunning asserts that the gateway health endpoint responds successfully. func (g *Gateway) gatewayIsRunning(ctx context.Context) error { url, err := g.adminURL("/health") if err != nil { return err } resp, err := g.funnel.Get(ctx, url, nil) if err != nil { return fmt.Errorf("the gateway is not answering its health endpoint: %w", err) } if !resp.Succeeded() { return fmt.Errorf("the gateway is unhealthy: %s", resp.Describe()) } return nil } func (g *Gateway) gatewayIsRunningAt(ctx context.Context, ordinalWord string) error { url, err := g.adminURLAt(ordinalWord, "/health") if err != nil { return err } resp, err := g.funnel.Get(ctx, url, nil) if err != nil { return fmt.Errorf("the %s gateway is not answering its health endpoint: %w", ordinalWord, err) } if !resp.Succeeded() { return fmt.Errorf("the %s gateway is unhealthy: %s", ordinalWord, resp.Describe()) } return nil } // authenticateAs stores the credentials subsequent steps send. // // The credentials are supplied by the test overlay and published in shared scope so all // gateway steps use the same authentication path. func (g *Gateway) authenticateAs(ctx context.Context, who string) error { userKey, passKey := frameworkruntime.KeyAdminUser, frameworkruntime.KeyAdminPass switch who { case "admin": case "consumer": userKey, passKey = frameworkruntime.KeyConsumerUser, frameworkruntime.KeyConsumerPass case "developer": userKey, passKey = frameworkruntime.KeyDeveloperUser, frameworkruntime.KeyDeveloperPass default: return fmt.Errorf("unknown actor %q: supported actors are admin, consumer, and developer", who) } user, err := tcontext.ResolveString(ctx, userKey) if err != nil { return err } pass, err := tcontext.ResolveString(ctx, passKey) if err != nil { return err } return tcontext.Set(ctx, keyAuthHeader, BasicAuthHeader(user, pass)) } // createResource creates one resource of the kind named in the step, which must match the kind // the definition declares. // // API creation keeps its own path: it alone registers cleanup and waits for the deployment to // become visible. func (g *Gateway) createResource(ctx context.Context, kind string, body *godog.DocString) error { spec, ok := resourceKinds[kind] if !ok { return fmt.Errorf("unknown resource kind %q", kind) } if got := kindFromDefinition(body.Content); got != spec.declared { return fmt.Errorf("step creates %s but the definition declares kind %q, want %q", kind, got, spec.declared) } if spec.collection == "" { return g.createAPI(ctx, body, "application/yaml") } return g.mutateResource(ctx, http.MethodPost, spec.collection, "", body) } func (g *Gateway) createJSONAPI(ctx context.Context, body *godog.DocString) error { return g.createAPI(ctx, body, "application/json") } // updateResource replaces an existing resource of the kind named in the step. // // The declared kind is checked only when the definition states one: a few LlmProxy update bodies // omit it, and rejecting those would fail scenarios over a field they never set. func (g *Gateway) updateResource( ctx context.Context, kind, name string, body *godog.DocString, ) error { spec, ok := resourceKinds[kind] if !ok { return fmt.Errorf("unknown resource kind %q", kind) } if got := kindFromDefinition(body.Content); got != "" && got != spec.declared { return fmt.Errorf("step updates %s but the definition declares kind %q, want %q", kind, got, spec.declared) } if spec.collection == "" { return g.updateAPI(ctx, name, body) } return g.mutateResource(ctx, http.MethodPut, spec.collection, name, body) } // getResource retrieves one resource of the kind named in the step. func (g *Gateway) getResource(ctx context.Context, kind, name string) error { spec, ok := resourceKinds[kind] if !ok { return fmt.Errorf("unknown resource kind %q", kind) } if spec.collection == "" { return g.getAPI(ctx, name) } resolved, err := stepscommon.Expand(ctx, name) if err != nil { return err } url, err := g.serviceURL(ctx, "gateway-controller", spec.collection+"/"+resolved) if err != nil { return err } _, err = g.funnel.Get(ctx, url, g.scenarioHeaders(ctx)) return err } // pluralResourceKinds maps the plural phrasing a "list all" step names to its resourceKinds key. var pluralResourceKinds = map[string]string{ "LLM providers": "LLM provider", "LLM provider templates": "LLM provider template", "MCP proxies": "MCP proxy", "LLM proxies": "LLM proxy", } // listResources lists every resource in the collection for the kind named in the step. func (g *Gateway) listResources(ctx context.Context, kindPlural string) error { kind, ok := pluralResourceKinds[kindPlural] if !ok { return fmt.Errorf("unknown resource kind %q", kindPlural) } spec := resourceKinds[kind] url, err := g.serviceURL(ctx, "gateway-controller", spec.collection) if err != nil { return err } _, err = g.funnel.Get(ctx, url, g.scenarioHeaders(ctx)) return err } // deleteResource removes a resource of the kind named in the step. func (g *Gateway) deleteResource(ctx context.Context, kind, name string) error { spec, ok := resourceKinds[kind] if !ok { return fmt.Errorf("unknown resource kind %q", kind) } if spec.collection == "" { return g.deleteAPI(ctx, name) } return g.mutateResource(ctx, http.MethodDelete, spec.collection, name, nil) } // resourceKinds maps the kind named in a step to the kind its definition must declare and the // controller collection it lives in. An empty collection means the API-specific handlers own // that path: they alone register and deregister cleanup. var resourceKinds = map[string]struct{ declared, collection string }{ "API": {"RestApi", ""}, "LLM provider": {"LlmProvider", collLLMProviders}, "LLM provider template": {"LlmProviderTemplate", collLLMTemplates}, "MCP proxy": {"Mcp", collMCPProxies}, "LLM proxy": {"LlmProxy", collLLMProxies}, "Agent": {"Agent", collAgents}, "GraphQL API": {"GraphQLApi", collGraphQLAPIs}, "Backend": {"Backend", collBackends}, } // kindFromDefinition returns the top-level kind a definition declares. func kindFromDefinition(def string) string { metadata := resourceMetadata{} if err := yaml.Unmarshal([]byte(def), &metadata); err != nil { return "" } return metadata.Kind } // createAPI posts an API definition and waits for it to become routable. // // The migrated behaviour differs from the original in ONE respect, deliberately: the // original slept a fixed second for xDS propagation. A fixed sleep is both too long when // propagation is fast and too short when a loaded host is slow — the second case being a // flake that looks like a product bug. This polls instead, bounded by the shared ceiling. func (g *Gateway) createAPI(ctx context.Context, body *godog.DocString, contentType string) error { // The definition may name resources; expanding placeholders is what keeps two concurrent // scenarios from creating the same API. definition, err := stepscommon.Expand(ctx, body.Content) if err != nil { return err } url, err := g.managementURL("/rest-apis") if err != nil { return err } if err := g.beforeAPIMutation(ctx); err != nil { return err } resp, err := g.funnel.Post(ctx, url, g.headerWith(ctx, "Content-Type", contentType), []byte(definition)) if err != nil { return err } if !resp.Succeeded() { // Returned rather than asserted here: a scenario may deploy an invalid definition on // purpose and assert the rejection itself. return nil } name := apiNameFrom(definition) if name == "" { return nil } expectInDump(ctx, name, true) if err := tcontext.Set(ctx, keyLastAPIName, name); err != nil { return err } // Registered as soon as it exists, and BEFORE anything else can fail: a resource created // but not registered is a resource that leaks when the next step errors. if err := cleanup.Register(ctx, cleanup.Resource{ Kind: cleanup.KindAPI, ID: name, Actor: "admin", Description: "deployed by " + scenarioLabel(ctx), }); err != nil { deleteURL, urlErr := g.managementURL("/rest-apis/" + name) if urlErr == nil { if deleteErr := g.compensateDelete(ctx, deleteURL, g.scenarioHeaders(ctx)); deleteErr != nil { return fmt.Errorf("registering API %q for cleanup: %w; compensation failed: %v", name, err, deleteErr) } } return fmt.Errorf("registering API %q for cleanup: %w", name, err) } return g.awaitDeployed(ctx, name) } // awaitDeployed polls the management API until the deployment is visible. func (g *Gateway) awaitDeployed(ctx context.Context, name string) error { url, err := g.managementURL("/rest-apis/" + name) if err != nil { return err } headers := g.scenarioHeaders(ctx) accept := func(r *httpx.Response) bool { return r != nil && r.Succeeded() } last, err := retry.Until(ctx, retry.Options{Timeout: 30 * time.Second, Interval: 200 * time.Millisecond}, func(ctx context.Context) (*httpx.Response, error) { // The raw client, not the funnel: this is an intermediate read and must not // become the response the next assertion targets. return g.funnel.Client().Do(ctx, httpx.Request{ Method: http.MethodGet, URL: url, Headers: headers, }, 0, 0) }, accept, ) return awaited(last, err, accept, fmt.Sprintf("API %q was accepted but never became retrievable", name)) } func (g *Gateway) getAPI(ctx context.Context, name string) error { resolved, err := stepscommon.Expand(ctx, name) if err != nil { return err } url, err := g.managementURL("/rest-apis/" + resolved) if err != nil { return err } _, err = g.funnel.Get(ctx, url, g.scenarioHeaders(ctx)) return err } func (g *Gateway) deleteAPI(ctx context.Context, name string) error { resolved, err := stepscommon.Expand(ctx, name) if err != nil { return err } url, err := g.managementURL("/rest-apis/" + resolved) if err != nil { return err } if err := g.beforeAPIMutation(ctx); err != nil { return err } resp, err := g.funnel.Delete(ctx, url, g.scenarioHeaders(ctx)) if err != nil { return err } if resp.Succeeded() { // The test deleted it and will assert on that, so teardown must not try again and // log a spurious leak. if reg, err := cleanup.Of(ctx); err == nil { reg.Deregister(cleanup.KindAPI, resolved) } // A later config-dump read must wait for the removal to propagate, not for the // handle to appear. expectInDump(ctx, resolved, false) } return nil } // ── Assertions, unchanged in meaning from the original suite ───────────────────── // serviceEndpoints maps the names features use for a SERVICE onto the component endpoint // that now serves it. // // These names are a legacy of the era when the gateway was two separately addressed // containers: "gateway-controller" and "gateway-controller-admin" name CONTAINERS, which is // precisely the distinction the platform-gateway component exists to hide. They are mapped // rather than rewritten because they appear 134 times across the suite, and rewriting them is // a vocabulary change worth doing deliberately in one pass rather than smuggling into a // migration. Recorded in the ledger as outstanding. // Each entry carries the API BASE PATH too, because features address these services with a // bare resource path ("/rest-apis", "/certificates"). The version prefix belongs to the // product's API contract, not to the feature, and keeping it here means a version bump is one // edit rather than a rewrite of every scenario. var serviceEndpoints = map[string]struct { component, endpoint, basePath string partitioned bool }{ "gateway-controller": {component: "platform-gateway", endpoint: "rest", basePath: ManagementBasePath}, "gateway-controller-admin": {component: "platform-gateway", endpoint: "admin", basePath: adminBasePath}, // The admin API of the controller that feeds the runtime over xDS, on a gateway that // runs it apart from the management controller. "gateway-controller-xds-admin": {component: "platform-gateway", endpoint: "xds-admin", basePath: adminBasePath}, "policy-engine": {component: "platform-gateway", endpoint: "policy-admin"}, "analytics": {component: "testbench", endpoint: "analytics", partitioned: true}, "capture": {component: "testbench", endpoint: "capture", partitioned: true}, "oauth2": {component: "testbench", endpoint: "oauth2", partitioned: true}, // The TypeSafe Jev API. Not block-partitioned here: a scenario addresses its own partition // as the first path segment, so request counts are scoped to the scenario. "jev": {component: "testbench", endpoint: "jev"}, "jev-tls": {component: "testbench", endpoint: "jev-tls"}, // The scripted MCP server, addressed like jev: a scenario names its own partition. "mcp-scripted": {component: "testbench", endpoint: "mcp-scripted"}, // The stateless MCP server (port 3009), addressed directly so a scenario can compare what // the gateway serves with what the server itself serves. "mcp": {component: "testbench", endpoint: "mcp"}, // Metrics live on a DIFFERENT compose service from the one tests normally address — // controller metrics on the controller, policy-engine metrics on the runtime — which the // component contract resolves via Endpoint.Service. No base path: a scrape is not an API. "controller-metrics": {component: "platform-gateway", endpoint: "metrics"}, "policy-engine-metrics": {component: "platform-gateway", endpoint: "pe-metrics"}, "envoy-admin": {component: "platform-gateway", endpoint: "envoy-admin"}, // The A2A agent behind every Agent, addressed directly so a scenario can compare what the // gateway serves with what the agent itself serves. "a2a-trip-planner": {component: "a2a-trip-planner", endpoint: "http"}, } // serviceURL resolves a feature's service name and path to a URL on the running topology. func (g *Gateway) serviceURL(ctx context.Context, service, path string) (string, error) { spec, ok := serviceEndpoints[service] if !ok { return "", fmt.Errorf("unknown service %q: this suite addresses %v", service, sortedServiceNames()) } base, err := g.topo.URL(spec.component, spec.endpoint) if err != nil { return "", err } resolved, err := stepscommon.Expand(ctx, path) if err != nil { return "", err } if resolved != "" && !strings.HasPrefix(resolved, "/") { resolved = "/" + resolved } if spec.partitioned { if g.topo.Block == nil { return "", fmt.Errorf("service %q requires a block partition, but the topology has no block", service) } resolved = "/" + g.topo.Block.PartitionKey() + resolved } return base + g.serviceBasePath(service, spec.basePath) + resolved, nil } // serviceUpstreamURL resolves a service address for a request originating inside the // block network. Unlike serviceURL, it must not use a mapped host port: containers in // the block reach shared services through their Docker network alias and container port. func (g *Gateway) serviceUpstreamURL(ctx context.Context, service, path string) (string, error) { spec, ok := serviceEndpoints[service] if !ok { return "", fmt.Errorf("unknown service %q: this suite addresses %v", service, sortedServiceNames()) } inst, err := g.topo.Component(spec.component) if err != nil { return "", err } base, err := inst.InternalURL(spec.endpoint) if err != nil { return "", err } resolved, err := stepscommon.Expand(ctx, path) if err != nil { return "", err } if resolved != "" && !strings.HasPrefix(resolved, "/") { resolved = "/" + resolved } if spec.partitioned { if g.topo.Block == nil { return "", fmt.Errorf("service %q requires a block partition, but the topology has no block", service) } resolved = "/" + g.topo.Block.PartitionKey() + resolved } return base + g.serviceBasePath(service, spec.basePath) + resolved, nil } func (g *Gateway) serviceBasePath(service, defaultPath string) string { switch service { case "gateway-controller": return gatewayManagementBasePath(g.topo) case "gateway-controller-admin", "gateway-controller-xds-admin": return gatewayAdminBasePath(g.topo) default: return defaultPath } } // resolveServiceURLAndStore resolves a testbench service's URL and stores it in local scope, for // use as an upstream target elsewhere in the scenario — most notably a partitioned service like // "capture", whose address includes a block key a feature cannot know in advance. func (g *Gateway) resolveServiceURLAndStore(ctx context.Context, service, path, key string) error { url, err := g.serviceUpstreamURL(ctx, service, path) if err != nil { return err } key = strings.TrimSpace(key) if key == "" { return fmt.Errorf("cannot store a resolved service URL with an empty key") } return tcontext.Set(ctx, key, url) } // adminResourceKinds maps the resource-kind names features use for the generic cleanup // registration step onto the cleanup kind and admin-API collection path that owns it. var adminResourceKinds = map[string]struct { kind cleanup.Kind collection string }{ "certificate": {cleanup.KindCertificate, "/certificates"}, "secret": {cleanup.KindSecret, "/secrets"}, } // registerAdminResourceForCleanup registers a resource managed through the gateway-controller // admin API (a certificate, a secret, ...) for cleanup, using an ID resolved from scenario // context. If registration itself fails, it makes a best-effort compensating delete against the // resource's own admin endpoint rather than leaking it. func (g *Gateway) registerAdminResourceForCleanup(ctx context.Context, kindName, idExpr string) error { entry, ok := adminResourceKinds[kindName] if !ok { return fmt.Errorf("unknown cleanup resource kind %q: this suite recognizes %v", kindName, sortedAdminResourceKindNames()) } id, err := stepscommon.Expand(ctx, idExpr) if err != nil { return err } if err := cleanup.Register(ctx, cleanup.Resource{ Kind: entry.kind, ID: id, Actor: "admin", Description: "created by " + scenarioLabel(ctx), }); err != nil { deleteURL, urlErr := g.serviceURL(ctx, "gateway-controller", entry.collection+"/"+id) if urlErr == nil { if deleteErr := g.compensateDelete(ctx, deleteURL, g.scenarioHeaders(ctx)); deleteErr != nil { return fmt.Errorf("registering %s %q for cleanup: %w; compensation failed: %v", kindName, id, err, deleteErr) } } return fmt.Errorf("registering %s %q for cleanup: %w", kindName, id, err) } return nil } func sortedAdminResourceKindNames() []string { out := make([]string, 0, len(adminResourceKinds)) for k := range adminResourceKinds { out = append(out, k) } sort.Strings(out) return out } func sortedServiceNames() []string { out := make([]string, 0, len(serviceEndpoints)) for k := range serviceEndpoints { out = append(out, k) } sort.Strings(out) return out } // serviceRequest invokes a component's own API, rather than the data plane. func (g *Gateway) serviceRequest(ctx context.Context, method, service, path string) error { return g.serviceRequestWithBody(ctx, method, service, path, nil) } // serviceRequestWithBody invokes a component's own API, with a body when one is given. func (g *Gateway) serviceRequestWithBody( ctx context.Context, method, service, path string, body *godog.DocString, ) error { method = strings.ToUpper(method) url, err := g.serviceURL(ctx, service, path) if err != nil { return err } if err := g.awaitMappedTestbenchService(ctx, service); err != nil { return err } // The config dump lags the deploy by one event-hub poll and nothing else exposes that, so // the framework waits here rather than making every scenario encode the timing. if method == http.MethodGet && service != "envoy-admin" && strings.HasPrefix(strings.TrimPrefix(path, "/"), "config_dump") { if err := g.awaitDumpConsistent(ctx, url); err != nil { return err } } var payload []byte headers := g.scenarioHeaders(ctx) if body != nil { content, expErr := stepscommon.Expand(ctx, body.Content) if expErr != nil { return expErr } payload = []byte(content) if headers["Content-Type"] == "" { headers["Content-Type"] = "application/json" } } return g.invokeService(ctx, method, url, headers, payload) } // A shared component is one container for the whole run, attached to each block's network // as that block boots and detached as it ends. Docker re-programs the container's // networking on every attachment, and its published host ports stop forwarding for a // moment while that happens — so a call over 127.0.0.1 landing in that window is refused, // by a container that is up, answered the same port moments earlier, and answers it again // moments later. The block that draws the short straw is whichever one happened to call // while a *different* block was booting, and it fails for a reason that has nothing to do // with what it was testing. // // StableHostPorts already exists for this hazard, and pins the port number across // attachments; it cannot keep the forwarding continuously live. A few short re-dials cover // the rest. Deliberately not the retry package: that one waits for a value to propagate and // floors its deadline at a minute, where this waits only for forwarding to come back, which // happens at once or not at all. Anything that answers — including an error status — is the // component's own reply and is returned as it is, so a scenario's real failure stays a fast // one. const ( serviceDialAttempts = 4 serviceDialPause = 400 * time.Millisecond ) // wsaeconnrefused is Winsock's code for a refused connection. Named here rather than taken // from golang.org/x/sys/windows so this file builds on every platform: on a non-Windows // build it is simply an errno nothing produces, which is exactly what makes it safe to test // for everywhere. const wsaeconnrefused = syscall.Errno(10061) // refusedConnection reports whether a request was turned away before a connection existed. // // Two conditions, both required. The dial is what must have failed — a component that // answered has answered, however badly, and its reply is never re-sent. And the failure // must be a refusal specifically: a dial that timed out is a host that is not coming back // within this step's patience, and retrying it three more times would spend three more // request timeouts to reach the same answer. Both platform spellings of "refused" count, // because the one Go names is a fabricated value on Windows. func refusedConnection(err error) bool { var dialErr *net.OpError if !errors.As(err, &dialErr) || dialErr.Op != "dial" { return false } return errors.Is(err, syscall.ECONNREFUSED) || errors.Is(err, wsaeconnrefused) } // invokeService calls a component's own API, re-dialling through a refused connection. func (g *Gateway) invokeService( ctx context.Context, method, url string, headers map[string]string, payload []byte, ) error { var err error for attempt := range serviceDialAttempts { err = g.invokeWith(ctx, method, url, headers, payload) if err == nil || !refusedConnection(err) { return err } if attempt == serviceDialAttempts-1 { break } select { case <-ctx.Done(): return ctx.Err() case <-time.After(serviceDialPause): } } return err } // awaitMappedTestbenchService verifies the host-side mapped port used by a direct // testbench request. Testbench exposes one health endpoint per service port; probing the // same endpoint as the request catches a stale or unavailable mapped port without changing // the request or its response assertion. func (g *Gateway) awaitMappedTestbenchService(ctx context.Context, service string) error { spec, ok := serviceEndpoints[service] if !ok || spec.component != "testbench" { return nil } base, err := g.topo.URL(spec.component, spec.endpoint) if err != nil { return err } healthURL := strings.TrimRight(base, "/") + "/testbench/health" return retry.Await(ctx, retry.Options{Interval: 2 * time.Second}, func(ctx context.Context) (*httpx.Response, error) { resp, requestErr := g.funnel.Client().Do(ctx, httpx.Request{ Method: http.MethodGet, URL: healthURL, }, 0, 0) if requestErr != nil { return nil, retry.Transient(requestErr) } return resp, nil }, func(resp *httpx.Response) bool { return resp != nil && resp.StatusCode == http.StatusOK }, fmt.Sprintf("waiting for testbench service %q mapped endpoint", service), ) } // serviceRequestUntilStatus polls a component endpoint until it returns the // expected status and publishes the confirming response. func (g *Gateway) serviceRequestUntilStatus(ctx context.Context, method, service, path string, want int) error { method = strings.ToUpper(method) url, err := g.serviceURL(ctx, service, path) if err != nil { return err } err = retry.Await(ctx, retry.Options{Interval: 2 * time.Second}, func(ctx context.Context) (*httpx.Response, error) { resp, requestErr := g.funnel.Send(ctx, httpx.Request{ Method: method, URL: url, Headers: g.scenarioHeaders(ctx), }) if requestErr != nil { return nil, retry.Transient(requestErr) } return resp, nil }, func(resp *httpx.Response) bool { return resp != nil && resp.StatusCode == want }, fmt.Sprintf("waiting for %s %s to return %d", method, url, want), ) if err != nil { return err } return nil } // serviceRequestUntilBodyNotContains polls a service path until its response body no longer // contains want - used for a diagnostic endpoint like config_dump, whose snapshot may lag a // moment behind a just-completed controller mutation. func (g *Gateway) serviceRequestUntilBodyNotContains( ctx context.Context, method, service, path, want string, ) error { method = strings.ToUpper(method) url, err := g.serviceURL(ctx, service, path) if err != nil { return err } resolvedWant, err := stepscommon.Expand(ctx, want) if err != nil { return err } err = retry.Await(ctx, retry.Options{Interval: 500 * time.Millisecond}, func(ctx context.Context) (*httpx.Response, error) { resp, requestErr := g.funnel.Send(ctx, httpx.Request{ Method: method, URL: url, Headers: g.scenarioHeaders(ctx), }) if requestErr != nil { return nil, retry.Transient(requestErr) } return resp, nil }, func(resp *httpx.Response) bool { return resp != nil && resp.Succeeded() && !strings.Contains(string(resp.Body), resolvedWant) }, fmt.Sprintf("waiting for %s %s to stop containing %q", method, url, resolvedWant), ) return err } // serviceLogsContain waits until the service log contains text, and fails if it never does. func (g *Gateway) serviceLogsContain(ctx context.Context, service, text string) error { text, err := expandLogText(ctx, text) if err != nil { return err } stack, resolved, err := g.topo.ServiceControl(service) if err != nil { return err } return awaitLogs(ctx, stack.Logs, func(logs string) bool { return strings.Contains(logs, text) }, fmt.Sprintf("%s logs for service %q to contain %q", resolved, service, text)) } // serviceLogsNotContain checks the service log as it stands. A log only grows, so absence is // checked once rather than waited for. func (g *Gateway) serviceLogsNotContain(ctx context.Context, service, text string) error { text, err := expandLogText(ctx, text) if err != nil { return err } stack, _, err := g.topo.ServiceControl(service) if err != nil { return err } if strings.Contains(stack.Logs(ctx), text) { return fmt.Errorf("service %q logs unexpectedly contain %q", service, text) } return nil } // serviceLogEvent waits for a log line containing marker. "should contain" passes once any such // line also contains text; "should not contain" checks the first such line, the event the marker // identifies, so a line another logger writes later with the same marker does not count. func (g *Gateway) serviceLogEvent(ctx context.Context, service, marker, negation, text string) error { marker, err := expandLogText(ctx, marker) if err != nil { return err } text, err = expandLogText(ctx, text) if err != nil { return err } stack, resolved, err := g.topo.ServiceControl(service) if err != nil { return err } return awaitLogEvent(ctx, stack.Logs, marker, text, negation == "", fmt.Sprintf("%s logs for service %q", resolved, service)) } // awaitLogEvent waits for a line of read's log containing marker. With want, it waits until one // such line also contains text; without, it fails if the first such line contains text. func awaitLogEvent( ctx context.Context, read func(context.Context) string, marker, text string, want bool, where string, ) error { var events []string accept := func(logs string) bool { events = linesContainingAll(logs, []string{marker}) if !want { return len(events) > 0 } return slices.ContainsFunc(events, func(line string) bool { return strings.Contains(line, text) }) } what := fmt.Sprintf("%s to hold an event containing %q", where, marker) if want { what = fmt.Sprintf("%s to hold an event containing %q and %q", where, marker, text) } if err := awaitLogs(ctx, read, accept, what); err != nil { return err } if !want && strings.Contains(events[0], text) { return fmt.Errorf("%s: the event containing %q also contains %q: %s", where, marker, text, truncateLog(events[0])) } return nil } // awaitLogs polls read until accept holds. It fails when accept never held, which a bare // retry.Until does not report; the failure shows the end of the last log read. func awaitLogs(ctx context.Context, read func(context.Context) string, accept func(string) bool, what string) error { return retry.Await(ctx, retry.Options{Interval: 200 * time.Millisecond, Fast: true}, func(ctx context.Context) (logSnapshot, error) { return logSnapshot(read(ctx)), nil }, func(logs logSnapshot) bool { return accept(string(logs)) }, "waiting for "+what) } // logSnapshot is one read of a service log. Describe keeps a failure message to the log's end, // where a missing line would have been written. type logSnapshot string // Describe returns the end of the log. func (l logSnapshot) Describe() string { return truncateLog(string(l)) } // expandLogText resolves ${CTX:...} and ${UNIQUE:...} placeholders in a log step's text, which // must not be blank once resolved. func expandLogText(ctx context.Context, text string) (string, error) { resolved, err := stepscommon.Expand(ctx, text) if err != nil { return "", err } if strings.TrimSpace(resolved) == "" { return "", fmt.Errorf("service log text %q must not be blank", text) } return resolved, nil } // logCountSettle is how long serviceLogLinesCount keeps reading after the count is reached, so a // duplicate logged a moment after the first line is still counted. const logCountSettle = 300 * time.Millisecond // serviceLogLinesCount asserts how many log lines of a service contain every fragment in the // table, one fragment per row. It waits until at least that many lines exist, reads the log again // after logCountSettle, then requires exactly that many, so a line logged once per key can be told // apart from one logged on every request. The log is shared by every scenario on the gateway, so // the fragments should include a value only this scenario produces. The count must be at least one: // a log that could not be read is returned as a note that matches no fragment, so a zero count // would pass without the log being read. func (g *Gateway) serviceLogLinesCount(ctx context.Context, service string, want int, table *godog.Table) error { if want < 1 { return fmt.Errorf("the log line count must be at least 1, got %d; use \"the %q service logs should not contain\" to check that a line is absent", want, service) } fragments, err := logFragments(ctx, table) if err != nil { return err } stack, resolved, err := g.topo.ServiceControl(service) if err != nil { return err } var matched []string _, err = retry.Until(ctx, retry.Options{Interval: 200 * time.Millisecond}, func(ctx context.Context) ([]string, error) { matched = linesContainingAll(stack.Logs(ctx), fragments) return matched, nil }, func(lines []string) bool { return len(lines) >= want }) if err != nil { return fmt.Errorf("waiting for %d %s log line(s) of service %q containing %q: found %d: %w", want, resolved, service, fragments, len(matched), err) } if len(matched) <= want { select { case <-ctx.Done(): return ctx.Err() case <-time.After(logCountSettle): } matched = linesContainingAll(stack.Logs(ctx), fragments) } if len(matched) != want { return fmt.Errorf("service %q has %d log line(s) containing %q, want %d: %s", service, len(matched), fragments, want, truncateLog(strings.Join(matched, "\n"))) } return nil } // logFragments reads one non-empty fragment from each row of a single-column table, expanding any // ${CTX:...} reference. func logFragments(ctx context.Context, table *godog.Table) ([]string, error) { if table == nil || len(table.Rows) == 0 { return nil, fmt.Errorf("the log line step needs a table of fragments, one per row") } fragments := make([]string, 0, len(table.Rows)) for i, row := range table.Rows { if len(row.Cells) != 1 { return nil, fmt.Errorf("log fragment row %d must have one cell, has %d", i+1, len(row.Cells)) } fragment, err := stepscommon.Expand(ctx, row.Cells[0].Value) if err != nil { return nil, err } if strings.TrimSpace(fragment) == "" { return nil, fmt.Errorf("log fragment row %d is empty", i+1) } fragments = append(fragments, fragment) } return fragments, nil } // linesContainingAll returns the lines of logs that contain every fragment. func linesContainingAll(logs string, fragments []string) []string { var matched []string for _, line := range strings.Split(logs, "\n") { all := true for _, fragment := range fragments { if !strings.Contains(line, fragment) { all = false break } } if all && len(fragments) > 0 { matched = append(matched, line) } } return matched } func truncateLog(logs string) string { const max = 2000 if len(logs) <= max { return logs } return logs[len(logs)-max:] } // serviceRequestUntilLazyDisplayName polls a policy-engine config dump until a template has // the expected runtime display name, then publishes the confirming response. func (g *Gateway) serviceRequestUntilLazyDisplayName( ctx context.Context, method, service, path, resourceID, displayName string, ) error { if strings.ToUpper(method) != http.MethodGet || service != "policy-engine" { return fmt.Errorf("lazy-resource readiness requires a GET request to the policy-engine") } url, err := g.serviceURL(ctx, service, path) if err != nil { return err } resourceID, err = stepscommon.Expand(ctx, resourceID) if err != nil { return err } displayName, err = stepscommon.Expand(ctx, displayName) if err != nil { return err } accept := func(resp *httpx.Response) bool { return resp != nil && resp.Succeeded() && lazyDisplayNameMatches(resp.Body, resourceID, displayName) } last, err := retry.Until(ctx, retry.Options{Interval: 200 * time.Millisecond}, func(ctx context.Context) (*httpx.Response, error) { resp, requestErr := g.funnel.Client().Do(ctx, httpx.Request{ Method: http.MethodGet, URL: url, Headers: g.scenarioHeaders(ctx), }, 0, 0) if requestErr != nil { return nil, retry.Transient(requestErr) } return resp, nil }, accept) if err != nil { return fmt.Errorf("waiting for lazy resource %q to have display name %q: %w", resourceID, displayName, err) } if !accept(last) { return fmt.Errorf("lazy resource %q did not have display name %q: %s", resourceID, displayName, last.Describe()) } return g.funnel.Publish(ctx, last) } func lazyDisplayNameMatches(body []byte, resourceID, displayName string) bool { var dump configDump if err := json.Unmarshal(body, &dump); err != nil { return false } for _, resource := range dump.Lazy.ResourcesByType["LlmProviderTemplate"] { if resource.ID != resourceID { continue } spec, ok := resource.Resource["spec"].(map[string]any) return ok && spec["displayName"] == displayName } return false } type analyticsEvent struct { Request struct { URI string `json:"uri"` Verb string `json:"verb"` Headers map[string][]string `json:"headers"` } `json:"request"` Response struct { Status int `json:"status"` Headers map[string][]string `json:"headers"` } `json:"response"` Metadata map[string]any `json:"metadata"` UserID string `json:"user_id"` // A2A is the event's first-class A2A dimension block, a sibling of metadata. It is decoded // as a map because the assertions are about the published document: a typed mirror would // let a renamed or relocated field compile and pass. A2A map[string]any `json:"a2a"` } func (g *Gateway) analyticsHeader( ctx context.Context, path, mode, plane, header, value string, ) error { path, err := stepscommon.Expand(ctx, path) if err != nil { return err } event, err := g.latestAnalyticsEvent(ctx, path) if err != nil { return err } var headers map[string][]string if plane == "request" { headers = event.Request.Headers } else { headers = event.Response.Headers } actual, present := analyticsHeaderValue(headers, header) switch mode { case "contain": if !present { return fmt.Errorf("analytics event for %q does not contain %s header %q", path, plane, header) } if value != "" && actual != value { return fmt.Errorf("analytics event for %q has %s header %q value %q, want %q", path, plane, header, actual, value) } case "not contain": if present { return fmt.Errorf("analytics event for %q contains denied %s header %q with value %q", path, plane, header, actual) } default: return fmt.Errorf("unsupported analytics header assertion %q", mode) } return nil } // analyticsHeaderValues asserts the complete, ordered value list recorded for a header. // want is a comma-separated list; each entry is trimmed of surrounding whitespace. The // expected representation follows the Gateway under test (analyticsJoinsRepeatedHeaderValues). func (g *Gateway) analyticsHeaderValues(ctx context.Context, path, plane, header, want string) error { path, err := stepscommon.Expand(ctx, path) if err != nil { return err } event, err := g.latestAnalyticsEvent(ctx, path) if err != nil { return err } headers := event.Response.Headers if plane == "request" { headers = event.Request.Headers } joined := analyticsJoinsRepeatedHeaderValues(gatewayVersion(g.topo)) if err := analyticsHeaderValuesMatch(headers, header, want, joined); err != nil { return fmt.Errorf("analytics event for %q %s header: %w", path, plane, err) } return nil } // analyticsJoinsRepeatedHeaderValues reports whether the Gateway publishes the values of a // repeated header to the analytics collector as one value joined with ", " (RFC 9110 section // 5.3). Gateway 1.2.0 and older publish each value separately; later releases, source builds // and the latest images join them. func analyticsJoinsRepeatedHeaderValues(version string) bool { version = strings.TrimPrefix(strings.TrimSpace(version), "v") version, _, _ = strings.Cut(version, "-") parts := strings.Split(version, ".") if len(parts) != 3 { return true } major, majorErr := strconv.Atoi(parts[0]) minor, minorErr := strconv.Atoi(parts[1]) patch, patchErr := strconv.Atoi(parts[2]) if majorErr != nil || minorErr != nil || patchErr != nil || major < 0 || minor < 0 || patch < 0 { return true } return major > 1 || (major == 1 && (minor > 2 || (minor == 2 && patch > 0))) } // analyticsHeaderValuesMatch reports whether the header, matched case-insensitively, carries // exactly the comma-separated values in want, in order. When joined is set, the values are // expected as the single value the Gateway records for a repeated header. func analyticsHeaderValuesMatch(headers map[string][]string, header, want string, joined bool) error { var expected []string for _, value := range strings.Split(want, ",") { if value = strings.TrimSpace(value); value != "" { expected = append(expected, value) } } if len(expected) == 0 { return fmt.Errorf("no expected values given for header %q", header) } if joined { expected = []string{strings.Join(expected, ", ")} } for name, actual := range headers { if !strings.EqualFold(name, header) { continue } if !slices.Equal(actual, expected) { return fmt.Errorf("header %q has values %q, want %q", header, actual, expected) } return nil } return fmt.Errorf("header %q is absent", header) } // fetchAnalyticsEvents reads every event the collector has buffered for this block. func (g *Gateway) fetchAnalyticsEvents(ctx context.Context) ([]analyticsEvent, error) { url, err := g.serviceURL(ctx, "analytics", "/test/events") if err != nil { return nil, err } response, err := g.funnel.Client().Do(ctx, httpx.Request{ Method: http.MethodGet, URL: url, Headers: g.scenarioHeaders(ctx), }, 0, 0) if err != nil { return nil, retry.Transient(err) } if !response.Succeeded() { return nil, retry.Transient(fmt.Errorf("analytics events returned %s", response.Describe())) } var events []analyticsEvent if err := json.Unmarshal(response.Body, &events); err != nil { return nil, err } return events, nil } func (g *Gateway) latestAnalyticsEvent(ctx context.Context, path string) (*analyticsEvent, error) { var observed []string accept := func(event *analyticsEvent) bool { return event != nil && analyticsEventMatchesPath(event.Request.URI, path) } pollCtx, cancel := context.WithTimeout(ctx, 30*time.Second) defer cancel() last, err := retry.Until(pollCtx, retry.Options{Interval: time.Second}, func(ctx context.Context) (*analyticsEvent, error) { events, err := g.fetchAnalyticsEvents(ctx) if err != nil { return nil, err } for i := len(events) - 1; i >= 0; i-- { if events[i].Request.URI != "" { observed = append(observed, events[i].Request.URI) } if accept(&events[i]) { return &events[i], nil } } return nil, nil }, accept) if err != nil { return nil, fmt.Errorf("reading analytics event for path %q (observed URIs: %v): %w", path, observed, err) } if last == nil { return nil, fmt.Errorf("no analytics event found for request path %q (observed URIs: %v)", path, observed) } return last, nil } type perRequestAnalyticsHeader struct { plane string header string present bool prefix string } // analyticsPerRequestHeaders asserts that the path has exactly n events, one for each // request "-" identified by keyHeader, and that each event carries the table's // headers for its own request i: "contain" rows must equal "-". func (g *Gateway) analyticsPerRequestHeaders( ctx context.Context, path string, n int, keyHeader, keyPrefix string, table *godog.Table, ) error { path, err := stepscommon.Expand(ctx, path) if err != nil { return err } keyPrefix, err = stepscommon.Expand(ctx, keyPrefix) if err != nil { return err } expectations, err := perRequestAnalyticsHeaders(ctx, table) if err != nil { return err } accept := func(events []analyticsEvent) bool { return len(analyticsEventsForPath(events, path)) >= n } events, err := retry.Until(ctx, retry.Options{Timeout: 30 * time.Second, Interval: time.Second}, g.fetchAnalyticsEvents, accept) if err != nil { return fmt.Errorf("reading analytics events for path %q: %w", path, err) } return verifyPerRequestAnalyticsEvents(analyticsEventsForPath(events, path), n, keyHeader, keyPrefix, expectations) } func perRequestAnalyticsHeaders(ctx context.Context, table *godog.Table) ([]perRequestAnalyticsHeader, error) { if table == nil || len(table.Rows) == 0 { return nil, fmt.Errorf("a per-request analytics headers table is required") } expectations := make([]perRequestAnalyticsHeader, 0, len(table.Rows)) for i, row := range table.Rows { if row == nil || len(row.Cells) != 4 { return nil, fmt.Errorf("per-request analytics headers row %d must contain plane, header, "+ "presence and value prefix cells", i+1) } cells := make([]string, 4) for j, cell := range row.Cells { value, err := stepscommon.Expand(ctx, strings.TrimSpace(cell.Value)) if err != nil { return nil, err } cells[j] = value } expectation := perRequestAnalyticsHeader{plane: cells[0], header: cells[1], prefix: cells[3]} if expectation.plane != "request" && expectation.plane != "response" { return nil, fmt.Errorf("row %d: plane must be request or response, got %q", i+1, cells[0]) } if expectation.header == "" { return nil, fmt.Errorf("row %d: header name is empty", i+1) } switch cells[2] { case "contain": expectation.present = true if expectation.prefix == "" { return nil, fmt.Errorf("row %d: a contained header needs a value prefix", i+1) } case "not contain": if expectation.prefix != "" { return nil, fmt.Errorf("row %d: an absent header takes no value prefix", i+1) } default: return nil, fmt.Errorf("row %d: presence must be contain or not contain, got %q", i+1, cells[2]) } expectations = append(expectations, expectation) } return expectations, nil } func analyticsEventsForPath(events []analyticsEvent, path string) []analyticsEvent { var matched []analyticsEvent for _, event := range events { if analyticsEventMatchesPath(event.Request.URI, path) { matched = append(matched, event) } } return matched } func verifyPerRequestAnalyticsEvents( events []analyticsEvent, n int, keyHeader, keyPrefix string, expectations []perRequestAnalyticsHeader, ) error { if n <= 0 { return fmt.Errorf("the expected request count must be positive, got %d", n) } if len(events) != n { return fmt.Errorf("expected %d analytics events, got %d", n, len(events)) } seen := make(map[int]bool, n) for _, event := range events { key, ok := analyticsHeaderValue(event.Request.Headers, keyHeader) if !ok { return fmt.Errorf("an analytics event has no request header %q", keyHeader) } index, err := strconv.Atoi(strings.TrimPrefix(key, keyPrefix+"-")) if !strings.HasPrefix(key, keyPrefix+"-") || err != nil || index < 1 || index > n { return fmt.Errorf("analytics event request header %q has unexpected value %q", keyHeader, key) } if seen[index] { return fmt.Errorf("request %q was recorded in more than one analytics event", key) } seen[index] = true for _, expectation := range expectations { headers := event.Response.Headers if expectation.plane == "request" { headers = event.Request.Headers } actual, present := analyticsHeaderValue(headers, expectation.header) switch { case !expectation.present && present: return fmt.Errorf("analytics event for request %q contains denied %s header %q with value %q", key, expectation.plane, expectation.header, actual) case expectation.present && !present: return fmt.Errorf("analytics event for request %q does not contain %s header %q", key, expectation.plane, expectation.header) case expectation.present && actual != fmt.Sprintf("%s-%d", expectation.prefix, index): return fmt.Errorf("analytics event for request %q has %s header %q value %q, want %q", key, expectation.plane, expectation.header, actual, fmt.Sprintf("%s-%d", expectation.prefix, index)) } } } return nil } // resetAnalyticsCollector clears every event the testbench analytics collector has buffered // for this block, so a scenario's own counts aren't inflated by earlier scenarios' traffic. func (g *Gateway) resetAnalyticsCollector(ctx context.Context) error { url, err := g.serviceURL(ctx, "analytics", "/test/reset") if err != nil { return err } return retry.Await(ctx, retry.Options{}, func(ctx context.Context) (*httpx.Response, error) { resp, err := g.funnel.Client().Do(ctx, httpx.Request{ Method: http.MethodPost, URL: url, Headers: g.scenarioHeaders(ctx), }, 0, 0) if err != nil { // Reset is idempotent. A connection reset can happen after the mock has // cleared the partition but before the response reaches the client, so // retry only transport failures and never assertions or non-2xx responses. return nil, retry.Transient(err) } if !resp.Succeeded() { return resp, fmt.Errorf("resetting the analytics collector failed: %s", resp.Describe()) } return resp, nil }, func(resp *httpx.Response) bool { return resp != nil && resp.Succeeded() }, "resetting the analytics collector") } func (g *Gateway) analyticsEventCount(ctx context.Context) (int, error) { url, err := g.serviceURL(ctx, "analytics", "/test/events/count") if err != nil { return 0, err } resp, err := g.funnel.Client().Do(ctx, httpx.Request{ Method: http.MethodGet, URL: url, Headers: g.scenarioHeaders(ctx), }, 0, 0) if err != nil { return 0, retry.Transient(err) } if !resp.Succeeded() { return 0, retry.Transient(fmt.Errorf("analytics count returned %s", resp.Describe())) } var out struct { Count int `json:"count"` } if err := json.Unmarshal(resp.Body, &out); err != nil { return 0, err } return out.Count, nil } // analyticsEventCountAtLeast polls until the collector has buffered at least want events. func (g *Gateway) analyticsEventCountAtLeast(ctx context.Context, want int) error { accept := func(n int) bool { return n >= want } last, err := retry.Until(ctx, retry.Options{Timeout: 15 * time.Second, Interval: 500 * time.Millisecond}, func(ctx context.Context) (int, error) { return g.analyticsEventCount(ctx) }, accept) if err != nil { return err } if !accept(last) { return fmt.Errorf("expected at least %d analytics events, got %d", want, last) } return nil } // analyticsQuietWindow is how long the collector's count must stay unchanged for the events // to count as settled. const analyticsQuietWindow = 3 * time.Second // mtlsAnalyticsQuietWindow is the quiet window for mutual TLS scenarios. const mtlsAnalyticsQuietWindow = 2 * time.Second // settleAnalyticsEventCount waits for the collector's count to stop changing for a quiet // period. A bare threshold poll would return the instant the count first reaches a target and // could miss a late-arriving duplicate event landing just after - this is what actually // verifies "no more are coming", used both to check an exact count and to drain a prior // request's own publish delay before a scenario resets the collector for its real assertion. func (g *Gateway) settleAnalyticsEventCount(ctx context.Context) (retry.Settled, error) { quiet := analyticsQuietWindow if isMTLSScenario(ctx) { quiet = mtlsAnalyticsQuietWindow } settled, err := retry.SettledCount(ctx, retry.Options{Timeout: 12 * time.Second}, quiet, func(ctx context.Context) (int, error) { return g.analyticsEventCount(ctx) }) if err != nil { return settled, err } if !settled.Quiet { return settled, fmt.Errorf("analytics event count did not settle: last observed %d after %d sample(s)", settled.Value, settled.Samples) } return settled, nil } func (g *Gateway) analyticsEventCountExactly(ctx context.Context, want int) error { settled, err := g.settleAnalyticsEventCount(ctx) if err != nil { return err } if settled.Value != want { return fmt.Errorf("expected exactly %d analytics events, got %d", want, settled.Value) } return nil } // waitForAnalyticsToSettle waits until the collector's event count stops changing, without // asserting a specific value. Use it before resetting the collector, so an earlier request's // own delayed publish (moesif_base_url's publish_interval) can't land after the reset and // contaminate a subsequent exact-count assertion. func (g *Gateway) waitForAnalyticsToSettle(ctx context.Context) error { _, err := g.settleAnalyticsEventCount(ctx) return err } // analyticsEventForPath resolves path's placeholders and returns the event latestAnalyticsEvent // finds for it. Filtering by the path a scenario already knows it requested is what makes this // robust: the events array's arrival order at the collector doesn't necessarily match request // chronology, so picking "the last element" can return an unrelated event. func (g *Gateway) analyticsEventForPath(ctx context.Context, path string) (*analyticsEvent, error) { resolved, err := stepscommon.Expand(ctx, path) if err != nil { return nil, err } return g.latestAnalyticsEvent(ctx, resolved) } func (g *Gateway) analyticsRequestMethod(ctx context.Context, path, want string) error { event, err := g.analyticsEventForPath(ctx, path) if err != nil { return err } if event.Request.Verb != want { return fmt.Errorf("analytics event for %q has request method %q, want %q", path, event.Request.Verb, want) } return nil } func (g *Gateway) analyticsResponseStatus(ctx context.Context, path string, want int) error { event, err := g.analyticsEventForPath(ctx, path) if err != nil { return err } if event.Response.Status != want { return fmt.Errorf("analytics event for %q has response status %d, want %d", path, event.Response.Status, want) } return nil } func (g *Gateway) analyticsMetadataField(ctx context.Context, path, field, want string) error { event, err := g.analyticsEventForPath(ctx, path) if err != nil { return err } resolved, err := stepscommon.Expand(ctx, want) if err != nil { return err } // MCP analytics nests its fields under mcpAnalytics, so the field is a dotted path rather // than a key. traverseJSON resolves both: a name with no dot is a plain lookup. value, ok := traverseJSON(event.Metadata, field) if !ok { return fmt.Errorf("latest analytics event metadata has no field %q", field) } if got := fmt.Sprintf("%v", value); got != resolved { return fmt.Errorf("latest analytics event metadata field %q is %q, want %q", field, got, resolved) } return nil } // analyticsNumericMetadataField compares a numeric metadata field with a number: equal within a // relative tolerance, since a cost computed in floating point rarely prints as the decimal a // scenario states, or at least that number. func (g *Gateway) analyticsNumericMetadataField(ctx context.Context, path, field, op, want string) error { event, err := g.analyticsEventForPath(ctx, path) if err != nil { return err } resolved, err := stepscommon.Expand(ctx, want) if err != nil { return err } value, ok := traverseJSON(event.Metadata, field) if !ok { return fmt.Errorf("latest analytics event metadata has no field %q", field) } return numericMatches(field, value, op, resolved) } func analyticsHeaderValue(headers map[string][]string, wanted string) (string, bool) { for name, value := range headers { if strings.EqualFold(name, wanted) { if len(value) == 0 { return "", true } return value[0], true } } return "", false } // analyticsEventMatchesPath reports whether an event's request URI is the requested path. Query // strings are ignored on both sides: some gateway versions record the URI with the query string and // others without it. func analyticsEventMatchesPath(eventURI, requestedPath string) bool { eventURI, _, _ = strings.Cut(strings.TrimSpace(eventURI), "?") requestedPath, _, _ = strings.Cut(strings.TrimSpace(requestedPath), "?") requestedPath = strings.TrimRight(requestedPath, "/") if eventURI == "" || requestedPath == "" { return false } return requestedPath == eventURI || strings.HasSuffix(requestedPath, "/"+strings.TrimPrefix(eventURI, "/")) } // updateAPI replaces an existing API definition. // // The body is YAML like a deploy, so the content type is set explicitly — the default of // application/json would make the controller reject it with a JSON parse error naming the // first byte of "apiVersion" rather than the content type. func (g *Gateway) updateAPI(ctx context.Context, name string, body *godog.DocString) error { resolvedName, err := stepscommon.Expand(ctx, name) if err != nil { return err } definition, err := stepscommon.Expand(ctx, body.Content) if err != nil { return err } url, err := g.serviceURL(ctx, "gateway-controller", "/rest-apis/"+resolvedName) if err != nil { return err } if err := g.beforeAPIMutation(ctx); err != nil { return err } headers := g.headerWith(ctx, "Content-Type", "application/yaml") return g.invokeWith(ctx, http.MethodPut, url, headers, []byte(definition)) } // handleFromDefinition extracts metadata.name from a YAML definition, which is the handle the // config dump reports. func handleFromDefinition(def string) string { return apiNameFrom(def) } // Controller-managed resource collections, by the path segment that addresses them. const ( collLLMProviders = "/llm-providers" collLLMTemplates = "/llm-provider-templates" collMCPProxies = "/mcp-proxies" collLLMProxies = "/llm-proxies" collAgents = "/agents" collGraphQLAPIs = "/graphql-apis" collBackends = "/backends" ) // mutateResource creates, replaces or removes a controller resource and waits for the change // to be OBSERVABLE before returning. // // The original slept a flat second after every mutation. That is the anti-pattern this // framework removes: too long when the controller is fast, too short when it is loaded, and // the second case is a flake that reads as a product bug. What the sleep was actually waiting // for is that the resource has reached its new state, so that is what this waits for — // present after a create or update, absent after a delete. // // Strictly stronger than the sleep, too: a create that the controller ACCEPTED but never // persisted used to pass, because a second elapsed either way. func (g *Gateway) mutateResource( ctx context.Context, method, collection, id string, body *godog.DocString, ) error { markGatewayChanged(ctx) resolvedID, err := stepscommon.Expand(ctx, id) if err != nil { return err } var expandedBody string if body != nil { expandedBody, err = stepscommon.Expand(ctx, body.Content) if err != nil { return err } } path := collection if resolvedID != "" { path = collection + "/" + resolvedID } if method == http.MethodDelete { expectInDump(ctx, resolvedID, false) } else if expandedBody != "" { if h := handleFromDefinition(expandedBody); h != "" { expectInDump(ctx, h, true) } } url, err := g.serviceURL(ctx, "gateway-controller", path) if err != nil { return err } var payload []byte if expandedBody != "" { payload = []byte(expandedBody) } // YAML, like every other definition this controller accepts. Defaulting to JSON here // produces "failed to parse JSON: invalid character 'a'" — the first byte of apiVersion. headers := g.headerWith(ctx, "Content-Type", "application/yaml") if err := g.invokeWith(ctx, method, url, headers, payload); err != nil { return err } // A mutation the controller REJECTED is a legitimate assertion in several scenarios, so // only a successful one is waited on: waiting for a rejected create to appear would hang // until the ceiling and report a timeout instead of the 4xx the scenario is asserting. resp, err := httpx.Published(ctx) if err != nil || resp == nil || !resp.Succeeded() { return nil //nolint:nilerr // the response is the assertion; the next Then reads it } resourceID := resolvedID if method == http.MethodPost { if resourceID == "" && expandedBody != "" { resourceID = handleFromDefinition(expandedBody) } if kind, ok := cleanupKindForCollection(collection); ok && resourceID != "" { if err := cleanup.Register(ctx, cleanup.Resource{ Kind: kind, ID: resourceID, Actor: "admin", Description: "created by " + scenarioLabel(ctx), }); err != nil { deleteURL, urlErr := g.serviceURL(ctx, "gateway-controller", collection+"/"+resourceID) if urlErr == nil { if deleteErr := g.compensateDelete(ctx, deleteURL, g.scenarioHeaders(ctx)); deleteErr != nil { return fmt.Errorf("registering %s %q for cleanup: %w; compensation failed: %v", collection, resourceID, err, deleteErr) } } return fmt.Errorf("registering %s %q for cleanup: %w", collection, resourceID, err) } } } // The published response must survive: the next step asserts on the MUTATION's response, // not on the poll's. So the wait uses a bare client and restores it afterwards. settled := resp waitFor := func(want bool) error { target := collection + "/" + resourceID if resourceID == "" { return nil // a create without a known id has nothing to poll for } pollURL, err := g.serviceURL(ctx, "gateway-controller", target) if err != nil { return err } accept := func(r *httpx.Response) bool { if r == nil { return false } if want { return r.Succeeded() } return r.StatusCode == http.StatusNotFound } last, err := retry.Until(ctx, retry.Options{Interval: 200 * time.Millisecond}, func(ctx context.Context) (*httpx.Response, error) { return g.funnel.Client().Do(ctx, httpx.Request{ Method: http.MethodGet, URL: pollURL, Headers: g.scenarioHeaders(ctx), }, 0, 0) }, accept, ) return awaited(last, err, accept, fmt.Sprintf("waiting for %s to exist=%t", target, want)) } var waitErr error if method == http.MethodDelete { waitErr = waitFor(false) } else { waitErr = waitFor(true) } if waitErr != nil { return fmt.Errorf("%s %s did not become observable: %w", method, path, waitErr) } if method == http.MethodDelete { if kind, ok := cleanupKindForCollection(collection); ok { if reg, err := cleanup.Of(ctx); err == nil { reg.Deregister(kind, resourceID) } } } return tcontext.Set(ctx, httpx.ResponseKey, settled) } func (g *Gateway) compensateDelete(ctx context.Context, url string, headers map[string]string) error { resp, err := g.funnel.Client().Do(ctx, httpx.Request{ Method: http.MethodDelete, URL: url, Headers: headers, }, 0, 0) if err != nil { return err } if !resp.Succeeded() { return fmt.Errorf("delete returned %s", resp.Describe()) } return nil } func cleanupKindForCollection(collection string) (cleanup.Kind, bool) { switch collection { case collLLMProviders: return cleanup.KindLLMProvider, true case collLLMProxies: return cleanup.KindLLMProxy, true case collLLMTemplates: return cleanup.KindLLMProviderTemplate, true case collMCPProxies: return cleanup.KindMCPProxy, true case collAgents: return cleanup.KindAgent, true case collGraphQLAPIs: return cleanup.KindGraphQLAPI, true case collBackends: return KindBackend, true default: return cleanup.Kind{}, false } } // expectInDump records what the config dump must eventually show for a handle. // // present=true after a create, false after a delete. Tracking BOTH matters: a scenario that // deploys, checks the dump, deletes and checks again is asserting the dump reflects the // removal — and waiting only for appearance would block forever on the second read. func expectInDump(ctx context.Context, handle string, present bool) { handle = strings.TrimSpace(handle) if handle == "" { return } expectations := map[string]bool{} if v, ok := tcontext.Get(ctx, keyDumpExpectations); ok { if existing, ok := v.(map[string]bool); ok { for k, val := range existing { expectations[k] = val } } } expectations[handle] = present _ = tcontext.Set(ctx, keyDumpExpectations, expectations) } // awaitDumpConsistent polls the config dump until it reflects everything this scenario has // deployed. // // The config dump is EVENTUALLY CONSISTENT and nothing else exposes that. Measured on a real // gateway, immediately after a deploy returns 201: // // GET /rest-apis/{name} visible after 2ms (database) // GET /rest-apis visible after 3ms (database) // GET /config_dump visible after 151ms (in-memory store, via the event-hub poll) // // Only the dump reads s.store, which the EventListener populates on its poll cycle. So there // is no cheaper condition to wait on — no management endpoint lags with it, and waiting for // the API to be "deployed" proves nothing about the dump. // // The original suite covered this with a flat 1s sleep after every deploy, which worked only // because it exceeded the poll interval. Polling the dump for the handles this scenario // actually created is the same wait expressed as a condition: it returns as soon as the data // is there, and it fails loudly rather than silently reading a stale dump. func (g *Gateway) awaitDumpConsistent(ctx context.Context, url string) error { v, ok := tcontext.Get(ctx, keyDumpExpectations) if !ok { return nil // nothing created or removed here; the dump cannot be stale } expectations, _ := v.(map[string]bool) if len(expectations) == 0 { return nil } accept := func(r *httpx.Response) bool { if r == nil || !r.Succeeded() { return false } body := r.Text() for handle, wantPresent := range expectations { if strings.Contains(body, handle) != wantPresent { return false } } return true } last, err := retry.Until(ctx, retry.Options{Interval: 50 * time.Millisecond}, func(ctx context.Context) (*httpx.Response, error) { return g.funnel.Client().Do(ctx, httpx.Request{ Method: http.MethodGet, URL: url, Headers: g.scenarioHeaders(ctx), }, 0, 0) }, accept, ) return awaited(last, err, accept, fmt.Sprintf("the config dump never became consistent with %v", expectations)) } // oobTemplateList asserts the listing carries the out-of-box provider templates, exactly. func (g *Gateway) oobTemplateList(ctx context.Context) error { resp, err := httpx.Published(ctx) if err != nil { return err } return oobTemplateListIs(resp) } // oobTemplateListIs asserts the response carries EXACTLY the out-of-box provider templates. // // Order-insensitive: two captures of this endpoint came back in different orders, so the templates // are matched on metadata.name rather than by index. Every other field is compared, which is what // makes this exact rather than a presence check — the version this replaced accepted "at least" // the expected count and ignored any extra or altered template. // // status is excluded because createdAt/updatedAt are stamped per boot. They were the ONLY fields // that differed between the two captures; everything else was byte-identical. func oobTemplateListIs(resp *httpx.Response) error { var doc struct { Count int `json:"count"` Templates []map[string]any `json:"templates"` } if err := json.Unmarshal(resp.Body, &doc); err != nil { return fmt.Errorf("parsing the template list: %w (%s)", err, resp.Describe()) } var want []any if err := json.Unmarshal([]byte(oobProviderTemplates), &want); err != nil { return fmt.Errorf("the expected template fixture is not valid JSON: %w", err) } got := make([]any, 0, len(doc.Templates)) for _, t := range doc.Templates { got = append(got, t) } ignore := []string{"status"} gotByName, err := keyElements(got, "metadata.name", ignore) if err != nil { return fmt.Errorf("indexing the returned templates: %w (%s)", err, resp.Describe()) } wantByName, err := keyElements(want, "metadata.name", ignore) if err != nil { return fmt.Errorf("indexing the expected templates: %w", err) } // Presence, not equality of the whole set: other runners in this block create templates of // their own and the listing returns every one, so "count == 7" and "no extras" both failed // (observed: expected count 7, got 8). What IS pinned is that each out-of-box template is // present and matches its shipped definition field for field. var missing []string for n := range wantByName { if _, ok := gotByName[n]; !ok { missing = append(missing, n) } } sort.Strings(missing) if len(missing) > 0 { return fmt.Errorf("out-of-box templates missing from the listing: %v (%s)", missing, resp.Describe()) } for _, n := range sortedElementKeys(wantByName) { if !reflect.DeepEqual(gotByName[n], wantByName[n]) { g, _ := json.Marshal(gotByName[n]) w, _ := json.Marshal(wantByName[n]) return fmt.Errorf("out-of-box template %q differs:\n expected %s\n got %s", n, w, g) } } return nil } // oobProviderTemplates is the shipped out-of-box template set, captured from the product and // reduced to the fields that are stable across boots. Regenerate the JSON deliberately when the // product intends to change what it ships — that review is the point of pinning it. // // Embedded rather than read at runtime: the suite is a test binary that runs from whichever // directory `go test` chose, so a relative path would be a failure mode with no upside. // //go:embed fixtures/oob_provider_templates.json var oobProviderTemplates string // keyElements indexes array elements by the value at keyPath, dropping the ignored paths. func keyElements(arr []any, keyPath string, drop []string) (map[string]any, error) { out := make(map[string]any, len(arr)) for i, el := range arr { m, ok := el.(map[string]any) if !ok { return nil, fmt.Errorf("element %d is %T, not an object", i, el) } k, ok := traverseJSON(m, keyPath) if !ok { return nil, fmt.Errorf("element %d has no %q", i, keyPath) } key := fmt.Sprintf("%v", k) if _, dup := out[key]; dup { return nil, fmt.Errorf("two elements share %s=%s", keyPath, key) } pruned := deepCopyWithout(m, drop) out[key] = pruned } return out, nil } // deepCopyWithout copies a decoded JSON object minus the given dotted paths. func deepCopyWithout(node any, drop []string) any { switch v := node.(type) { case map[string]any: out := make(map[string]any, len(v)) for k, child := range v { skip := false var nested []string for _, d := range drop { d = strings.TrimSpace(d) if d == k { skip = true break } if rest, ok := strings.CutPrefix(d, k+"."); ok { nested = append(nested, rest) } } if skip { continue } out[k] = deepCopyWithout(child, nested) } return out case []any: out := make([]any, 0, len(v)) for _, child := range v { out = append(out, deepCopyWithout(child, drop)) } return out default: return node } } func sortedElementKeys(m map[string]any) []string { out := make([]string, 0, len(m)) for k := range m { out = append(out, k) } sort.Strings(out) return out }