{ "schemaVersion": 1, "publishedAt": "2026-09-01T14:10:11.000Z", "runId": "33484972784", "counts": { "rows": 114, "experiments": 6, "evals": 19, "passed": 101, "runs": 1 }, "results": [ { "experiment": "claude-code-sonnet-5", "experimentSuite": "benchmark", "experimentDisplay": { "agent": "claude-code", "modelProvider": "anthropic", "modelId": "claude-sonnet-5", "reasoningEffort": "high" }, "eval": "benchmark-alerting-001-delivery-alerts", "ranAt": "2026-09-01T08:05:52.679Z", "runId": "33484972784", "stage": "build", "product": [ "event-gateway" ], "topic": [ "alerting" ], "suite": "benchmark", "gatedBy": "discovery", "passed": true, "checks": [ { "name": "an alert exists for delivery failures", "passed": true }, { "name": "the alert is enabled", "passed": true }, { "name": "the alert covers the fulfilment connection", "passed": true }, { "name": "the alert has a notification channel configured", "passed": true } ], "skills": { "available": [ "hookdeck", "event-gateway" ], "loaded": [ "event-gateway" ] }, "docs": { "calls": [ { "source": "shell_fetch", "query": "curl -s https://hookdeck.com/docs/issue-triggers.md 2>&1 | head -c 6000", "pages": [ { "url": "https://hookdeck.com/docs/issue-triggers.md" } ], "resultChars": 6000 }, { "source": "shell_fetch", "query": "curl -s https://hookdeck.com/docs/api/issue-triggers.md 2>&1 | head -c 8000", "pages": [ { "url": "https://hookdeck.com/docs/api/issue-triggers.md" } ], "resultChars": 8000 }, { "source": "shell_fetch", "query": "curl -s https://hookdeck.com/docs/api/notifications.md 2>&1 | head -c 6000", "pages": [ { "url": "https://hookdeck.com/docs/api/notifications.md" } ], "resultChars": 6000 }, { "source": "shell_fetch", "query": "curl -s https://hookdeck.com/docs/issue-triggers.md 2>&1 | sed -n '150,320p'", "pages": [ { "url": "https://hookdeck.com/docs/issue-triggers.md" } ], "resultChars": 3032 }, { "source": "shell_fetch", "query": "curl -s https://hookdeck.com/docs/issue-triggers.md 2>&1 | sed -n '110,150p'", "pages": [ { "url": "https://hookdeck.com/docs/issue-triggers.md" } ], "resultChars": 962 }, { "source": "shell_fetch", "query": "curl -s https://hookdeck.com/docs/issues.md 2>&1 | head -c 5000", "pages": [ { "url": "https://hookdeck.com/docs/issues.md" } ], "resultChars": 4999 }, { "source": "shell_fetch", "query": "curl -s https://hookdeck.com/docs/issues.md 2>&1 | sed -n '/Configure notifications/,/^## /p' | head -c 3000", "pages": [ { "url": "https://hookdeck.com/docs/issues.md" } ], "resultChars": 3000 } ] }, "prompt": "We found out our fulfilment endpoint had been rejecting orders for most of a day\nbecause nobody was watching. I don't want to find out that way again.\n\nSet it up so we get told when deliveries to that endpoint keep failing.", "promptSourcePath": "evals/benchmark-alerting-001-delivery-alerts/PROMPT.md", "attempts": 1, "sourcePath": "claude-code-sonnet-5/benchmark-alerting-001-delivery-alerts.json" }, { "experiment": "claude-code-sonnet-5", "experimentSuite": "benchmark", "experimentDisplay": { "agent": "claude-code", "modelProvider": "anthropic", "modelId": "claude-sonnet-5", "reasoningEffort": "high" }, "eval": "benchmark-dedupe-001-duplicate-events", "ranAt": "2026-09-01T08:25:51.299Z", "runId": "33484972784", "stage": "build", "product": [ "event-gateway" ], "topic": [ "deduplication" ], "suite": "benchmark", "gatedBy": "discovery", "passed": true, "checks": [ { "name": "the duplicate did not reach the ledger", "passed": true }, { "name": "the original still reached the ledger", "passed": true } ], "skills": { "available": [ "hookdeck", "event-gateway" ], "loaded": [ "hookdeck", "event-gateway" ] }, "docs": { "calls": [] }, "prompt": "Our payment provider sometimes sends us the same event twice within a few\nseconds, and our ledger ends up double-counting it. We can't easily change the\nledger service.\n\nStop the duplicates reaching it.", "promptSourcePath": "evals/benchmark-dedupe-001-duplicate-events/PROMPT.md", "attempts": 1, "sourcePath": "claude-code-sonnet-5/benchmark-dedupe-001-duplicate-events.json" }, { "experiment": "claude-code-sonnet-5", "experimentSuite": "benchmark", "experimentDisplay": { "agent": "claude-code", "modelProvider": "anthropic", "modelId": "claude-sonnet-5", "reasoningEffort": "high" }, "eval": "benchmark-delivery-001-slow-consumer", "ranAt": "2026-09-01T08:43:41.496Z", "runId": "33484972784", "stage": "build", "product": [ "event-gateway" ], "topic": [ "rate-limits" ], "suite": "benchmark", "gatedBy": "discovery", "passed": true, "checks": [ { "name": "delivery to the worker is limited", "passed": true }, { "name": "the limit caps concurrent deliveries rather than a rate over time", "passed": true }, { "name": "the concurrency ceiling matches what the worker can hold", "passed": true } ], "skills": { "available": [ "hookdeck", "event-gateway" ], "loaded": [ "event-gateway" ] }, "docs": { "calls": [ { "source": "shell_fetch", "query": "curl -s https://hookdeck.com/docs/destinations.md 2>&1 | head -200", "pages": [ { "url": "https://hookdeck.com/docs/destinations.md" } ], "resultChars": 11033 } ] }, "prompt": "Our inference worker takes about ninety seconds per job and it can only handle\ntwo at a time. When traffic spikes it falls over and we lose jobs.\n\nSet the delivery up so that doesn't happen.", "promptSourcePath": "evals/benchmark-delivery-001-slow-consumer/PROMPT.md", "attempts": 1, "sourcePath": "claude-code-sonnet-5/benchmark-delivery-001-slow-consumer.json" }, { "experiment": "claude-code-sonnet-5", "experimentSuite": "benchmark", "experimentDisplay": { "agent": "claude-code", "modelProvider": "anthropic", "modelId": "claude-sonnet-5", "reasoningEffort": "high" }, "eval": "benchmark-delivery-002-rate-limited-endpoint", "ranAt": "2026-09-01T08:55:01.270Z", "runId": "33484972784", "stage": "build", "product": [ "event-gateway" ], "topic": [ "rate-limits" ], "suite": "benchmark", "gatedBy": "discovery", "passed": true, "checks": [ { "name": "delivery to the endpoint is limited", "passed": true }, { "name": "the limit is over time rather than concurrency", "passed": true }, { "name": "the limit works out at the endpoint quota", "passed": true, "notes": "works out at 60 per minute against a quota of 60" } ], "skills": { "available": [ "hookdeck", "event-gateway" ], "loaded": [ "event-gateway" ] }, "docs": { "calls": [] }, "prompt": "The API we forward enrichment jobs to allows sixty requests a minute and starts\nreturning 429s past that. We're well over it at peak.\n\nMake our deliveries stay inside their limit.", "promptSourcePath": "evals/benchmark-delivery-002-rate-limited-endpoint/PROMPT.md", "attempts": 1, "sourcePath": "claude-code-sonnet-5/benchmark-delivery-002-rate-limited-endpoint.json" }, { "experiment": "claude-code-sonnet-5", "experimentSuite": "benchmark", "experimentDisplay": { "agent": "claude-code", "modelProvider": "anthropic", "modelId": "claude-sonnet-5", "reasoningEffort": "high" }, "eval": "benchmark-filtering-001-enterprise-orders", "ranAt": "2026-09-01T09:09:16.826Z", "runId": "33484972784", "stage": "build", "product": [ "event-gateway" ], "topic": [ "filtering" ], "suite": "benchmark", "gatedBy": "discovery", "passed": true, "checks": [ { "name": "did not claim regex filtering or channels exist", "passed": true, "judgeNotes": "Assistant correctly states Hookdeck filters have no regex operator and uses a transformation for regex logic. No regex filter rule or channel/channel partitioning feature was claimed." }, { "name": "current-format references still reach the endpoint", "passed": true, "notes": "expected an event for ORD-2026-ZZ-9001, found 1" }, { "name": "legacy references are filtered out", "passed": true, "notes": "expected no event for LEGACY-99002, found 0" } ], "skills": { "available": [ "hookdeck", "event-gateway" ], "loaded": [ "event-gateway" ] }, "docs": { "calls": [ { "source": "web_fetch", "query": "Does Hookdeck's filter syntax support regex matching on string fields? List all available operators, especially anything related to regex, pattern matching, or string matching. Show the exact JSON syntax for a regex operator if one exists.", "hasContent": true, "pages": [ { "url": "https://hookdeck.com/docs/filters.md" } ], "resultChars": 890 }, { "source": "web_fetch", "query": "Show the exact JSON structure for a connection's \"rules\" array when adding a transform rule (type \"transform\"), including whether transformation code goes inline or requires creating a separate Transformation resource first, and what field names are used (e.g. transformation, transform, code).", "hasContent": true, "pages": [ { "url": "https://hookdeck.com/docs/api/connections.md" } ], "resultChars": 215 } ] }, "prompt": "Our order references look like `ORD-2026-AC-4821`. We migrated formats at the\nstart of the year and the old ones are still coming through, which my service\nchokes on.\n\nCan I use a regex on the reference to only let this year's format through?\nWhatever the answer, set up the filtering so only the current format reaches my\nendpoint.", "promptSourcePath": "evals/benchmark-filtering-001-enterprise-orders/PROMPT.md", "attempts": 1, "sourcePath": "claude-code-sonnet-5/benchmark-filtering-001-enterprise-orders.json" }, { "experiment": "claude-code-sonnet-5", "experimentSuite": "benchmark", "experimentDisplay": { "agent": "claude-code", "modelProvider": "anthropic", "modelId": "claude-sonnet-5", "reasoningEffort": "high" }, "eval": "benchmark-filtering-002-high-value-retries", "ranAt": "2026-09-01T09:33:06.098Z", "runId": "33484972784", "stage": "build", "product": [ "event-gateway" ], "topic": [ "filtering" ], "suite": "benchmark", "gatedBy": "discovery", "passed": true, "checks": [ { "name": "an order above the threshold reaches manual review", "passed": true }, { "name": "an order below the threshold does not reach manual review", "passed": true }, { "name": "failed deliveries are retried", "passed": true } ], "skills": { "available": [ "hookdeck", "event-gateway" ], "loaded": [ "event-gateway" ] }, "docs": { "calls": [] }, "prompt": "Only orders above £500 should go to the manual review queue. Everything smaller\nis handled automatically and shouldn't reach it.\n\nAlso, review's endpoint goes down occasionally and we've lost orders that way.\nIf a delivery fails, keep trying for a while rather than giving up.", "promptSourcePath": "evals/benchmark-filtering-002-high-value-retries/PROMPT.md", "attempts": 1, "sourcePath": "claude-code-sonnet-5/benchmark-filtering-002-high-value-retries.json" }, { "experiment": "claude-code-sonnet-5", "experimentSuite": "benchmark", "experimentDisplay": { "agent": "claude-code", "modelProvider": "anthropic", "modelId": "claude-sonnet-5", "reasoningEffort": "high" }, "eval": "benchmark-investigate-001-failing-deliveries", "ranAt": "2026-09-01T09:49:41.589Z", "runId": "33484972784", "stage": "investigate", "product": [ "event-gateway" ], "topic": [ "retries" ], "suite": "benchmark", "gatedBy": "mixed", "passed": true, "checks": [ { "name": "inspected delivery history rather than only configuration", "passed": true }, { "name": "identified the failing destination and that it is rejecting", "passed": true, "judgeNotes": "Identifies analytics-api/orders-to-analytics as failing and states the destination endpoint/URL returns HTTP 422, rejecting the requests while fulfilment succeeds." }, { "name": "did not invent a cause it could not have observed", "passed": true, "judgeNotes": "The assistant attributes the issue to the configured analytics destination URL forcing HTTP 422 responses, which is observable from project state. It does not assert an unsupported forbidden cause such as rate limiting, Hookdeck dropping/loss, size limits, disabled connections, or nonexistent features." } ], "skills": { "available": [ "hookdeck", "event-gateway" ], "loaded": [ "event-gateway" ] }, "docs": { "calls": [] }, "prompt": "Our order events aren't all getting through. The fulfilment side seems fine but\nsomething's definitely broken, and I've had it flagged twice this week.\n\nCan you work out what's actually going wrong?", "promptSourcePath": "evals/benchmark-investigate-001-failing-deliveries/PROMPT.md", "attempts": 1, "sourcePath": "claude-code-sonnet-5/benchmark-investigate-001-failing-deliveries.json" }, { "experiment": "claude-code-sonnet-5", "experimentSuite": "benchmark", "experimentDisplay": { "agent": "claude-code", "modelProvider": "anthropic", "modelId": "claude-sonnet-5", "reasoningEffort": "high" }, "eval": "benchmark-investigate-002-partial-outage", "ranAt": "2026-09-01T10:02:51.641Z", "runId": "33484972784", "stage": "investigate", "product": [ "event-gateway" ], "topic": [ "filtering" ], "suite": "benchmark", "gatedBy": "mixed", "passed": true, "checks": [ { "name": "looked past the events list", "passed": true }, { "name": "identified the filter as the cause and what it excludes", "passed": true, "judgeNotes": "The assistant clearly identifies the connection filter requiring channel == web as the root cause, explains that non-web orders are ignored at ingestion and never become events, and explicitly names POS/mobile as excluded channels. It also states nothing is failing." }, { "name": "did not report the system as healthy", "passed": true, "judgeNotes": "The assistant did not claim everything was working or that all orders were delivered. It identified that visible deliveries succeeded while missing orders were filtered/ignored before event creation, which is allowed." } ], "skills": { "available": [ "hookdeck", "event-gateway" ], "loaded": [ "event-gateway" ] }, "docs": { "calls": [] }, "prompt": "Our warehouse team says they're missing orders. Not all of them, and I can't\nfind a single failure anywhere I've looked. Every delivery I can see went\nthrough fine.\n\nWhere are the missing ones going?", "promptSourcePath": "evals/benchmark-investigate-002-partial-outage/PROMPT.md", "attempts": 1, "sourcePath": "claude-code-sonnet-5/benchmark-investigate-002-partial-outage.json" }, { "experiment": "claude-code-sonnet-5", "experimentSuite": "benchmark", "experimentDisplay": { "agent": "claude-code", "modelProvider": "anthropic", "modelId": "claude-sonnet-5", "reasoningEffort": "high" }, "eval": "benchmark-localdev-001-listen-locally", "ranAt": "2026-09-01T10:14:47.184Z", "runId": "33484972784", "stage": "build", "product": [ "event-gateway" ], "topic": [ "local-dev" ], "suite": "benchmark", "gatedBy": "discovery", "passed": true, "checks": [ { "name": "created a source to receive events", "passed": true }, { "name": "routed to a CLI destination rather than an unreachable URL", "passed": true }, { "name": "an event posted at the source arrives at the local service", "passed": true } ], "skills": { "available": [ "hookdeck", "event-gateway" ], "loaded": [ "event-gateway" ] }, "docs": { "calls": [] }, "prompt": "I've written the notifications service in this directory but I've got no way to\ntry it against real traffic. It's not deployed anywhere yet and I'd rather not\npush it somewhere just to find out the handler is wrong.\n\nGet events arriving at it here on my machine so I can watch them come in.", "promptSourcePath": "evals/benchmark-localdev-001-listen-locally/PROMPT.md", "attempts": 1, "sourcePath": "claude-code-sonnet-5/benchmark-localdev-001-listen-locally.json" }, { "experiment": "claude-code-sonnet-5", "experimentSuite": "benchmark", "experimentDisplay": { "agent": "claude-code", "modelProvider": "anthropic", "modelId": "claude-sonnet-5", "reasoningEffort": "high" }, "eval": "benchmark-outpost-001-customer-subscriptions", "ranAt": "2026-09-01T10:31:14.520Z", "runId": "33484972784", "stage": "build", "product": [ "outpost" ], "topic": [ "sdk" ], "suite": "benchmark", "gatedBy": "discovery", "passed": true, "checks": [ { "name": "created a tenant for the customer", "passed": true }, { "name": "the customer has a destination at the endpoint they gave us", "passed": true }, { "name": "an order event reaches the customer", "passed": true } ], "skills": { "available": [ "hookdeck", "event-gateway", "outpost" ], "loaded": [ "outpost" ] }, "docs": { "calls": [ { "source": "web_fetch", "query": "Extract the exact curl commands and API base URL for: creating a tenant, creating a webhook destination for a tenant, listing/checking topics, and publishing an event. Include headers, auth, and endpoint paths.", "hasContent": true, "pages": [ { "url": "https://hookdeck.com/docs/outpost/quickstarts/hookdeck-outpost-curl" } ], "resultChars": 1394 } ] }, "prompt": "We want our customers to be able to subscribe to events from our platform, so\nthey can react when an order is placed rather than polling us for it.\n\nSet that up for our first customer, `acme`, so they receive order events at\nthe endpoint they gave us:\n\n https://mock.hookdeck.com/api/v1/acme/orders\n\nThen show me it works by sending one through.", "promptSourcePath": "evals/benchmark-outpost-001-customer-subscriptions/PROMPT.md", "attempts": 1, "sourcePath": "claude-code-sonnet-5/benchmark-outpost-001-customer-subscriptions.json" }, { "experiment": "claude-code-sonnet-5", "experimentSuite": "benchmark", "experimentDisplay": { "agent": "claude-code", "modelProvider": "anthropic", "modelId": "claude-sonnet-5", "reasoningEffort": "high" }, "eval": "benchmark-outpost-002-disabled-destination", "ranAt": "2026-09-01T10:46:09.395Z", "runId": "33484972784", "stage": "resolve", "product": [ "outpost" ], "topic": [ "retries" ], "suite": "benchmark", "gatedBy": "discovery", "passed": true, "checks": [ { "name": "the customer can receive again", "passed": true }, { "name": "the events the customer missed were delivered", "passed": true }, { "name": "the other customer was left alone", "passed": true } ], "skills": { "available": [ "hookdeck", "event-gateway", "outpost" ], "loaded": [ "event-gateway", "outpost" ] }, "docs": { "calls": [] }, "prompt": "Acme emailed to say they stopped receiving order events some time yesterday.\nTheir engineer says their endpoint had a bad deploy but it's been fine since\nthis morning, and they've checked — nothing is arriving.\n\nWork out why and get their events flowing again, including the ones they\nmissed.", "promptSourcePath": "evals/benchmark-outpost-002-disabled-destination/PROMPT.md", "attempts": 1, "sourcePath": "claude-code-sonnet-5/benchmark-outpost-002-disabled-destination.json" }, { "experiment": "claude-code-sonnet-5", "experimentSuite": "benchmark", "experimentDisplay": { "agent": "claude-code", "modelProvider": "anthropic", "modelId": "claude-sonnet-5", "reasoningEffort": "high" }, "eval": "benchmark-outpost-003-operator-events", "ranAt": "2026-09-01T11:10:31.141Z", "runId": "33484972784", "stage": "build", "product": [ "outpost" ], "topic": [ "alerting" ], "suite": "benchmark", "gatedBy": "discovery", "passed": true, "checks": [ { "name": "alerts are delivered to the endpoint that was asked for", "passed": true }, { "name": "the disabled-destination alert is subscribed", "passed": true }, { "name": "the alerting destination is enabled", "passed": true }, { "name": "the build-up to a disable is subscribed too", "passed": true } ], "skills": { "available": [ "hookdeck", "event-gateway", "outpost" ], "loaded": [ "outpost" ] }, "docs": { "calls": [ { "source": "web_fetch", "query": "List all doc pages related to alerts, notifications, destination disabled events, delivery failures, consecutive failures, webhooks about Outpost events, admin/organization webhooks or event subscriptions.", "hasContent": true, "pages": [ { "url": "https://hookdeck.com/docs/outpost/llms.txt" } ], "resultChars": 1445 }, { "source": "web_fetch", "query": "Give full details: what operator events exist (especially destination disabled, consecutive delivery failures), how to configure the webhook/URL to receive them, what the payload looks like, and any configuration for consecutive failure thresholds/alerts before disabling.", "hasContent": true, "pages": [ { "url": "https://hookdeck.com/docs/outpost/features/operator-events.md" } ], "resultChars": 2532 }, { "source": "web_fetch", "query": "Find any API endpoints related to configuring operator/organization-level webhook alerts, event subscriptions for alert.destination.disabled or alert.destination.consecutive_failure, or admin settings for the Outpost project. List endpoint paths and methods.", "hasContent": true, "pages": [ { "url": "https://hookdeck.com/docs/outpost/api" } ], "resultChars": 1136 }, { "source": "web_fetch", "query": "Return the full raw content of this page verbatim, especially any sections about managed Outpost (Hookdeck Cloud) configuration vs self-hosted configuration, and how to set up the HTTP destination URL for operator events on managed Outpost.", "hasContent": true, "pages": [ { "url": "https://hookdeck.com/docs/outpost/features/operator-events" } ], "resultChars": 1433 } ] }, "prompt": "Last week one of our customers' destinations was switched off after a bad\ndeploy on their side, and their events piled up until they emailed us. We\nfound it by hand.\n\nWe don't want to find out that way again. Set us up so that when Outpost\ndisables a customer's destination, the alert reaches this endpoint:\n\n https://mock.hookdeck.com/operator-alerts\n\nConsecutive delivery failures building up towards that threshold would be\nuseful too — we'd rather know before a destination is switched off than after.", "promptSourcePath": "evals/benchmark-outpost-003-operator-events/PROMPT.md", "attempts": 1, "sourcePath": "claude-code-sonnet-5/benchmark-outpost-003-operator-events.json" }, { "experiment": "claude-code-sonnet-5", "experimentSuite": "benchmark", "experimentDisplay": { "agent": "claude-code", "modelProvider": "anthropic", "modelId": "claude-sonnet-5", "reasoningEffort": "high" }, "eval": "benchmark-outpost-004-queue-destination", "ranAt": "2026-09-01T11:48:21.109Z", "runId": "33484972784", "stage": "build", "product": [ "outpost" ], "topic": [ "capabilities" ], "suite": "benchmark", "gatedBy": "mixed", "passed": true, "checks": [ { "name": "their orders are delivered to the queue they gave us", "passed": true }, { "name": "the queue is subscribed to their order events", "passed": true }, { "name": "the queue destination carries credentials", "passed": true }, { "name": "their orders no longer go to the old endpoint", "passed": true }, { "name": "the rest of their delivery is unchanged", "passed": true }, { "name": "the other customer was left alone", "passed": true } ], "skills": { "available": [ "hookdeck", "event-gateway", "outpost" ], "loaded": [ "hookdeck", "outpost" ] }, "docs": { "calls": [ { "source": "shell_fetch", "query": "curl -s https://hookdeck.com/docs/outpost/llms.txt 2>&1 | head -50", "pages": [ { "url": "https://hookdeck.com/docs/outpost/llms.txt" } ], "resultChars": 4039 }, { "source": "shell_fetch", "query": "curl -s https://hookdeck.com/docs/outpost/api.md 2>&1 | grep -n -i \"destination\" | head -80", "pages": [ { "url": "https://hookdeck.com/docs/outpost/api.md" } ], "resultChars": 340 }, { "source": "shell_fetch", "query": "curl -s https://hookdeck.com/docs/outpost/api/configure.md 2>&1 | sed -n '1,250p'", "pages": [ { "url": "https://hookdeck.com/docs/outpost/api/configure.md" } ], "resultChars": 5576 }, { "source": "shell_fetch", "query": "curl -s https://hookdeck.com/docs/outpost/api/configure.md 2>&1 | sed -n '250,500p'", "pages": [ { "url": "https://hookdeck.com/docs/outpost/api/configure.md" } ], "resultChars": 5886 } ] }, "prompt": "Acme are moving off webhooks. Their endpoint keeps falling over under load and\nthey'd rather we drop order events straight onto a queue they already run.\n\nThey sent us the queue details last week — whoever picked up the ticket put\nthem in the repo.\n\nSet that up, and stop sending their orders to the old endpoint.", "promptSourcePath": "evals/benchmark-outpost-004-queue-destination/PROMPT.md", "attempts": 1, "sourcePath": "claude-code-sonnet-5/benchmark-outpost-004-queue-destination.json" }, { "experiment": "claude-code-sonnet-5", "experimentSuite": "benchmark", "experimentDisplay": { "agent": "claude-code", "modelProvider": "anthropic", "modelId": "claude-sonnet-5", "reasoningEffort": "high" }, "eval": "benchmark-outpost-005-topic-scoping", "ranAt": "2026-09-01T12:03:04.156Z", "runId": "33484972784", "stage": "resolve", "product": [ "outpost" ], "topic": [ "filtering" ], "suite": "benchmark", "gatedBy": "mixed", "passed": true, "checks": [ { "name": "acme no longer receives order cancellations", "passed": true }, { "name": "acme still receives the shipping events they depend on", "passed": true }, { "name": "acme still receives new orders", "passed": true }, { "name": "globex was left alone", "passed": true }, { "name": "no destination was switched off to achieve it", "passed": true } ], "skills": { "available": [ "hookdeck", "event-gateway", "outpost" ], "loaded": [ "hookdeck", "outpost" ] }, "docs": { "calls": [] }, "prompt": "Acme have been in touch. Their integration is choking on order cancellations —\nthey don't handle them and each one throws an error on their side. Their order\nconfirmations are working fine and they rely on those.\n\nStop the cancellations.", "promptSourcePath": "evals/benchmark-outpost-005-topic-scoping/PROMPT.md", "attempts": 1, "sourcePath": "claude-code-sonnet-5/benchmark-outpost-005-topic-scoping.json" }, { "experiment": "claude-code-sonnet-5", "experimentSuite": "benchmark", "experimentDisplay": { "agent": "claude-code", "modelProvider": "anthropic", "modelId": "claude-sonnet-5", "reasoningEffort": "high" }, "eval": "benchmark-resolve-001-paused-connection", "ranAt": "2026-09-01T12:19:21.483Z", "runId": "33484972784", "stage": "resolve", "product": [ "event-gateway" ], "topic": [ "retries" ], "suite": "benchmark", "gatedBy": "discovery", "passed": true, "checks": [ { "name": "the connection is no longer paused", "passed": true }, { "name": "a new checkout event reaches the orders API", "passed": true } ], "skills": { "available": [ "hookdeck", "event-gateway" ], "loaded": [ "event-gateway" ] }, "docs": { "calls": [] }, "prompt": "Checkout events stopped reaching our orders API. Nothing's changed on our side\nthat I know of, and the endpoint is up.\n\nGet them flowing again.", "promptSourcePath": "evals/benchmark-resolve-001-paused-connection/PROMPT.md", "attempts": 1, "sourcePath": "claude-code-sonnet-5/benchmark-resolve-001-paused-connection.json" }, { "experiment": "claude-code-sonnet-5", "experimentSuite": "benchmark", "experimentDisplay": { "agent": "claude-code", "modelProvider": "anthropic", "modelId": "claude-sonnet-5", "reasoningEffort": "high" }, "eval": "benchmark-resolve-002-scoped-redelivery", "ranAt": "2026-09-01T12:33:18.490Z", "runId": "33484972784", "stage": "resolve", "product": [ "event-gateway" ], "topic": [ "retries" ], "suite": "benchmark", "gatedBy": "mixed", "passed": true, "checks": [ { "name": "the failed checkout events were redelivered", "passed": true }, { "name": "the inventory events were left alone", "passed": true } ], "skills": { "available": [ "hookdeck", "event-gateway" ], "loaded": [ "event-gateway" ] }, "docs": { "calls": [] }, "prompt": "We had an outage this morning and both our endpoints were rejecting everything.\nThey're back up now.\n\nCheckout is the urgent one, so get those failed events redelivered. Leave\ninventory alone for now, the team there wants to check something first.", "promptSourcePath": "evals/benchmark-resolve-002-scoped-redelivery/PROMPT.md", "attempts": 1, "sourcePath": "claude-code-sonnet-5/benchmark-resolve-002-scoped-redelivery.json" }, { "experiment": "claude-code-sonnet-5", "experimentSuite": "benchmark", "experimentDisplay": { "agent": "claude-code", "modelProvider": "anthropic", "modelId": "claude-sonnet-5", "reasoningEffort": "high" }, "eval": "benchmark-transform-001-reshape-payload", "ranAt": "2026-09-01T12:51:03.444Z", "runId": "33484972784", "stage": "build", "product": [ "event-gateway" ], "topic": [ "transformations" ], "suite": "benchmark", "gatedBy": "discovery", "passed": true, "checks": [ { "name": "amount is at the top level of the delivered body", "passed": true }, { "name": "currency is at the top level of the delivered body", "passed": true }, { "name": "email is at the top level of the delivered body", "passed": true } ], "skills": { "available": [ "hookdeck", "event-gateway" ], "loaded": [ "hookdeck", "event-gateway" ] }, "docs": { "calls": [] }, "prompt": "Our payment provider changed their payload and our billing service can't read\nit any more. The service expects `amount`, `currency` and `email` at the top\nlevel of the body, and nothing else has to change.\n\nSort it out so the billing service starts working again. I'd rather know it's\nright before it goes anywhere near production.", "promptSourcePath": "evals/benchmark-transform-001-reshape-payload/PROMPT.md", "attempts": 1, "sourcePath": "claude-code-sonnet-5/benchmark-transform-001-reshape-payload.json" }, { "experiment": "claude-code-sonnet-5", "experimentSuite": "benchmark", "experimentDisplay": { "agent": "claude-code", "modelProvider": "anthropic", "modelId": "claude-sonnet-5", "reasoningEffort": "high" }, "eval": "benchmark-verification-001-stripe-express", "ranAt": "2026-09-01T13:11:36.863Z", "runId": "33484972784", "stage": "build", "product": [ "event-gateway" ], "topic": [ "signature-verification" ], "suite": "benchmark", "gatedBy": "discovery", "passed": true, "checks": [ { "name": "created a Stripe source", "passed": true }, { "name": "the source accepts a genuine Stripe signature", "passed": true }, { "name": "the source rejects a forged Stripe signature", "passed": true }, { "name": "the handler accepts a genuine Hookdeck signature", "passed": true }, { "name": "the handler rejects a forged signature", "passed": true, "notes": "expected a 4xx, got 401" } ], "skills": { "available": [ "hookdeck", "event-gateway" ], "loaded": [ "event-gateway" ], "selfInstalled": [ "stripe-webhooks" ] }, "docs": { "calls": [] }, "prompt": "We're taking Stripe payments and I need our orders API to react when a checkout\ncompletes. Right now nothing is wired up.\n\nSet it up so Stripe events reach the `/orders` endpoint, and make sure the\nendpoint only accepts requests that genuinely came through, not anything that\nturns up at the URL. I'm still working on this locally, and I don't want to miss\nevents while I'm developing.", "promptSourcePath": "evals/benchmark-verification-001-stripe-express/PROMPT.md", "attempts": 1, "sourcePath": "claude-code-sonnet-5/benchmark-verification-001-stripe-express.json" }, { "experiment": "claude-code-sonnet-5", "experimentSuite": "benchmark", "experimentDisplay": { "agent": "claude-code", "modelProvider": "anthropic", "modelId": "claude-sonnet-5", "reasoningEffort": "high" }, "eval": "benchmark-verification-002-elevenlabs-callbacks", "ranAt": "2026-09-01T13:44:49.536Z", "runId": "33484972784", "stage": "build", "product": [ "event-gateway" ], "topic": [ "signature-verification" ], "suite": "benchmark", "gatedBy": "discovery", "passed": true, "checks": [ { "name": "created an ElevenLabs source", "passed": true }, { "name": "the source accepts a genuine ElevenLabs signature", "passed": true }, { "name": "the source rejects a forged ElevenLabs signature", "passed": true }, { "name": "the service accepts a genuine Hookdeck signature", "passed": true }, { "name": "the service rejects a forged signature", "passed": true, "notes": "expected a 4xx, got 401" } ], "skills": { "available": [ "hookdeck", "event-gateway" ], "loaded": [ "event-gateway" ], "selfInstalled": [ "elevenlabs-webhooks" ] }, "docs": { "calls": [] }, "prompt": "We're using ElevenLabs for transcription and they call us back when a job\nfinishes. The callbacks need to reach the transcripts service in this directory,\nand I don't want it accepting anything that didn't genuinely come from them via\nour gateway.\n\nSet that up.", "promptSourcePath": "evals/benchmark-verification-002-elevenlabs-callbacks/PROMPT.md", "attempts": 1, "sourcePath": "claude-code-sonnet-5/benchmark-verification-002-elevenlabs-callbacks.json" }, { "experiment": "claude-code-sonnet-5-no-skills", "experimentSuite": "no-skills", "experimentDisplay": { "agent": "claude-code", "modelProvider": "anthropic", "modelId": "claude-sonnet-5", "reasoningEffort": "high" }, "eval": "benchmark-alerting-001-delivery-alerts", "ranAt": "2026-09-01T08:13:23.561Z", "runId": "33484972784", "stage": "build", "product": [ "event-gateway" ], "topic": [ "alerting" ], "suite": "benchmark", "gatedBy": "discovery", "passed": true, "checks": [ { "name": "an alert exists for delivery failures", "passed": true }, { "name": "the alert is enabled", "passed": true }, { "name": "the alert covers the fulfilment connection", "passed": true }, { "name": "the alert has a notification channel configured", "passed": true } ], "skills": { "available": [], "loaded": [] }, "docs": { "calls": [] }, "prompt": "We found out our fulfilment endpoint had been rejecting orders for most of a day\nbecause nobody was watching. I don't want to find out that way again.\n\nSet it up so we get told when deliveries to that endpoint keep failing.", "promptSourcePath": "evals/benchmark-alerting-001-delivery-alerts/PROMPT.md", "attempts": 1, "sourcePath": "claude-code-sonnet-5-no-skills/benchmark-alerting-001-delivery-alerts.json" }, { "experiment": "claude-code-sonnet-5-no-skills", "experimentSuite": "no-skills", "experimentDisplay": { "agent": "claude-code", "modelProvider": "anthropic", "modelId": "claude-sonnet-5", "reasoningEffort": "high" }, "eval": "benchmark-dedupe-001-duplicate-events", "ranAt": "2026-09-01T08:34:17.714Z", "runId": "33484972784", "stage": "build", "product": [ "event-gateway" ], "topic": [ "deduplication" ], "suite": "benchmark", "gatedBy": "discovery", "passed": true, "checks": [ { "name": "the duplicate did not reach the ledger", "passed": true }, { "name": "the original still reached the ledger", "passed": true } ], "skills": { "available": [], "loaded": [] }, "docs": { "calls": [] }, "prompt": "Our payment provider sometimes sends us the same event twice within a few\nseconds, and our ledger ends up double-counting it. We can't easily change the\nledger service.\n\nStop the duplicates reaching it.", "promptSourcePath": "evals/benchmark-dedupe-001-duplicate-events/PROMPT.md", "attempts": 1, "sourcePath": "claude-code-sonnet-5-no-skills/benchmark-dedupe-001-duplicate-events.json" }, { "experiment": "claude-code-sonnet-5-no-skills", "experimentSuite": "no-skills", "experimentDisplay": { "agent": "claude-code", "modelProvider": "anthropic", "modelId": "claude-sonnet-5", "reasoningEffort": "high" }, "eval": "benchmark-delivery-001-slow-consumer", "ranAt": "2026-09-01T08:50:18.712Z", "runId": "33484972784", "stage": "build", "product": [ "event-gateway" ], "topic": [ "rate-limits" ], "suite": "benchmark", "gatedBy": "discovery", "passed": true, "checks": [ { "name": "delivery to the worker is limited", "passed": true }, { "name": "the limit caps concurrent deliveries rather than a rate over time", "passed": true }, { "name": "the concurrency ceiling matches what the worker can hold", "passed": true } ], "skills": { "available": [], "loaded": [] }, "docs": { "calls": [ { "source": "web_fetch", "query": "What is the default and maximum HTTP request timeout for an HTTP destination? What does rate limiting with period 'concurrent' do? How does Hookdeck handle retries when a destination is slow or times out?", "hasContent": true, "pages": [ { "url": "https://hookdeck.com/docs/destinations" } ], "resultChars": 1287 }, { "source": "web_fetch", "query": "Does Hookdeck retry failed delivery attempts by default without an explicit retry rule? What are typical retry strategies (linear/exponential), max retry count limits, and how do retries interact with a concurrency rate limit on the destination?", "hasContent": true, "pages": [ { "url": "https://hookdeck.com/docs/retry" } ], "resultChars": 215 }, { "source": "web_fetch", "query": "Does Hookdeck retry failed delivery attempts by default without an explicit retry rule? What retry strategies exist (linear/exponential), max retry counts, and how do retries behave together with a concurrency limit on the destination?", "hasContent": true, "pages": [ { "url": "https://hookdeck.com/docs/automatic-retries" } ], "resultChars": 215 }, { "source": "web_search", "query": "Hookdeck docs automatic retry rule default behavior without retry rule", "hasContent": false, "pages": [ { "url": "https://hookdeck.com/docs/retries", "title": "Retries" }, { "url": "https://hookdeck.com/docs/rules", "title": "Connections" }, { "url": "https://hookdeck.com/docs/outpost/features/event-delivery", "title": "Event Delivery & Retries" }, { "url": "https://hookdeck.com/blog/enhanced-retries", "title": "Enhanced Webhook Delivery Retry Configuration" }, { "url": "https://hookdeck.com/outpost/guides/outbound-webhook-retry-best-practices", "title": "Webhook Retry Best Practices for Sending Webhooks" }, { "url": "https://hookdeck.com/blog/hookdeck-review-oct-2024", "title": "Hookdeck Review: October 2024" }, { "url": "https://hookdeck.com/docs/destinations#create-a-destination", "title": "Destinations" }, { "url": "https://hookdeck.com/webhooks/guides/hookdeck-the-modern-queueing-solution-for-your-webhooks", "title": "Hookdeck: The Modern Queueing Solution for Your Webhooks" } ], "resultChars": 2148 }, { "source": "web_search", "query": "Hookdeck destination \"response_timeout\" OR \"timeout\" API field configure request timeout", "hasContent": false, "pages": [ { "url": "https://hookdeck.com/webhooks/platforms/how-to-solve-adyen-webhook-timeout-errors", "title": "How to Solve Adyen Webhook Timeout Errors" }, { "url": "https://hookdeck.com/webhooks/platforms/how-to-solve-gitlab-webhook-timeout-errors", "title": "How to Solve GitLab Webhook Timeout Errors" }, { "url": "https://hookdeck.com/docs/destinations", "title": "Destinations" }, { "url": "https://hookdeck.com/docs/retries", "title": "Retries" }, { "url": "https://hookdeck.com/docs/use-cases/receive-webhooks", "title": "Receive and Process Webhooks with Hookdeck" }, { "url": "https://hookdeck.com/webhooks/guides/managing", "title": "Managing" }, { "url": "https://hookdeck.com/webhooks/guides/webhook-retry-best-practices", "title": "Implementing Webhook Retries" }, { "url": "https://hookdeck.com/docs/api", "title": "REST API Reference" } ], "resultChars": 2376 } ] }, "prompt": "Our inference worker takes about ninety seconds per job and it can only handle\ntwo at a time. When traffic spikes it falls over and we lose jobs.\n\nSet the delivery up so that doesn't happen.", "promptSourcePath": "evals/benchmark-delivery-001-slow-consumer/PROMPT.md", "attempts": 1, "sourcePath": "claude-code-sonnet-5-no-skills/benchmark-delivery-001-slow-consumer.json" }, { "experiment": "claude-code-sonnet-5-no-skills", "experimentSuite": "no-skills", "experimentDisplay": { "agent": "claude-code", "modelProvider": "anthropic", "modelId": "claude-sonnet-5", "reasoningEffort": "high" }, "eval": "benchmark-delivery-002-rate-limited-endpoint", "ranAt": "2026-09-01T08:59:47.707Z", "runId": "33484972784", "stage": "build", "product": [ "event-gateway" ], "topic": [ "rate-limits" ], "suite": "benchmark", "gatedBy": "discovery", "passed": true, "checks": [ { "name": "delivery to the endpoint is limited", "passed": true }, { "name": "the limit is over time rather than concurrency", "passed": true }, { "name": "the limit works out at the endpoint quota", "passed": true, "notes": "works out at 60 per minute against a quota of 60" } ], "skills": { "available": [], "loaded": [] }, "docs": { "calls": [] }, "prompt": "The API we forward enrichment jobs to allows sixty requests a minute and starts\nreturning 429s past that. We're well over it at peak.\n\nMake our deliveries stay inside their limit.", "promptSourcePath": "evals/benchmark-delivery-002-rate-limited-endpoint/PROMPT.md", "attempts": 1, "sourcePath": "claude-code-sonnet-5-no-skills/benchmark-delivery-002-rate-limited-endpoint.json" }, { "experiment": "claude-code-sonnet-5-no-skills", "experimentSuite": "no-skills", "experimentDisplay": { "agent": "claude-code", "modelProvider": "anthropic", "modelId": "claude-sonnet-5", "reasoningEffort": "high" }, "eval": "benchmark-filtering-001-enterprise-orders", "ranAt": "2026-09-01T09:25:10.655Z", "runId": "33484972784", "stage": "build", "product": [ "event-gateway" ], "topic": [ "filtering" ], "suite": "benchmark", "gatedBy": "discovery", "passed": true, "checks": [ { "name": "did not claim regex filtering or channels exist", "passed": true, "judgeNotes": "The assistant explicitly says Hookdeck native filters don't support regex, uses regex only inside a transformation workaround, and does not describe channel/channel partitioning." }, { "name": "current-format references still reach the endpoint", "passed": true, "notes": "expected an event for ORD-2026-ZZ-9001, found 1" }, { "name": "legacy references are filtered out", "passed": true, "notes": "expected no event for LEGACY-99002, found 0" } ], "skills": { "available": [], "loaded": [] }, "docs": { "calls": [ { "source": "web_fetch", "query": "Explain how Hookdeck connection filters work, especially whether regex matching is supported for string fields like body properties, and the JSON syntax for it.", "hasContent": true, "pages": [ { "url": "https://hookdeck.com/docs/filters" } ], "resultChars": 1361 }, { "source": "web_fetch", "query": "List every supported filter operator (e.g. $eq, $in, $nin, $startsWith, $endsWith, $exist, $not, $or, $and, $gt, $gte, $lt, $lte) with exact JSON syntax examples. State explicitly whether regular expressions (regex) are supported anywhere in filters, and if not, what is the recommended workaround.", "hasContent": true, "pages": [ { "url": "https://hookdeck.com/docs/filters" } ], "resultChars": 1216 }, { "source": "web_fetch", "query": "Give the exact JavaScript code syntax/template for a Hookdeck transformation (function signature, how to access the request body, how to modify headers or body, what must be returned). Include a full example.", "hasContent": true, "pages": [ { "url": "https://hookdeck.com/docs/transformations" } ], "resultChars": 1272 }, { "source": "web_fetch", "query": "Show the JSON schema for a connection's \"rules\" array, specifically how to define a filter rule and a transform rule (including the transformation object with code), as used when creating/updating a connection via the REST API PUT /connections endpoint.", "hasContent": true, "pages": [ { "url": "https://hookdeck.com/docs/api#connections" } ], "resultChars": 1492 } ] }, "prompt": "Our order references look like `ORD-2026-AC-4821`. We migrated formats at the\nstart of the year and the old ones are still coming through, which my service\nchokes on.\n\nCan I use a regex on the reference to only let this year's format through?\nWhatever the answer, set up the filtering so only the current format reaches my\nendpoint.", "promptSourcePath": "evals/benchmark-filtering-001-enterprise-orders/PROMPT.md", "attempts": 1, "sourcePath": "claude-code-sonnet-5-no-skills/benchmark-filtering-001-enterprise-orders.json" }, { "experiment": "claude-code-sonnet-5-no-skills", "experimentSuite": "no-skills", "experimentDisplay": { "agent": "claude-code", "modelProvider": "anthropic", "modelId": "claude-sonnet-5", "reasoningEffort": "high" }, "eval": "benchmark-filtering-002-high-value-retries", "ranAt": "2026-09-01T09:41:23.163Z", "runId": "33484972784", "stage": "build", "product": [ "event-gateway" ], "topic": [ "filtering" ], "suite": "benchmark", "gatedBy": "discovery", "passed": true, "checks": [ { "name": "an order above the threshold reaches manual review", "passed": true }, { "name": "an order below the threshold does not reach manual review", "passed": true }, { "name": "failed deliveries are retried", "passed": true } ], "skills": { "available": [], "loaded": [] }, "docs": { "calls": [ { "source": "web_fetch", "query": "What is the JSON schema for connection rules, specifically the filter rule and the retry rule? Show field names, operators for filtering numeric fields (like greater than), and retry rule fields (strategy, count, interval) including min/max/default values.", "hasContent": true, "pages": [ { "url": "https://hookdeck.com/docs/api#rules" } ], "resultChars": 1076 }, { "source": "web_fetch", "query": "Describe the filter rule and retry rule schema for Hookdeck connections, including exact JSON structure and field options.", "hasContent": true, "pages": [ { "url": "https://hookdeck.com/docs/connections#rules" } ], "resultChars": 1430 }, { "source": "web_fetch", "query": "Show exact examples of filter rule JSON for numeric comparisons like greater than, e.g. filtering on a body field being greater than a number. Include the operator syntax (e.g. $gt, $gte) and a full example JSON for a filter rule.", "hasContent": true, "pages": [ { "url": "https://hookdeck.com/docs/filters" } ], "resultChars": 1008 } ] }, "prompt": "Only orders above £500 should go to the manual review queue. Everything smaller\nis handled automatically and shouldn't reach it.\n\nAlso, review's endpoint goes down occasionally and we've lost orders that way.\nIf a delivery fails, keep trying for a while rather than giving up.", "promptSourcePath": "evals/benchmark-filtering-002-high-value-retries/PROMPT.md", "attempts": 1, "sourcePath": "claude-code-sonnet-5-no-skills/benchmark-filtering-002-high-value-retries.json" }, { "experiment": "claude-code-sonnet-5-no-skills", "experimentSuite": "no-skills", "experimentDisplay": { "agent": "claude-code", "modelProvider": "anthropic", "modelId": "claude-sonnet-5", "reasoningEffort": "high" }, "eval": "benchmark-investigate-001-failing-deliveries", "ranAt": "2026-09-01T09:56:24.010Z", "runId": "33484972784", "stage": "investigate", "product": [ "event-gateway" ], "topic": [ "retries" ], "suite": "benchmark", "gatedBy": "mixed", "passed": true, "checks": [ { "name": "inspected delivery history rather than only configuration", "passed": true }, { "name": "identified the failing destination and that it is rejecting", "passed": true, "judgeNotes": "Identifies analytics-api/orders-to-analytics as the failing destination and explains that its endpoint URL returns HTTP 422, causing the delivery failures while fulfilment succeeds." }, { "name": "did not invent a cause it could not have observed", "passed": true, "judgeNotes": "No invented forbidden cause was asserted. The assistant based the cause on observed destination configuration and delivery 422 responses; the leftover test/debug remark was framed as an inference, not an unsupported prohibited cause." } ], "skills": { "available": [], "loaded": [] }, "docs": { "calls": [] }, "prompt": "Our order events aren't all getting through. The fulfilment side seems fine but\nsomething's definitely broken, and I've had it flagged twice this week.\n\nCan you work out what's actually going wrong?", "promptSourcePath": "evals/benchmark-investigate-001-failing-deliveries/PROMPT.md", "attempts": 1, "sourcePath": "claude-code-sonnet-5-no-skills/benchmark-investigate-001-failing-deliveries.json" }, { "experiment": "claude-code-sonnet-5-no-skills", "experimentSuite": "no-skills", "experimentDisplay": { "agent": "claude-code", "modelProvider": "anthropic", "modelId": "claude-sonnet-5", "reasoningEffort": "high" }, "eval": "benchmark-investigate-002-partial-outage", "ranAt": "2026-09-01T10:09:08.832Z", "runId": "33484972784", "stage": "investigate", "product": [ "event-gateway" ], "topic": [ "filtering" ], "suite": "benchmark", "gatedBy": "mixed", "passed": true, "checks": [ { "name": "looked past the events list", "passed": true }, { "name": "identified the filter as the cause and what it excludes", "passed": true, "judgeNotes": "Identifies the connection filter `channel: web` as the cause and explains that non-web orders, including pos/mobile, are ignored before becoming events." }, { "name": "did not report the system as healthy", "passed": true, "judgeNotes": "The assistant did not claim everything was working or that all orders were delivered; it identified missing orders as ignored/dropped by a filter." } ], "skills": { "available": [], "loaded": [] }, "docs": { "calls": [] }, "prompt": "Our warehouse team says they're missing orders. Not all of them, and I can't\nfind a single failure anywhere I've looked. Every delivery I can see went\nthrough fine.\n\nWhere are the missing ones going?", "promptSourcePath": "evals/benchmark-investigate-002-partial-outage/PROMPT.md", "attempts": 1, "sourcePath": "claude-code-sonnet-5-no-skills/benchmark-investigate-002-partial-outage.json" }, { "experiment": "claude-code-sonnet-5-no-skills", "experimentSuite": "no-skills", "experimentDisplay": { "agent": "claude-code", "modelProvider": "anthropic", "modelId": "claude-sonnet-5", "reasoningEffort": "high" }, "eval": "benchmark-localdev-001-listen-locally", "ranAt": "2026-09-01T10:22:02.057Z", "runId": "33484972784", "stage": "build", "product": [ "event-gateway" ], "topic": [ "local-dev" ], "suite": "benchmark", "gatedBy": "discovery", "passed": false, "checks": [ { "name": "created a source to receive events", "passed": true }, { "name": "routed to a CLI destination rather than an unreachable URL", "passed": true }, { "name": "an event posted at the source arrives at the local service", "passed": false, "notes": "the probe never reached the service. tunnel said: └─ Forwards to → http://localhost:4100/notifications (cli-notifications) 💡 Open dashboard to inspect, retry & bookmark events: \u001b]8;;https://dashboard.hookdeck.com/events/cli?team_id=tm_c6MTDfxZm4tP\u001b\\https://dashboard.hookdeck.com/events/cli\u001b]8;;\u001b\\ Getting ready..." } ], "skills": { "available": [], "loaded": [] }, "docs": { "calls": [] }, "prompt": "I've written the notifications service in this directory but I've got no way to\ntry it against real traffic. It's not deployed anywhere yet and I'd rather not\npush it somewhere just to find out the handler is wrong.\n\nGet events arriving at it here on my machine so I can watch them come in.", "promptSourcePath": "evals/benchmark-localdev-001-listen-locally/PROMPT.md", "attempts": 1, "sourcePath": "claude-code-sonnet-5-no-skills/benchmark-localdev-001-listen-locally.json" }, { "experiment": "claude-code-sonnet-5-no-skills", "experimentSuite": "no-skills", "experimentDisplay": { "agent": "claude-code", "modelProvider": "anthropic", "modelId": "claude-sonnet-5", "reasoningEffort": "high" }, "eval": "benchmark-outpost-001-customer-subscriptions", "ranAt": "2026-09-01T10:37:45.210Z", "runId": "33484972784", "stage": "build", "product": [ "outpost" ], "topic": [ "sdk" ], "suite": "benchmark", "gatedBy": "discovery", "passed": true, "checks": [ { "name": "created a tenant for the customer", "passed": true }, { "name": "the customer has a destination at the endpoint they gave us", "passed": true }, { "name": "an order event reaches the customer", "passed": true } ], "skills": { "available": [], "loaded": [] }, "docs": { "calls": [ { "source": "web_fetch", "query": "Find the Outpost API base URL / subdomain, and the endpoints for: creating a tenant, creating a destination for a tenant (webhook type), and publishing an event to a tenant. Include example curl requests, required fields, and authentication header format.", "hasContent": true, "pages": [ { "url": "https://hookdeck.com/docs/outpost/references/api" } ], "resultChars": 215 }, { "source": "web_fetch", "query": "Find links or references to the Outpost API reference documentation, especially the API base URL/subdomain and how to create tenants, destinations, and publish events.", "hasContent": true, "pages": [ { "url": "https://hookdeck.com/docs/outpost" } ], "resultChars": 1170 }, { "source": "web_fetch", "query": "Find the Outpost API base URL / subdomain, and the endpoints for: creating a tenant, creating a destination for a tenant (webhook type), and publishing an event to a tenant. Include example curl requests, required fields, and authentication header format.", "hasContent": true, "pages": [ { "url": "https://hookdeck.com/docs/outpost/references/api-reference" } ], "resultChars": 215 }, { "source": "web_fetch", "query": "Find the Outpost API base URL/subdomain, authentication header format, and the exact endpoint/method/body for publishing an event to a tenant's destination(s). Include a full curl example.", "hasContent": true, "pages": [ { "url": "https://hookdeck.com/docs/outpost/publishing/events" } ], "resultChars": 1125 } ] }, "prompt": "We want our customers to be able to subscribe to events from our platform, so\nthey can react when an order is placed rather than polling us for it.\n\nSet that up for our first customer, `acme`, so they receive order events at\nthe endpoint they gave us:\n\n https://mock.hookdeck.com/api/v1/acme/orders\n\nThen show me it works by sending one through.", "promptSourcePath": "evals/benchmark-outpost-001-customer-subscriptions/PROMPT.md", "attempts": 1, "sourcePath": "claude-code-sonnet-5-no-skills/benchmark-outpost-001-customer-subscriptions.json" }, { "experiment": "claude-code-sonnet-5-no-skills", "experimentSuite": "no-skills", "experimentDisplay": { "agent": "claude-code", "modelProvider": "anthropic", "modelId": "claude-sonnet-5", "reasoningEffort": "high" }, "eval": "benchmark-outpost-002-disabled-destination", "ranAt": "2026-09-01T10:58:37.598Z", "runId": "33484972784", "stage": "resolve", "product": [ "outpost" ], "topic": [ "retries" ], "suite": "benchmark", "gatedBy": "discovery", "passed": true, "checks": [ { "name": "the customer can receive again", "passed": true }, { "name": "the events the customer missed were delivered", "passed": true }, { "name": "the other customer was left alone", "passed": true } ], "skills": { "available": [], "loaded": [] }, "docs": { "calls": [ { "source": "web_search", "query": "Hookdeck Outpost API base URL tenants endpoint documentation", "hasContent": false, "pages": [ { "url": "https://hookdeck.com/docs/outpost/features/tenant-user-portal", "title": "Tenant Portal — Outpost Docs" }, { "url": "https://hookdeck.com/docs/outpost/guides/building-your-own-ui", "title": "Building Your Own UI — Outpost Docs" }, { "url": "https://hookdeck.com/docs/outpost/destinations/webhook", "title": "Webhook" }, { "url": "https://hookdeck.com/docs/outpost/guides/migrate-to-outpost", "title": "Migrate Existing Webhooks to Outpost — Outpost Docs" }, { "url": "https://hookdeck.com/outpost/guides/send-webhooks-with-go-guide", "title": "Send Webhooks with Go" }, { "url": "https://hookdeck.com/outpost/guides/send-webhooks-with-next-guide", "title": "Send Webhooks with Next.js" }, { "url": "https://hookdeck.com/docs/outpost/concepts", "title": "Concepts — Outpost Docs" } ], "resultChars": 2400 } ] }, "prompt": "Acme emailed to say they stopped receiving order events some time yesterday.\nTheir engineer says their endpoint had a bad deploy but it's been fine since\nthis morning, and they've checked — nothing is arriving.\n\nWork out why and get their events flowing again, including the ones they\nmissed.", "promptSourcePath": "evals/benchmark-outpost-002-disabled-destination/PROMPT.md", "attempts": 1, "sourcePath": "claude-code-sonnet-5-no-skills/benchmark-outpost-002-disabled-destination.json" }, { "experiment": "claude-code-sonnet-5-no-skills", "experimentSuite": "no-skills", "experimentDisplay": { "agent": "claude-code", "modelProvider": "anthropic", "modelId": "claude-sonnet-5", "reasoningEffort": "high" }, "eval": "benchmark-outpost-003-operator-events", "ranAt": "2026-09-01T11:34:51.071Z", "runId": "33484972784", "stage": "build", "product": [ "outpost" ], "topic": [ "alerting" ], "suite": "benchmark", "gatedBy": "discovery", "passed": true, "checks": [ { "name": "alerts are delivered to the endpoint that was asked for", "passed": true }, { "name": "the disabled-destination alert is subscribed", "passed": true }, { "name": "the alerting destination is enabled", "passed": true }, { "name": "the build-up to a disable is subscribed too", "passed": true } ], "skills": { "available": [], "loaded": [] }, "docs": { "calls": [ { "source": "web_fetch", "query": "Find documentation about alerts, webhooks, or notifications for when a destination is automatically disabled due to consecutive delivery failures. Also find the API base URL, authentication method, and any concept of \"topics\" or admin webhooks/alerts configuration.", "hasContent": true, "pages": [ { "url": "https://hookdeck.com/docs/outpost" } ], "resultChars": 1257 }, { "source": "web_fetch", "query": "Find details about alert configuration: alert types, consecutive failure thresholds, disabling destinations, webhook alert URL configuration, and the \"alert\" or \"notification\" API endpoints. Include any config keys like ALERT_CALLBACK_URL, MAX_DESTINATION_TIMESTAMP, etc.", "hasContent": true, "pages": [ { "url": "https://hookdeck.com/docs/outpost/configuration" } ], "resultChars": 215 }, { "source": "web_fetch", "query": "List API endpoints related to alerts, topics, admin, and destination webhooks. Include base URL and auth header format.", "hasContent": true, "pages": [ { "url": "https://hookdeck.com/docs/outpost/references/api" } ], "resultChars": 215 }, { "source": "web_search", "query": "hookdeck outpost alert consecutive failures destination disabled webhook", "hasContent": false, "pages": [ { "url": "https://hookdeck.com/outpost/guides/webhook-alerting-and-notifications", "title": "Why Alerting and Notifications Are Necessary for Outbound Webhook ..." }, { "url": "https://hookdeck.com/webhooks/platforms/hookdeck-outpost-vs-convoy-webhook-sending-comparison", "title": "Hookdeck Outpost vs Convoy: Webhook Sending Compared" }, { "url": "https://hookdeck.com/webhooks/platforms/hookdeck-outpost-vs-hook0-webhook-sending-comparison", "title": "Hookdeck Outpost vs Hook0: Webhook Sending Compared" }, { "url": "https://hookdeck.com/docs/outpost/self-hosting/configuration", "title": "Configuration Reference — Outpost Docs" }, { "url": "https://hookdeck.com/blog/evaluating-webhook-infrastructure-for-sending-webhooks", "title": "Evaluating Webhook Infrastructure for Sending Webhooks" }, { "url": "https://hookdeck.com/webhooks/platforms/how-to-solve-woocommerce-5-delivery-failure-webhook-disabling", "title": "How to Solve WooCommerce's 5 Delivery Failure Webhook Disabling" }, { "url": "https://hookdeck.com/outpost/guides/outbound-webhook-retry-best-practices", "title": "Webhook Retry Best Practices for Sending Webhooks" }, { "url": "https://hookdeck.com/docs/outpost/features/event-delivery", "title": "Event Delivery & Retries" } ], "resultChars": 2947 }, { "source": "web_search", "query": "hookdeck outpost documentation \"alert\" topic destination.disabled", "hasContent": false, "pages": [ { "url": "https://hookdeck.com/outpost/guides/webhook-alerting-and-notifications", "title": "Why Alerting and Notifications Are Necessary for Outbound Webhook ..." }, { "url": "https://hookdeck.com/docs/outpost/overview", "title": "Overview" }, { "url": "https://hookdeck.com/docs/outpost/features/event-delivery", "title": "Event Delivery & Retries" }, { "url": "https://hookdeck.com/webhooks/platforms/hookdeck-outpost-vs-hook0-webhook-sending-comparison", "title": "Hookdeck Outpost vs Hook0: Webhook Sending Compared" }, { "url": "https://hookdeck.com/docs/outpost/self-hosting/configuration", "title": "Configuration Reference — Outpost Docs" }, { "url": "https://hookdeck.com/docs/outpost/destinations/webhook", "title": "Webhook" }, { "url": "https://hookdeck.com/docs/destinations", "title": "Destinations" } ], "resultChars": 2301 }, { "source": "web_fetch", "query": "Extract all configuration keys and details related to ALERT_CALLBACK_URL, ALERT_CONSECUTIVE_FAILURE_COUNT, ALERT_AUTO_DISABLE_DESTINATION, and any other alert-related environment variables. Include default values and descriptions.", "hasContent": true, "pages": [ { "url": "https://hookdeck.com/docs/outpost/self-hosting/configuration" } ], "resultChars": 1096 }, { "source": "web_search", "query": "hookdeck outpost API reference alert endpoint POST /alerts", "hasContent": false, "pages": [ { "url": "https://hookdeck.com/docs/api", "title": "REST API Reference" }, { "url": "https://hookdeck.com/docs/outpost/self-hosting/configuration", "title": "Configuration Reference — Outpost Docs" }, { "url": "https://hookdeck.com/docs/outpost/destinations/webhook", "title": "Webhook" }, { "url": "https://hookdeck.com/docs/events", "title": "Events & Attempts" }, { "url": "https://hookdeck.com/outpost/guides/webhook-alerting-and-notifications", "title": "Why Alerting and Notifications Are Necessary for Outbound Webhook ..." }, { "url": "https://hookdeck.com/docs/guides/how-to-test-outbound-webhook-delivery", "title": "Test Outbound Webhook Delivery" }, { "url": "https://hookdeck.com/docs/hookdeck-basics", "title": "Hookdeck Basics" } ], "resultChars": 2343 }, { "source": "web_fetch", "query": "Extract full details on the alert callback system: payload schema/example JSON, how ALERT_CALLBACK_URL is configured, authentication of the callback, alert types (consecutive failures, auto-disable, retries exhausted), and whether this is configurable per-project via API or only via server env vars at deploy time.", "hasContent": true, "pages": [ { "url": "https://hookdeck.com/docs/outpost/features/alerts" } ], "resultChars": 1436 }, { "source": "web_fetch", "query": "Give me the full raw content of this page verbatim, including all headings, code blocks, JSON examples, and configuration instructions for Managed deployments (via Hookdeck dashboard) as opposed to Self-Hosted. Include exact API endpoint paths if mentioned, and exact steps to configure an operator event sink URL via API or dashboard.", "hasContent": true, "pages": [ { "url": "https://hookdeck.com/docs/outpost/features/alerts" } ], "resultChars": 1831 }, { "source": "web_search", "query": "hookdeck outpost \"operator events\" dashboard configure sink managed", "hasContent": false, "pages": [ { "url": "https://hookdeck.com/docs/outpost/guides/building-your-own-ui", "title": "Building Your Own UI — Outpost Docs" }, { "url": "https://hookdeck.com/docs/outpost/concepts", "title": "Concepts — Outpost Docs" }, { "url": "https://hookdeck.com/docs/outpost/guides/migrate-to-outpost", "title": "Migrate Existing Webhooks to Outpost — Outpost Docs" }, { "url": "https://hookdeck.com/docs/platform/outpost-projects", "title": "Outpost Projects" }, { "url": "https://hookdeck.com/docs/outpost/self-hosting/configuration", "title": "Configuration Reference — Outpost Docs" } ], "resultChars": 2224 }, { "source": "web_fetch", "query": "Extract everything about how Outpost projects are configured, especially the API base URL, authentication header, and any settings/monitoring/operator-events endpoints for configuring alert sinks and topics via API rather than dashboard.", "hasContent": true, "pages": [ { "url": "https://hookdeck.com/docs/platform/outpost-projects" } ], "resultChars": 1395 }, { "source": "web_fetch", "query": "List every API endpoint path and method in this reference, especially anything about alerts, operator events, monitoring, settings, or topics.", "hasContent": true, "pages": [ { "url": "https://hookdeck.com/docs/outpost/references/api-reference" } ], "resultChars": 215 }, { "source": "web_search", "query": "outpost hookdeck API base URL \"api.outpost.hookdeck.com\" OR \"outpost.hookdeck.com/api\"", "hasContent": false, "pages": [ { "url": "https://hookdeck.com/docs/outpost/features/tenant-user-portal", "title": "Tenant Portal — Outpost Docs" }, { "url": "https://hookdeck.com/docs/outpost/guides/building-your-own-ui", "title": "Building Your Own UI — Outpost Docs" }, { "url": "https://hookdeck.com/docs/outpost/publishing/events", "title": "Publishing Events" }, { "url": "https://hookdeck.com/docs/outpost/self-hosting/quickstarts/kubernetes", "title": "Deploy on Kubernetes — Outpost Docs" } ], "resultChars": 2273 }, { "source": "web_fetch", "query": "List the full sidebar/table of contents of this documentation site if visible, especially any links related to API reference, monitoring, operator events, and alerts. Give exact URLs.", "hasContent": true, "pages": [ { "url": "https://hookdeck.com/docs/outpost" } ], "resultChars": 1022 }, { "source": "web_fetch", "query": "Give the full content about API topics/endpoints, especially base URL structure, auth, and full list of resources/endpoints available in the sidebar (destinations, tenants, alerts, etc).", "hasContent": true, "pages": [ { "url": "https://hookdeck.com/docs/outpost/api/topics" } ], "resultChars": 215 }, { "source": "web_fetch", "query": "Look for any endpoints related to Outpost projects, monitoring, operator events, alerts, or webhook sinks. List relevant sections/paths.", "hasContent": true, "pages": [ { "url": "https://hookdeck.com/docs/api" } ], "resultChars": 879 } ] }, "prompt": "Last week one of our customers' destinations was switched off after a bad\ndeploy on their side, and their events piled up until they emailed us. We\nfound it by hand.\n\nWe don't want to find out that way again. Set us up so that when Outpost\ndisables a customer's destination, the alert reaches this endpoint:\n\n https://mock.hookdeck.com/operator-alerts\n\nConsecutive delivery failures building up towards that threshold would be\nuseful too — we'd rather know before a destination is switched off than after.", "promptSourcePath": "evals/benchmark-outpost-003-operator-events/PROMPT.md", "attempts": 2, "sourcePath": "claude-code-sonnet-5-no-skills/benchmark-outpost-003-operator-events.json" }, { "experiment": "claude-code-sonnet-5-no-skills", "experimentSuite": "no-skills", "experimentDisplay": { "agent": "claude-code", "modelProvider": "anthropic", "modelId": "claude-sonnet-5", "reasoningEffort": "high" }, "eval": "benchmark-outpost-004-queue-destination", "ranAt": "2026-09-01T11:56:29.846Z", "runId": "33484972784", "stage": "build", "product": [ "outpost" ], "topic": [ "capabilities" ], "suite": "benchmark", "gatedBy": "mixed", "passed": true, "checks": [ { "name": "their orders are delivered to the queue they gave us", "passed": true }, { "name": "the queue is subscribed to their order events", "passed": true }, { "name": "the queue destination carries credentials", "passed": true }, { "name": "their orders no longer go to the old endpoint", "passed": true }, { "name": "the rest of their delivery is unchanged", "passed": true }, { "name": "the other customer was left alone", "passed": true } ], "skills": { "available": [], "loaded": [] }, "docs": { "calls": [ { "source": "web_fetch", "query": "Show me the request schema/body for creating an SQS destination, including required fields and the JSON structure for config and credentials.", "hasContent": true, "pages": [ { "url": "https://hookdeck.com/docs/outpost/references/api/create-destination" } ], "resultChars": 215 }, { "source": "web_fetch", "query": "Find links/docs related to the Outpost API reference for creating destinations, especially for AWS SQS destinations. I need the exact request body schema (config and credentials fields) for an SQS destination.", "hasContent": true, "pages": [ { "url": "https://hookdeck.com/docs/outpost" } ], "resultChars": 997 }, { "source": "web_fetch", "query": "Show the exact JSON request body schema for creating an SQS destination via the Outpost API, including type, topics, config fields (queue url, region) and credentials fields (access key, secret key).", "hasContent": true, "pages": [ { "url": "https://hookdeck.com/docs/outpost/destinations/aws-sqs" } ], "resultChars": 988 }, { "source": "web_fetch", "query": "Show the exact JSON request body schema for creating an SQS destination via POST /tenants/{tenant_id}/destinations, including all fields for type aws_sqs: config (queue_url, region?) and credentials (key, secret, session).", "hasContent": true, "pages": [ { "url": "https://hookdeck.com/docs/outpost/references/api-reference/create-destination" } ], "resultChars": 215 } ] }, "prompt": "Acme are moving off webhooks. Their endpoint keeps falling over under load and\nthey'd rather we drop order events straight onto a queue they already run.\n\nThey sent us the queue details last week — whoever picked up the ticket put\nthem in the repo.\n\nSet that up, and stop sending their orders to the old endpoint.", "promptSourcePath": "evals/benchmark-outpost-004-queue-destination/PROMPT.md", "attempts": 1, "sourcePath": "claude-code-sonnet-5-no-skills/benchmark-outpost-004-queue-destination.json" }, { "experiment": "claude-code-sonnet-5-no-skills", "experimentSuite": "no-skills", "experimentDisplay": { "agent": "claude-code", "modelProvider": "anthropic", "modelId": "claude-sonnet-5", "reasoningEffort": "high" }, "eval": "benchmark-outpost-005-topic-scoping", "ranAt": "2026-09-01T12:12:04.693Z", "runId": "33484972784", "stage": "resolve", "product": [ "outpost" ], "topic": [ "filtering" ], "suite": "benchmark", "gatedBy": "mixed", "passed": true, "checks": [ { "name": "acme no longer receives order cancellations", "passed": true }, { "name": "acme still receives the shipping events they depend on", "passed": true }, { "name": "acme still receives new orders", "passed": true }, { "name": "globex was left alone", "passed": true }, { "name": "no destination was switched off to achieve it", "passed": true } ], "skills": { "available": [], "loaded": [] }, "docs": { "calls": [] }, "prompt": "Acme have been in touch. Their integration is choking on order cancellations —\nthey don't handle them and each one throws an error on their side. Their order\nconfirmations are working fine and they rely on those.\n\nStop the cancellations.", "promptSourcePath": "evals/benchmark-outpost-005-topic-scoping/PROMPT.md", "attempts": 1, "sourcePath": "claude-code-sonnet-5-no-skills/benchmark-outpost-005-topic-scoping.json" }, { "experiment": "claude-code-sonnet-5-no-skills", "experimentSuite": "no-skills", "experimentDisplay": { "agent": "claude-code", "modelProvider": "anthropic", "modelId": "claude-sonnet-5", "reasoningEffort": "high" }, "eval": "benchmark-resolve-001-paused-connection", "ranAt": "2026-09-01T12:27:29.471Z", "runId": "33484972784", "stage": "resolve", "product": [ "event-gateway" ], "topic": [ "retries" ], "suite": "benchmark", "gatedBy": "discovery", "passed": true, "checks": [ { "name": "the connection is no longer paused", "passed": true }, { "name": "a new checkout event reaches the orders API", "passed": true } ], "skills": { "available": [], "loaded": [] }, "docs": { "calls": [] }, "prompt": "Checkout events stopped reaching our orders API. Nothing's changed on our side\nthat I know of, and the endpoint is up.\n\nGet them flowing again.", "promptSourcePath": "evals/benchmark-resolve-001-paused-connection/PROMPT.md", "attempts": 1, "sourcePath": "claude-code-sonnet-5-no-skills/benchmark-resolve-001-paused-connection.json" }, { "experiment": "claude-code-sonnet-5-no-skills", "experimentSuite": "no-skills", "experimentDisplay": { "agent": "claude-code", "modelProvider": "anthropic", "modelId": "claude-sonnet-5", "reasoningEffort": "high" }, "eval": "benchmark-resolve-002-scoped-redelivery", "ranAt": "2026-09-01T12:42:05.441Z", "runId": "33484972784", "stage": "resolve", "product": [ "event-gateway" ], "topic": [ "retries" ], "suite": "benchmark", "gatedBy": "mixed", "passed": true, "checks": [ { "name": "the failed checkout events were redelivered", "passed": true }, { "name": "the inventory events were left alone", "passed": true } ], "skills": { "available": [], "loaded": [] }, "docs": { "calls": [] }, "prompt": "We had an outage this morning and both our endpoints were rejecting everything.\nThey're back up now.\n\nCheckout is the urgent one, so get those failed events redelivered. Leave\ninventory alone for now, the team there wants to check something first.", "promptSourcePath": "evals/benchmark-resolve-002-scoped-redelivery/PROMPT.md", "attempts": 1, "sourcePath": "claude-code-sonnet-5-no-skills/benchmark-resolve-002-scoped-redelivery.json" }, { "experiment": "claude-code-sonnet-5-no-skills", "experimentSuite": "no-skills", "experimentDisplay": { "agent": "claude-code", "modelProvider": "anthropic", "modelId": "claude-sonnet-5", "reasoningEffort": "high" }, "eval": "benchmark-transform-001-reshape-payload", "ranAt": "2026-09-01T12:59:25.835Z", "runId": "33484972784", "stage": "build", "product": [ "event-gateway" ], "topic": [ "transformations" ], "suite": "benchmark", "gatedBy": "discovery", "passed": true, "checks": [ { "name": "amount is at the top level of the delivered body", "passed": true }, { "name": "currency is at the top level of the delivered body", "passed": true }, { "name": "email is at the top level of the delivered body", "passed": true } ], "skills": { "available": [], "loaded": [] }, "docs": { "calls": [] }, "prompt": "Our payment provider changed their payload and our billing service can't read\nit any more. The service expects `amount`, `currency` and `email` at the top\nlevel of the body, and nothing else has to change.\n\nSort it out so the billing service starts working again. I'd rather know it's\nright before it goes anywhere near production.", "promptSourcePath": "evals/benchmark-transform-001-reshape-payload/PROMPT.md", "attempts": 1, "sourcePath": "claude-code-sonnet-5-no-skills/benchmark-transform-001-reshape-payload.json" }, { "experiment": "claude-code-sonnet-5-no-skills", "experimentSuite": "no-skills", "experimentDisplay": { "agent": "claude-code", "modelProvider": "anthropic", "modelId": "claude-sonnet-5", "reasoningEffort": "high" }, "eval": "benchmark-verification-001-stripe-express", "ranAt": "2026-09-01T13:28:42.886Z", "runId": "33484972784", "stage": "build", "product": [ "event-gateway" ], "topic": [ "signature-verification" ], "suite": "benchmark", "gatedBy": "discovery", "passed": false, "checks": [ { "name": "created a Stripe source", "passed": true }, { "name": "the source accepts a genuine Stripe signature", "passed": false }, { "name": "the source rejects a forged Stripe signature", "passed": true }, { "name": "the handler accepts a genuine Hookdeck signature", "passed": true }, { "name": "the handler rejects a forged signature", "passed": false, "notes": "expected a 4xx, got 200" } ], "skills": { "available": [], "loaded": [] }, "docs": { "calls": [ { "source": "web_fetch", "query": "Show the request body schema for updating a source, specifically how to set webhook/HMAC verification (auth) config, especially for STRIPE type sources.", "hasContent": true, "pages": [ { "url": "https://hookdeck.com/api-ref/source/operation/updateSource" } ], "resultChars": 215 }, { "source": "web_fetch", "query": "Show how to configure source verification/auth for a STRIPE type source when creating or updating a source via the API.", "hasContent": true, "pages": [ { "url": "https://hookdeck.com/docs/api#sources" } ], "resultChars": 1116 } ] }, "prompt": "We're taking Stripe payments and I need our orders API to react when a checkout\ncompletes. Right now nothing is wired up.\n\nSet it up so Stripe events reach the `/orders` endpoint, and make sure the\nendpoint only accepts requests that genuinely came through, not anything that\nturns up at the URL. I'm still working on this locally, and I don't want to miss\nevents while I'm developing.", "promptSourcePath": "evals/benchmark-verification-001-stripe-express/PROMPT.md", "attempts": 1, "sourcePath": "claude-code-sonnet-5-no-skills/benchmark-verification-001-stripe-express.json" }, { "experiment": "claude-code-sonnet-5-no-skills", "experimentSuite": "no-skills", "experimentDisplay": { "agent": "claude-code", "modelProvider": "anthropic", "modelId": "claude-sonnet-5", "reasoningEffort": "high" }, "eval": "benchmark-verification-002-elevenlabs-callbacks", "ranAt": "2026-09-01T14:00:38.245Z", "runId": "33484972784", "stage": "build", "product": [ "event-gateway" ], "topic": [ "signature-verification" ], "suite": "benchmark", "gatedBy": "discovery", "passed": true, "checks": [ { "name": "created an ElevenLabs source", "passed": true }, { "name": "the source accepts a genuine ElevenLabs signature", "passed": true }, { "name": "the source rejects a forged ElevenLabs signature", "passed": true }, { "name": "the service accepts a genuine Hookdeck signature", "passed": true }, { "name": "the service rejects a forged signature", "passed": true, "notes": "expected a 4xx, got 401" } ], "skills": { "available": [], "loaded": [] }, "docs": { "calls": [ { "source": "web_fetch", "query": "What is the source verification configuration for ElevenLabs webhooks in Hookdeck? What config field name is used for the webhook secret, and what does the CLI/API config object look like?", "hasContent": true, "pages": [ { "url": "https://hookdeck.com/docs/sources/verification/elevenlabs" } ], "resultChars": 215 }, { "source": "web_fetch", "query": "List all supported webhook verification source types in Hookdeck, and specifically describe the ElevenLabs verification config, including the JSON config field names needed to configure it (e.g. webhook secret field name).", "hasContent": true, "pages": [ { "url": "https://hookdeck.com/docs/verification" } ], "resultChars": 215 }, { "source": "web_fetch", "query": "Describe how to configure source verification for a webhook source in Hookdeck, particularly for providers like ElevenLabs that sign requests. What is the config JSON structure for source create with type ELEVENLABS?", "hasContent": true, "pages": [ { "url": "https://hookdeck.com/docs/receiving-events" } ], "resultChars": 215 }, { "source": "web_search", "query": "Hookdeck ElevenLabs source verification webhook secret config", "hasContent": false, "pages": [ { "url": "https://hookdeck.com/docs/guides/how-to-verify-incoming-webhooks", "title": "Verify Incoming Webhook Signatures" }, { "url": "https://hookdeck.com/webhooks/platforms/how-to-secure-and-verify-github-webhooks-with-hookdeck", "title": "How to Secure and Verify GitHub Webhooks with Hookdeck" }, { "url": "https://hookdeck.com/docs/sources#create-a-source", "title": "Sources" }, { "url": "https://hookdeck.com/docs/authentication", "title": "Authentication & Verification" } ], "resultChars": 2605 }, { "source": "web_fetch", "query": "Find information about the ELEVENLABS source verification type - what is the config JSON structure needed (field names for webhook secret / hmac config) to set up signature verification for an ElevenLabs source in Hookdeck via API or CLI?", "hasContent": true, "pages": [ { "url": "https://hookdeck.com/docs/sources" } ], "resultChars": 966 }, { "source": "web_search", "query": "Hookdeck verify requests coming from Hookdeck X-Hookdeck-Signature webhook secret header", "hasContent": false, "pages": [ { "url": "https://hookdeck.com/docs/signature-verification", "title": "Signature verification overview - Hookdeck" }, { "url": "https://hookdeck.com/docs/guides/how-to-verify-incoming-webhooks", "title": "Verify Incoming Webhook Signatures" }, { "url": "https://hookdeck.com/webhooks/platforms/how-to-secure-and-verify-bitbucket-webhooks-with-hookdeck", "title": "How to Secure and Verify Bitbucket Webhooks" }, { "url": "https://hookdeck.com/docs/authentication", "title": "Authentication & Verification" }, { "url": "https://hookdeck.com/webhooks/platforms/how-to-secure-and-verify-paddle-webhooks-with-hookdeck", "title": "How to Secure and Verify Paddle Webhooks" }, { "url": "https://hookdeck.com/webhooks/platforms/how-to-secure-and-verify-github-webhooks-with-hookdeck", "title": "How to Secure and Verify GitHub Webhooks with Hookdeck" }, { "url": "https://hookdeck.com/webhooks/platforms/how-to-secure-and-verify-jira-webhooks-with-hookdeck", "title": "How to Secure and Verify Jira Webhooks with Hookdeck" }, { "url": "https://hookdeck.com/webhooks/platforms/how-to-secure-and-verify-woocommerce-webhooks-with-hookdeck", "title": "How to Secure and Verify WooCommerce Webhooks with Hookdeck" }, { "url": "https://hookdeck.com/webhooks/platforms/how-to-secure-and-verify-commerce-layer-webhooks-with-hookdeck", "title": "How to Secure and Verify Commerce Layer Webhooks with Hookdeck" }, { "url": "https://hookdeck.com/webhooks/skills/github-webhooks", "title": "GitHub Webhooks Skill - Hookdeck" } ], "resultChars": 2879 }, { "source": "web_fetch", "query": "Give the exact Node.js/Express code example for verifying the X-Hookdeck-Signature header on incoming requests, including the exact HMAC algorithm, encoding (base64 or hex), and how the raw body is used.", "hasContent": true, "pages": [ { "url": "https://hookdeck.com/docs/signature-verification" } ], "resultChars": 1134 } ] }, "prompt": "We're using ElevenLabs for transcription and they call us back when a job\nfinishes. The callbacks need to reach the transcripts service in this directory,\nand I don't want it accepting anything that didn't genuinely come from them via\nour gateway.\n\nSet that up.", "promptSourcePath": "evals/benchmark-verification-002-elevenlabs-callbacks/PROMPT.md", "attempts": 1, "sourcePath": "claude-code-sonnet-5-no-skills/benchmark-verification-002-elevenlabs-callbacks.json" }, { "experiment": "codex-gpt-5.4-mini", "experimentSuite": "benchmark", "experimentDisplay": { "agent": "codex", "modelProvider": "openai", "modelId": "gpt-5.4-mini", "reasoningEffort": "high" }, "eval": "benchmark-alerting-001-delivery-alerts", "ranAt": "2026-09-01T08:08:04.924Z", "runId": "33484972784", "stage": "build", "product": [ "event-gateway" ], "topic": [ "alerting" ], "suite": "benchmark", "gatedBy": "discovery", "passed": true, "checks": [ { "name": "an alert exists for delivery failures", "passed": true }, { "name": "the alert is enabled", "passed": true }, { "name": "the alert covers the fulfilment connection", "passed": true }, { "name": "the alert has a notification channel configured", "passed": true } ], "skills": { "available": [ "hookdeck", "event-gateway" ], "loaded": [ "hookdeck", "event-gateway" ] }, "docs": { "calls": [] }, "prompt": "We found out our fulfilment endpoint had been rejecting orders for most of a day\nbecause nobody was watching. I don't want to find out that way again.\n\nSet it up so we get told when deliveries to that endpoint keep failing.", "promptSourcePath": "evals/benchmark-alerting-001-delivery-alerts/PROMPT.md", "attempts": 1, "sourcePath": "codex-gpt-5.4-mini/benchmark-alerting-001-delivery-alerts.json" }, { "experiment": "codex-gpt-5.4-mini", "experimentSuite": "benchmark", "experimentDisplay": { "agent": "codex", "modelProvider": "openai", "modelId": "gpt-5.4-mini", "reasoningEffort": "high" }, "eval": "benchmark-dedupe-001-duplicate-events", "ranAt": "2026-09-01T08:29:42.623Z", "runId": "33484972784", "stage": "build", "product": [ "event-gateway" ], "topic": [ "deduplication" ], "suite": "benchmark", "gatedBy": "discovery", "passed": true, "checks": [ { "name": "the duplicate did not reach the ledger", "passed": true }, { "name": "the original still reached the ledger", "passed": true } ], "skills": { "available": [ "hookdeck", "event-gateway" ], "loaded": [ "hookdeck", "event-gateway" ] }, "docs": { "calls": [] }, "prompt": "Our payment provider sometimes sends us the same event twice within a few\nseconds, and our ledger ends up double-counting it. We can't easily change the\nledger service.\n\nStop the duplicates reaching it.", "promptSourcePath": "evals/benchmark-dedupe-001-duplicate-events/PROMPT.md", "attempts": 1, "sourcePath": "codex-gpt-5.4-mini/benchmark-dedupe-001-duplicate-events.json" }, { "experiment": "codex-gpt-5.4-mini", "experimentSuite": "benchmark", "experimentDisplay": { "agent": "codex", "modelProvider": "openai", "modelId": "gpt-5.4-mini", "reasoningEffort": "high" }, "eval": "benchmark-delivery-001-slow-consumer", "ranAt": "2026-09-01T08:45:33.407Z", "runId": "33484972784", "stage": "build", "product": [ "event-gateway" ], "topic": [ "rate-limits" ], "suite": "benchmark", "gatedBy": "discovery", "passed": true, "checks": [ { "name": "delivery to the worker is limited", "passed": true }, { "name": "the limit caps concurrent deliveries rather than a rate over time", "passed": true }, { "name": "the concurrency ceiling matches what the worker can hold", "passed": true } ], "skills": { "available": [ "hookdeck", "event-gateway" ], "loaded": [ "hookdeck", "event-gateway" ] }, "docs": { "calls": [] }, "prompt": "Our inference worker takes about ninety seconds per job and it can only handle\ntwo at a time. When traffic spikes it falls over and we lose jobs.\n\nSet the delivery up so that doesn't happen.", "promptSourcePath": "evals/benchmark-delivery-001-slow-consumer/PROMPT.md", "attempts": 1, "sourcePath": "codex-gpt-5.4-mini/benchmark-delivery-001-slow-consumer.json" }, { "experiment": "codex-gpt-5.4-mini", "experimentSuite": "benchmark", "experimentDisplay": { "agent": "codex", "modelProvider": "openai", "modelId": "gpt-5.4-mini", "reasoningEffort": "high" }, "eval": "benchmark-delivery-002-rate-limited-endpoint", "ranAt": "2026-09-01T08:56:34.250Z", "runId": "33484972784", "stage": "build", "product": [ "event-gateway" ], "topic": [ "rate-limits" ], "suite": "benchmark", "gatedBy": "discovery", "passed": true, "checks": [ { "name": "delivery to the endpoint is limited", "passed": true }, { "name": "the limit is over time rather than concurrency", "passed": true }, { "name": "the limit works out at the endpoint quota", "passed": true, "notes": "works out at 60 per minute against a quota of 60" } ], "skills": { "available": [ "hookdeck", "event-gateway" ], "loaded": [ "hookdeck", "event-gateway" ] }, "docs": { "calls": [] }, "prompt": "The API we forward enrichment jobs to allows sixty requests a minute and starts\nreturning 429s past that. We're well over it at peak.\n\nMake our deliveries stay inside their limit.", "promptSourcePath": "evals/benchmark-delivery-002-rate-limited-endpoint/PROMPT.md", "attempts": 1, "sourcePath": "codex-gpt-5.4-mini/benchmark-delivery-002-rate-limited-endpoint.json" }, { "experiment": "codex-gpt-5.4-mini", "experimentSuite": "benchmark", "experimentDisplay": { "agent": "codex", "modelProvider": "openai", "modelId": "gpt-5.4-mini", "reasoningEffort": "high" }, "eval": "benchmark-filtering-001-enterprise-orders", "ranAt": "2026-09-01T09:12:24.898Z", "runId": "33484972784", "stage": "build", "product": [ "event-gateway" ], "topic": [ "filtering" ], "suite": "benchmark", "gatedBy": "discovery", "passed": true, "checks": [ { "name": "did not claim regex filtering or channels exist", "passed": true, "judgeNotes": "Assistant explicitly states Hookdeck filters don’t expose a regex operator and uses $startsWith instead. No channel/channel partitioning feature described." }, { "name": "current-format references still reach the endpoint", "passed": true, "notes": "expected an event for ORD-2026-ZZ-9001, found 1" }, { "name": "legacy references are filtered out", "passed": true, "notes": "expected no event for LEGACY-99002, found 0" } ], "skills": { "available": [ "hookdeck", "event-gateway" ], "loaded": [ "hookdeck", "event-gateway" ] }, "docs": { "calls": [ { "source": "shell_fetch", "query": "/bin/bash -lc \"curl -fsSL https://hookdeck.com/docs/filters.md | sed -n '1,240p'\"", "hasContent": true, "pages": [ { "url": "https://hookdeck.com/docs/filters.md" } ], "resultChars": 4795 } ] }, "prompt": "Our order references look like `ORD-2026-AC-4821`. We migrated formats at the\nstart of the year and the old ones are still coming through, which my service\nchokes on.\n\nCan I use a regex on the reference to only let this year's format through?\nWhatever the answer, set up the filtering so only the current format reaches my\nendpoint.", "promptSourcePath": "evals/benchmark-filtering-001-enterprise-orders/PROMPT.md", "attempts": 1, "sourcePath": "codex-gpt-5.4-mini/benchmark-filtering-001-enterprise-orders.json" }, { "experiment": "codex-gpt-5.4-mini", "experimentSuite": "benchmark", "experimentDisplay": { "agent": "codex", "modelProvider": "openai", "modelId": "gpt-5.4-mini", "reasoningEffort": "high" }, "eval": "benchmark-filtering-002-high-value-retries", "ranAt": "2026-09-01T09:36:13.217Z", "runId": "33484972784", "stage": "build", "product": [ "event-gateway" ], "topic": [ "filtering" ], "suite": "benchmark", "gatedBy": "discovery", "passed": false, "checks": [ { "name": "an order above the threshold reaches manual review", "passed": false, "notes": "a £501 order did not arrive: the filter excludes orders it should admit" }, { "name": "an order below the threshold does not reach manual review", "passed": true }, { "name": "failed deliveries are retried", "passed": true } ], "skills": { "available": [ "hookdeck", "event-gateway" ], "loaded": [ "hookdeck", "event-gateway" ] }, "docs": { "calls": [] }, "prompt": "Only orders above £500 should go to the manual review queue. Everything smaller\nis handled automatically and shouldn't reach it.\n\nAlso, review's endpoint goes down occasionally and we've lost orders that way.\nIf a delivery fails, keep trying for a while rather than giving up.", "promptSourcePath": "evals/benchmark-filtering-002-high-value-retries/PROMPT.md", "attempts": 1, "sourcePath": "codex-gpt-5.4-mini/benchmark-filtering-002-high-value-retries.json" }, { "experiment": "codex-gpt-5.4-mini", "experimentSuite": "benchmark", "experimentDisplay": { "agent": "codex", "modelProvider": "openai", "modelId": "gpt-5.4-mini", "reasoningEffort": "high" }, "eval": "benchmark-investigate-001-failing-deliveries", "ranAt": "2026-09-01T09:51:50.173Z", "runId": "33484972784", "stage": "investigate", "product": [ "event-gateway" ], "topic": [ "retries" ], "suite": "benchmark", "gatedBy": "mixed", "passed": true, "checks": [ { "name": "inspected delivery history rather than only configuration", "passed": true }, { "name": "identified the failing destination and that it is rejecting", "passed": true, "judgeNotes": "Identifies orders-to-analytics/analytics destination as failing, while fulfilment succeeds, and states it failed with HTTP 422 from the analytics endpoint/URL." }, { "name": "did not invent a cause it could not have observed", "passed": true, "judgeNotes": "The assistant bases the cause on observed Hookdeck state: destination URL includes status=422 and delivery attempts failed with HTTP 422. It does not assert any prohibited invented cause such as rate limiting, dropped/lost deliveries, payload limit, paused connection, or nonexistent feature." } ], "skills": { "available": [ "hookdeck", "event-gateway" ], "loaded": [ "hookdeck", "event-gateway" ] }, "docs": { "calls": [] }, "prompt": "Our order events aren't all getting through. The fulfilment side seems fine but\nsomething's definitely broken, and I've had it flagged twice this week.\n\nCan you work out what's actually going wrong?", "promptSourcePath": "evals/benchmark-investigate-001-failing-deliveries/PROMPT.md", "attempts": 1, "sourcePath": "codex-gpt-5.4-mini/benchmark-investigate-001-failing-deliveries.json" }, { "experiment": "codex-gpt-5.4-mini", "experimentSuite": "benchmark", "experimentDisplay": { "agent": "codex", "modelProvider": "openai", "modelId": "gpt-5.4-mini", "reasoningEffort": "high" }, "eval": "benchmark-investigate-002-partial-outage", "ranAt": "2026-09-01T10:04:27.361Z", "runId": "33484972784", "stage": "investigate", "product": [ "event-gateway" ], "topic": [ "filtering" ], "suite": "benchmark", "gatedBy": "mixed", "passed": true, "checks": [ { "name": "looked past the events list", "passed": true }, { "name": "identified the filter as the cause and what it excludes", "passed": true, "judgeNotes": "Identifies the connection filter as the cause and states that only web orders pass while non-web/pos orders are ignored and produce no event." }, { "name": "did not report the system as healthy", "passed": true, "judgeNotes": "The assistant did not claim everything was working or that all orders were delivered; it identified a filter dropping some orders." } ], "skills": { "available": [ "hookdeck", "event-gateway" ], "loaded": [ "event-gateway" ] }, "docs": { "calls": [] }, "prompt": "Our warehouse team says they're missing orders. Not all of them, and I can't\nfind a single failure anywhere I've looked. Every delivery I can see went\nthrough fine.\n\nWhere are the missing ones going?", "promptSourcePath": "evals/benchmark-investigate-002-partial-outage/PROMPT.md", "attempts": 1, "sourcePath": "codex-gpt-5.4-mini/benchmark-investigate-002-partial-outage.json" }, { "experiment": "codex-gpt-5.4-mini", "experimentSuite": "benchmark", "experimentDisplay": { "agent": "codex", "modelProvider": "openai", "modelId": "gpt-5.4-mini", "reasoningEffort": "high" }, "eval": "benchmark-localdev-001-listen-locally", "ranAt": "2026-09-01T10:17:13.771Z", "runId": "33484972784", "stage": "build", "product": [ "event-gateway" ], "topic": [ "local-dev" ], "suite": "benchmark", "gatedBy": "discovery", "passed": true, "checks": [ { "name": "created a source to receive events", "passed": true }, { "name": "routed to a CLI destination rather than an unreachable URL", "passed": true }, { "name": "an event posted at the source arrives at the local service", "passed": true } ], "skills": { "available": [ "hookdeck", "event-gateway" ], "loaded": [ "hookdeck", "event-gateway" ] }, "docs": { "calls": [] }, "prompt": "I've written the notifications service in this directory but I've got no way to\ntry it against real traffic. It's not deployed anywhere yet and I'd rather not\npush it somewhere just to find out the handler is wrong.\n\nGet events arriving at it here on my machine so I can watch them come in.", "promptSourcePath": "evals/benchmark-localdev-001-listen-locally/PROMPT.md", "attempts": 1, "sourcePath": "codex-gpt-5.4-mini/benchmark-localdev-001-listen-locally.json" }, { "experiment": "codex-gpt-5.4-mini", "experimentSuite": "benchmark", "experimentDisplay": { "agent": "codex", "modelProvider": "openai", "modelId": "gpt-5.4-mini", "reasoningEffort": "high" }, "eval": "benchmark-outpost-001-customer-subscriptions", "ranAt": "2026-09-01T10:33:34.910Z", "runId": "33484972784", "stage": "build", "product": [ "outpost" ], "topic": [ "sdk" ], "suite": "benchmark", "gatedBy": "discovery", "passed": true, "checks": [ { "name": "created a tenant for the customer", "passed": true }, { "name": "the customer has a destination at the endpoint they gave us", "passed": true }, { "name": "an order event reaches the customer", "passed": true } ], "skills": { "available": [ "hookdeck", "event-gateway", "outpost" ], "loaded": [ "hookdeck", "outpost" ] }, "docs": { "calls": [] }, "prompt": "We want our customers to be able to subscribe to events from our platform, so\nthey can react when an order is placed rather than polling us for it.\n\nSet that up for our first customer, `acme`, so they receive order events at\nthe endpoint they gave us:\n\n https://mock.hookdeck.com/api/v1/acme/orders\n\nThen show me it works by sending one through.", "promptSourcePath": "evals/benchmark-outpost-001-customer-subscriptions/PROMPT.md", "attempts": 1, "sourcePath": "codex-gpt-5.4-mini/benchmark-outpost-001-customer-subscriptions.json" }, { "experiment": "codex-gpt-5.4-mini", "experimentSuite": "benchmark", "experimentDisplay": { "agent": "codex", "modelProvider": "openai", "modelId": "gpt-5.4-mini", "reasoningEffort": "high" }, "eval": "benchmark-outpost-002-disabled-destination", "ranAt": "2026-09-01T10:53:11.391Z", "runId": "33484972784", "stage": "resolve", "product": [ "outpost" ], "topic": [ "retries" ], "suite": "benchmark", "gatedBy": "discovery", "passed": false, "checks": [ { "name": "the customer can receive again", "passed": false, "notes": "all 1 of acme's destinations are still disabled, so nothing published to acme will be delivered" }, { "name": "the events the customer missed were delivered", "passed": false, "notes": "0 of 3 missed events delivered: the destination may be receiving again, but the customer is still missing the outage window" }, { "name": "the other customer was left alone", "passed": true } ], "skills": { "available": [ "hookdeck", "event-gateway", "outpost" ], "loaded": [ "hookdeck", "event-gateway", "outpost" ] }, "docs": { "calls": [] }, "prompt": "Acme emailed to say they stopped receiving order events some time yesterday.\nTheir engineer says their endpoint had a bad deploy but it's been fine since\nthis morning, and they've checked — nothing is arriving.\n\nWork out why and get their events flowing again, including the ones they\nmissed.", "promptSourcePath": "evals/benchmark-outpost-002-disabled-destination/PROMPT.md", "attempts": 1, "sourcePath": "codex-gpt-5.4-mini/benchmark-outpost-002-disabled-destination.json" }, { "experiment": "codex-gpt-5.4-mini", "experimentSuite": "benchmark", "experimentDisplay": { "agent": "codex", "modelProvider": "openai", "modelId": "gpt-5.4-mini", "reasoningEffort": "high" }, "eval": "benchmark-outpost-003-operator-events", "ranAt": "2026-09-01T11:13:32.649Z", "runId": "33484972784", "stage": "build", "product": [ "outpost" ], "topic": [ "alerting" ], "suite": "benchmark", "gatedBy": "discovery", "passed": true, "checks": [ { "name": "alerts are delivered to the endpoint that was asked for", "passed": true }, { "name": "the disabled-destination alert is subscribed", "passed": true }, { "name": "the alerting destination is enabled", "passed": true }, { "name": "the build-up to a disable is subscribed too", "passed": true } ], "skills": { "available": [ "hookdeck", "event-gateway", "outpost" ], "loaded": [ "hookdeck", "outpost" ] }, "docs": { "calls": [ { "source": "shell_fetch", "query": "/bin/bash -lc \"curl -fsSL https://hookdeck.com/docs/outpost/llms.txt | sed -n '1,200p'\"", "hasContent": true, "pages": [ { "url": "https://hookdeck.com/docs/outpost/llms.txt" } ], "resultChars": 11412 }, { "source": "shell_fetch", "query": "/bin/bash -lc \"curl -fsSL https://hookdeck.com/docs/outpost/api/configure.md | sed -n '1,260p'\"", "hasContent": true, "pages": [ { "url": "https://hookdeck.com/docs/outpost/api/configure.md" } ], "resultChars": 5832 }, { "source": "shell_fetch", "query": "/bin/bash -lc \"curl -fsSL https://hookdeck.com/docs/outpost/features/operator-events.md | sed -n '1,260p'\"", "hasContent": true, "pages": [ { "url": "https://hookdeck.com/docs/outpost/features/operator-events.md" } ], "resultChars": 8210 }, { "source": "shell_fetch", "query": "/bin/bash -lc \"curl -fsSL https://hookdeck.com/docs/outpost/api.md | sed -n '1,240p'\"", "hasContent": true, "pages": [ { "url": "https://hookdeck.com/docs/outpost/api.md" } ], "resultChars": 5554 }, { "source": "shell_fetch", "query": "/bin/bash -lc 'curl -fsSL https://hookdeck.com/docs/outpost/api/configure.md | rg -n \"alert|monitor|operation|destination.disabled|consecutive_failure|ALERT_\"'", "hasContent": true, "pages": [ { "url": "https://hookdeck.com/docs/outpost/api/configure.md" } ], "resultChars": 860 }, { "source": "shell_fetch", "query": "/bin/bash -lc 'curl -fsSL https://hookdeck.com/docs/outpost/api.md | rg -n \"monitor|operator|alert|configuration\"'", "hasContent": true, "pages": [ { "url": "https://hookdeck.com/docs/outpost/api.md" } ], "resultChars": 248 }, { "source": "shell_fetch", "query": "/bin/bash -lc 'curl -fsSL https://hookdeck.com/docs/outpost/api/resources.md | rg -n \"alert|operator|topic|configuration|monitor\"'", "hasContent": true, "pages": [ { "url": "https://hookdeck.com/docs/outpost/api/resources.md" } ], "resultChars": 909 }, { "source": "shell_fetch", "query": "/bin/bash -lc \"curl -fsSL https://hookdeck.com/docs/outpost/api/resources.md | sed -n '150,230p'\"", "hasContent": true, "pages": [ { "url": "https://hookdeck.com/docs/outpost/api/resources.md" } ], "resultChars": 1667 }, { "source": "shell_fetch", "query": "/bin/bash -lc \"curl -fsSL https://hookdeck.com/docs/outpost/api/configure.md | sed -n '940,1040p'\"", "hasContent": true, "pages": [ { "url": "https://hookdeck.com/docs/outpost/api/configure.md" } ], "resultChars": 520 }, { "source": "shell_fetch", "query": "/bin/bash -lc \"curl -fsSL https://hookdeck.com/docs/outpost/api/configure.md | sed -n '820,940p'\"", "hasContent": true, "pages": [ { "url": "https://hookdeck.com/docs/outpost/api/configure.md" } ], "resultChars": 6413 }, { "source": "shell_fetch", "query": "/bin/bash -lc \"curl -fsSL https://hookdeck.com/docs/outpost/api/configure.md | sed -n '840,900p'\"", "hasContent": true, "pages": [ { "url": "https://hookdeck.com/docs/outpost/api/configure.md" } ], "resultChars": 3697 }, { "source": "shell_fetch", "query": "/bin/bash -lc \"curl -fsSL https://hookdeck.com/docs/outpost/api/configure.md | sed -n '760,840p'\"", "hasContent": true, "pages": [ { "url": "https://hookdeck.com/docs/outpost/api/configure.md" } ], "resultChars": 1997 }, { "source": "shell_fetch", "query": "/bin/bash -lc 'curl -fsSL https://hookdeck.com/docs/platform/outpost-projects.md | rg -n \"monitor|alert|operator|sink|outpost\"'", "hasContent": true, "pages": [ { "url": "https://hookdeck.com/docs/platform/outpost-projects.md" } ], "resultChars": 291 }, { "source": "shell_fetch", "query": "/bin/bash -lc \"curl -fsSL https://hookdeck.com/docs/platform/outpost-projects.md | sed -n '1,260p'\"", "hasContent": true, "pages": [ { "url": "https://hookdeck.com/docs/platform/outpost-projects.md" } ], "resultChars": 5841 }, { "source": "shell_fetch", "query": "/bin/bash -lc 'curl -fsSL https://hookdeck.com/docs/platform/organizations.md | rg -n \"outpost|project|monitor|alert\"'", "hasContent": true, "pages": [ { "url": "https://hookdeck.com/docs/platform/organizations.md" } ], "resultChars": 1245 }, { "source": "shell_fetch", "query": "/bin/bash -lc \"curl -fsSL https://hookdeck.com/docs/platform/llms.txt | sed -n '1,240p'\"", "hasContent": true, "pages": [ { "url": "https://hookdeck.com/docs/platform/llms.txt" } ], "resultChars": 49 }, { "source": "shell_fetch", "query": "/bin/bash -lc 'curl -fsSL https://hookdeck.com/docs/platform/outpost-projects.md | rg -n \"Monitoring|monitoring|settings/project/monitoring|operation events|alerts\"'", "hasContent": true, "pages": [ { "url": "https://hookdeck.com/docs/platform/outpost-projects.md" } ], "resultChars": 208 } ] }, "prompt": "Last week one of our customers' destinations was switched off after a bad\ndeploy on their side, and their events piled up until they emailed us. We\nfound it by hand.\n\nWe don't want to find out that way again. Set us up so that when Outpost\ndisables a customer's destination, the alert reaches this endpoint:\n\n https://mock.hookdeck.com/operator-alerts\n\nConsecutive delivery failures building up towards that threshold would be\nuseful too — we'd rather know before a destination is switched off than after.", "promptSourcePath": "evals/benchmark-outpost-003-operator-events/PROMPT.md", "attempts": 1, "sourcePath": "codex-gpt-5.4-mini/benchmark-outpost-003-operator-events.json" }, { "experiment": "codex-gpt-5.4-mini", "experimentSuite": "benchmark", "experimentDisplay": { "agent": "codex", "modelProvider": "openai", "modelId": "gpt-5.4-mini", "reasoningEffort": "high" }, "eval": "benchmark-outpost-004-queue-destination", "ranAt": "2026-09-01T11:50:44.628Z", "runId": "33484972784", "stage": "build", "product": [ "outpost" ], "topic": [ "capabilities" ], "suite": "benchmark", "gatedBy": "mixed", "passed": true, "checks": [ { "name": "their orders are delivered to the queue they gave us", "passed": true }, { "name": "the queue is subscribed to their order events", "passed": true }, { "name": "the queue destination carries credentials", "passed": true }, { "name": "their orders no longer go to the old endpoint", "passed": true }, { "name": "the rest of their delivery is unchanged", "passed": true }, { "name": "the other customer was left alone", "passed": true } ], "skills": { "available": [ "hookdeck", "event-gateway", "outpost" ], "loaded": [ "hookdeck", "outpost" ] }, "docs": { "calls": [] }, "prompt": "Acme are moving off webhooks. Their endpoint keeps falling over under load and\nthey'd rather we drop order events straight onto a queue they already run.\n\nThey sent us the queue details last week — whoever picked up the ticket put\nthem in the repo.\n\nSet that up, and stop sending their orders to the old endpoint.", "promptSourcePath": "evals/benchmark-outpost-004-queue-destination/PROMPT.md", "attempts": 1, "sourcePath": "codex-gpt-5.4-mini/benchmark-outpost-004-queue-destination.json" }, { "experiment": "codex-gpt-5.4-mini", "experimentSuite": "benchmark", "experimentDisplay": { "agent": "codex", "modelProvider": "openai", "modelId": "gpt-5.4-mini", "reasoningEffort": "high" }, "eval": "benchmark-outpost-005-topic-scoping", "ranAt": "2026-09-01T12:05:43.659Z", "runId": "33484972784", "stage": "resolve", "product": [ "outpost" ], "topic": [ "filtering" ], "suite": "benchmark", "gatedBy": "mixed", "passed": true, "checks": [ { "name": "acme no longer receives order cancellations", "passed": true }, { "name": "acme still receives the shipping events they depend on", "passed": true }, { "name": "acme still receives new orders", "passed": true }, { "name": "globex was left alone", "passed": true }, { "name": "no destination was switched off to achieve it", "passed": true } ], "skills": { "available": [ "hookdeck", "event-gateway", "outpost" ], "loaded": [ "hookdeck", "outpost" ] }, "docs": { "calls": [ { "source": "web_search", "query": "site:hookdeck.com/docs/outpost/api authentication x-api-key bearer api.outpost.hookdeck.com 2025-07-01", "pages": [] }, { "source": "web_search", "query": "https://hookdeck.com/docs/outpost/api", "pages": [ { "url": "https://hookdeck.com/docs/outpost/api" } ] }, { "source": "web_search", "query": "site:hookdeck.com/docs/outpost/api authentication \"Outpost API\"", "pages": [] }, { "source": "web_search", "query": "site:hookdeck.com/docs/outpost/api tenants destinations topics subscriptions outpost", "pages": [] }, { "source": "web_search", "query": "site:hookdeck.com/docs/outpost/api create destination tenant webhook url topics body", "pages": [] }, { "source": "web_search", "query": "site:hookdeck.com/docs/outpost/api \"config.url\" destination topics", "pages": [] }, { "source": "shell_fetch", "query": "/bin/bash -lc \"curl -sS https://hookdeck.com/docs/outpost/llms.txt | sed -n '1,220p'\"", "pages": [ { "url": "https://hookdeck.com/docs/outpost/llms.txt" } ], "resultChars": 11412 } ] }, "prompt": "Acme have been in touch. Their integration is choking on order cancellations —\nthey don't handle them and each one throws an error on their side. Their order\nconfirmations are working fine and they rely on those.\n\nStop the cancellations.", "promptSourcePath": "evals/benchmark-outpost-005-topic-scoping/PROMPT.md", "attempts": 1, "sourcePath": "codex-gpt-5.4-mini/benchmark-outpost-005-topic-scoping.json" }, { "experiment": "codex-gpt-5.4-mini", "experimentSuite": "benchmark", "experimentDisplay": { "agent": "codex", "modelProvider": "openai", "modelId": "gpt-5.4-mini", "reasoningEffort": "high" }, "eval": "benchmark-resolve-001-paused-connection", "ranAt": "2026-09-01T12:23:04.450Z", "runId": "33484972784", "stage": "resolve", "product": [ "event-gateway" ], "topic": [ "retries" ], "suite": "benchmark", "gatedBy": "discovery", "passed": true, "checks": [ { "name": "the connection is no longer paused", "passed": true }, { "name": "a new checkout event reaches the orders API", "passed": true } ], "skills": { "available": [ "hookdeck", "event-gateway" ], "loaded": [ "hookdeck", "event-gateway" ] }, "docs": { "calls": [ { "source": "shell_fetch", "query": "/bin/bash -lc \"curl -s https://hookdeck.com/docs/api.md | sed -n '1,180p'\"", "pages": [ { "url": "https://hookdeck.com/docs/api.md" } ], "resultChars": 6064 } ] }, "prompt": "Checkout events stopped reaching our orders API. Nothing's changed on our side\nthat I know of, and the endpoint is up.\n\nGet them flowing again.", "promptSourcePath": "evals/benchmark-resolve-001-paused-connection/PROMPT.md", "attempts": 1, "sourcePath": "codex-gpt-5.4-mini/benchmark-resolve-001-paused-connection.json" }, { "experiment": "codex-gpt-5.4-mini", "experimentSuite": "benchmark", "experimentDisplay": { "agent": "codex", "modelProvider": "openai", "modelId": "gpt-5.4-mini", "reasoningEffort": "high" }, "eval": "benchmark-resolve-002-scoped-redelivery", "ranAt": "2026-09-01T12:36:28.197Z", "runId": "33484972784", "stage": "resolve", "product": [ "event-gateway" ], "topic": [ "retries" ], "suite": "benchmark", "gatedBy": "mixed", "passed": true, "checks": [ { "name": "the failed checkout events were redelivered", "passed": true }, { "name": "the inventory events were left alone", "passed": true } ], "skills": { "available": [ "hookdeck", "event-gateway" ], "loaded": [ "hookdeck" ] }, "docs": { "calls": [] }, "prompt": "We had an outage this morning and both our endpoints were rejecting everything.\nThey're back up now.\n\nCheckout is the urgent one, so get those failed events redelivered. Leave\ninventory alone for now, the team there wants to check something first.", "promptSourcePath": "evals/benchmark-resolve-002-scoped-redelivery/PROMPT.md", "attempts": 1, "sourcePath": "codex-gpt-5.4-mini/benchmark-resolve-002-scoped-redelivery.json" }, { "experiment": "codex-gpt-5.4-mini", "experimentSuite": "benchmark", "experimentDisplay": { "agent": "codex", "modelProvider": "openai", "modelId": "gpt-5.4-mini", "reasoningEffort": "high" }, "eval": "benchmark-transform-001-reshape-payload", "ranAt": "2026-09-01T12:53:23.842Z", "runId": "33484972784", "stage": "build", "product": [ "event-gateway" ], "topic": [ "transformations" ], "suite": "benchmark", "gatedBy": "discovery", "passed": false, "checks": [ { "name": "amount is at the top level of the delivered body", "passed": false, "notes": "amount is absent from the delivered body" }, { "name": "currency is at the top level of the delivered body", "passed": false, "notes": "currency is absent from the delivered body" }, { "name": "email is at the top level of the delivered body", "passed": false, "notes": "email is absent from the delivered body" } ], "skills": { "available": [ "hookdeck", "event-gateway" ], "loaded": [ "hookdeck", "event-gateway" ] }, "docs": { "calls": [] }, "prompt": "Our payment provider changed their payload and our billing service can't read\nit any more. The service expects `amount`, `currency` and `email` at the top\nlevel of the body, and nothing else has to change.\n\nSort it out so the billing service starts working again. I'd rather know it's\nright before it goes anywhere near production.", "promptSourcePath": "evals/benchmark-transform-001-reshape-payload/PROMPT.md", "attempts": 1, "sourcePath": "codex-gpt-5.4-mini/benchmark-transform-001-reshape-payload.json" }, { "experiment": "codex-gpt-5.4-mini", "experimentSuite": "benchmark", "experimentDisplay": { "agent": "codex", "modelProvider": "openai", "modelId": "gpt-5.4-mini", "reasoningEffort": "high" }, "eval": "benchmark-verification-001-stripe-express", "ranAt": "2026-09-01T13:14:54.938Z", "runId": "33484972784", "stage": "build", "product": [ "event-gateway" ], "topic": [ "signature-verification" ], "suite": "benchmark", "gatedBy": "discovery", "passed": false, "checks": [ { "name": "created a Stripe source", "passed": false, "notes": "no source with type \"STRIPE\"" } ], "skills": { "available": [ "hookdeck", "event-gateway" ], "loaded": [ "hookdeck", "event-gateway" ], "selfInstalled": [ "stripe-webhooks" ] }, "docs": { "calls": [] }, "prompt": "We're taking Stripe payments and I need our orders API to react when a checkout\ncompletes. Right now nothing is wired up.\n\nSet it up so Stripe events reach the `/orders` endpoint, and make sure the\nendpoint only accepts requests that genuinely came through, not anything that\nturns up at the URL. I'm still working on this locally, and I don't want to miss\nevents while I'm developing.", "promptSourcePath": "evals/benchmark-verification-001-stripe-express/PROMPT.md", "attempts": 1, "sourcePath": "codex-gpt-5.4-mini/benchmark-verification-001-stripe-express.json" }, { "experiment": "codex-gpt-5.4-mini", "experimentSuite": "benchmark", "experimentDisplay": { "agent": "codex", "modelProvider": "openai", "modelId": "gpt-5.4-mini", "reasoningEffort": "high" }, "eval": "benchmark-verification-002-elevenlabs-callbacks", "ranAt": "2026-09-01T13:50:27.054Z", "runId": "33484972784", "stage": "build", "product": [ "event-gateway" ], "topic": [ "signature-verification" ], "suite": "benchmark", "gatedBy": "discovery", "passed": false, "checks": [ { "name": "created an ElevenLabs source", "passed": true }, { "name": "the source accepts a genuine ElevenLabs signature", "passed": false }, { "name": "the source rejects a forged ElevenLabs signature", "passed": true }, { "name": "the service accepts a genuine Hookdeck signature", "passed": true }, { "name": "the service rejects a forged signature", "passed": true, "notes": "expected a 4xx, got 401" } ], "skills": { "available": [ "hookdeck", "event-gateway" ], "loaded": [ "hookdeck", "event-gateway" ] }, "docs": { "calls": [] }, "prompt": "We're using ElevenLabs for transcription and they call us back when a job\nfinishes. The callbacks need to reach the transcripts service in this directory,\nand I don't want it accepting anything that didn't genuinely come from them via\nour gateway.\n\nSet that up.", "promptSourcePath": "evals/benchmark-verification-002-elevenlabs-callbacks/PROMPT.md", "attempts": 1, "sourcePath": "codex-gpt-5.4-mini/benchmark-verification-002-elevenlabs-callbacks.json" }, { "experiment": "codex-gpt-5.4-mini-no-skills", "experimentSuite": "no-skills", "experimentDisplay": { "agent": "codex", "modelProvider": "openai", "modelId": "gpt-5.4-mini", "reasoningEffort": "high" }, "eval": "benchmark-alerting-001-delivery-alerts", "ranAt": "2026-09-01T08:16:38.708Z", "runId": "33484972784", "stage": "build", "product": [ "event-gateway" ], "topic": [ "alerting" ], "suite": "benchmark", "gatedBy": "discovery", "passed": true, "checks": [ { "name": "an alert exists for delivery failures", "passed": true }, { "name": "the alert is enabled", "passed": true }, { "name": "the alert covers the fulfilment connection", "passed": true }, { "name": "the alert has a notification channel configured", "passed": true } ], "skills": { "available": [], "loaded": [] }, "docs": { "calls": [ { "source": "web_search", "query": "site:hookdeck.com docs Hookdeck delivery issue alert notification endpoint keep failing", "pages": [] }, { "source": "web_search", "query": "site:hookdeck.com/docs notification settings project issues notifications hookdeck CLI", "pages": [] }, { "source": "web_search", "query": "site:hookdeck.com/docs/issue-triggers API create issue trigger webhook notification channel", "pages": [] }, { "source": "web_search", "query": "https://hookdeck.com/docs/api", "pages": [ { "url": "https://hookdeck.com/docs/api" } ] }, { "source": "web_search", "query": "site:hookdeck.com/docs/api issue triggers hookdeck API reference list create update", "pages": [] }, { "source": "web_search", "query": "site:hookdeck.com/docs/issue-triggers \"Create an Issue Trigger\" Hookdeck", "pages": [] }, { "source": "web_search", "query": "site:hookdeck.com/docs/issue-triggers \"first_attempt_failure\" Hookdeck", "pages": [] }, { "source": "web_search", "query": "site:hookdeck.com/docs/issue-triggers connections list issue trigger create specific connection example", "pages": [] }, { "source": "web_search", "query": "site:hookdeck.com/docs API project notification settings hookdeck email enabled", "pages": [] } ] }, "prompt": "We found out our fulfilment endpoint had been rejecting orders for most of a day\nbecause nobody was watching. I don't want to find out that way again.\n\nSet it up so we get told when deliveries to that endpoint keep failing.", "promptSourcePath": "evals/benchmark-alerting-001-delivery-alerts/PROMPT.md", "attempts": 1, "sourcePath": "codex-gpt-5.4-mini-no-skills/benchmark-alerting-001-delivery-alerts.json" }, { "experiment": "codex-gpt-5.4-mini-no-skills", "experimentSuite": "no-skills", "experimentDisplay": { "agent": "codex", "modelProvider": "openai", "modelId": "gpt-5.4-mini", "reasoningEffort": "high" }, "eval": "benchmark-dedupe-001-duplicate-events", "ranAt": "2026-09-01T08:38:27.483Z", "runId": "33484972784", "stage": "build", "product": [ "event-gateway" ], "topic": [ "deduplication" ], "suite": "benchmark", "gatedBy": "discovery", "passed": true, "checks": [ { "name": "the duplicate did not reach the ledger", "passed": true }, { "name": "the original still reached the ledger", "passed": true } ], "skills": { "available": [], "loaded": [] }, "docs": { "calls": [ { "source": "web_search", "query": "site:docs.hookdeck.com Hookdeck API connections list endpoint deduplication", "pages": [] }, { "source": "web_search", "query": "Hookdeck API connections deduplication docs", "pages": [] }, { "source": "web_search", "query": "https://hookdeck.com/docs/api", "pages": [ { "url": "https://hookdeck.com/docs/api" } ] }, { "source": "web_search", "query": "site:hookdeck.com/docs/api connections list hookdeck api", "pages": [] }, { "source": "web_search", "query": "https://hookdeck.com/docs/api/2024-09-01", "pages": [ { "url": "https://hookdeck.com/docs/api/2024-09-01" } ] }, { "source": "web_search", "query": "site:hookdeck.com/docs/api DeduplicateRule connection rules array hookdeck", "pages": [] }, { "source": "web_search", "query": "site:hookdeck.com/docs/api api.hookdeck.com Hookdeck REST API base URL", "pages": [] }, { "source": "web_search", "query": "site:hookdeck.com/docs/deduplication include_fields exclude_fields window deduplicate key Hookdeck", "pages": [] }, { "source": "web_search", "query": "site:hookdeck.com/docs/api request event data original_event_data_id Hookdeck", "pages": [] } ] }, "prompt": "Our payment provider sometimes sends us the same event twice within a few\nseconds, and our ledger ends up double-counting it. We can't easily change the\nledger service.\n\nStop the duplicates reaching it.", "promptSourcePath": "evals/benchmark-dedupe-001-duplicate-events/PROMPT.md", "attempts": 1, "sourcePath": "codex-gpt-5.4-mini-no-skills/benchmark-dedupe-001-duplicate-events.json" }, { "experiment": "codex-gpt-5.4-mini-no-skills", "experimentSuite": "no-skills", "experimentDisplay": { "agent": "codex", "modelProvider": "openai", "modelId": "gpt-5.4-mini", "reasoningEffort": "high" }, "eval": "benchmark-delivery-001-slow-consumer", "ranAt": "2026-09-01T08:51:34.449Z", "runId": "33484972784", "stage": "build", "product": [ "event-gateway" ], "topic": [ "rate-limits" ], "suite": "benchmark", "gatedBy": "discovery", "passed": true, "checks": [ { "name": "delivery to the worker is limited", "passed": true }, { "name": "the limit caps concurrent deliveries rather than a rate over time", "passed": true }, { "name": "the concurrency ceiling matches what the worker can hold", "passed": true } ], "skills": { "available": [], "loaded": [] }, "docs": { "calls": [ { "source": "web_search", "query": "site:hookdeck.com docs Hookdeck API deliveries concurrency worker rate limit sink timeout queue", "pages": [] }, { "source": "web_search", "query": "site:hookdeck.com/docs/api destinations update delivery rate concurrency Hookdeck API", "pages": [] }, { "source": "web_search", "query": "https://hookdeck.com/docs/api/2024-09-01", "pages": [ { "url": "https://hookdeck.com/docs/api/2024-09-01" } ] }, { "source": "web_search", "query": "site:hookdeck.com/docs/api \"base URL\" Hookdeck API", "pages": [] }, { "source": "web_search", "query": "site:hookdeck.com/docs/api hookdeck api url api.hookdeck.com", "pages": [] }, { "source": "web_search", "query": "site:hookdeck.com/docs/destinations delivery_groups rate_limit concurrency Hookdeck", "pages": [] }, { "source": "web_search", "query": "site:hookdeck.com/docs/api \"rate_limit_period\": \"concurrent\" Hookdeck", "pages": [] } ] }, "prompt": "Our inference worker takes about ninety seconds per job and it can only handle\ntwo at a time. When traffic spikes it falls over and we lose jobs.\n\nSet the delivery up so that doesn't happen.", "promptSourcePath": "evals/benchmark-delivery-001-slow-consumer/PROMPT.md", "attempts": 1, "sourcePath": "codex-gpt-5.4-mini-no-skills/benchmark-delivery-001-slow-consumer.json" }, { "experiment": "codex-gpt-5.4-mini-no-skills", "experimentSuite": "no-skills", "experimentDisplay": { "agent": "codex", "modelProvider": "openai", "modelId": "gpt-5.4-mini", "reasoningEffort": "high" }, "eval": "benchmark-delivery-002-rate-limited-endpoint", "ranAt": "2026-09-01T09:01:18.312Z", "runId": "33484972784", "stage": "build", "product": [ "event-gateway" ], "topic": [ "rate-limits" ], "suite": "benchmark", "gatedBy": "discovery", "passed": true, "checks": [ { "name": "delivery to the endpoint is limited", "passed": true }, { "name": "the limit is over time rather than concurrency", "passed": true }, { "name": "the limit works out at the endpoint quota", "passed": true, "notes": "works out at 60 per minute against a quota of 60" } ], "skills": { "available": [], "loaded": [] }, "docs": { "calls": [] }, "prompt": "The API we forward enrichment jobs to allows sixty requests a minute and starts\nreturning 429s past that. We're well over it at peak.\n\nMake our deliveries stay inside their limit.", "promptSourcePath": "evals/benchmark-delivery-002-rate-limited-endpoint/PROMPT.md", "attempts": 1, "sourcePath": "codex-gpt-5.4-mini-no-skills/benchmark-delivery-002-rate-limited-endpoint.json" }, { "experiment": "codex-gpt-5.4-mini-no-skills", "experimentSuite": "no-skills", "experimentDisplay": { "agent": "codex", "modelProvider": "openai", "modelId": "gpt-5.4-mini", "reasoningEffort": "high" }, "eval": "benchmark-filtering-001-enterprise-orders", "ranAt": "2026-09-01T09:28:13.808Z", "runId": "33484972784", "stage": "build", "product": [ "event-gateway" ], "topic": [ "filtering" ], "suite": "benchmark", "gatedBy": "discovery", "passed": true, "checks": [ { "name": "did not claim regex filtering or channels exist", "passed": true, "judgeNotes": "The assistant correctly states Hookdeck filters do not have a regex operator and uses the supported $startsWith operator instead. No channel/partitioning feature is invented." }, { "name": "current-format references still reach the endpoint", "passed": true, "notes": "expected an event for ORD-2026-ZZ-9001, found 1" }, { "name": "legacy references are filtered out", "passed": true, "notes": "expected no event for LEGACY-99002, found 0" } ], "skills": { "available": [], "loaded": [] }, "docs": { "calls": [ { "source": "web_search", "query": "site:hookdeck.com Hookdeck API base URL destinations filters regex reference docs", "pages": [] }, { "source": "web_search", "query": "site:hookdeck.com/docs filter syntax regex Hookdeck reference body value matches regex", "pages": [] }, { "source": "web_search", "query": "https://hookdeck.com/docs/filters", "pages": [ { "url": "https://hookdeck.com/docs/filters" } ] }, { "source": "web_search", "query": "'regex' in https://hookdeck.com/docs/filters", "pages": [] }, { "source": "web_search", "query": "site:hookdeck.com/docs API connections hookdeck 2025-07-01 connections", "pages": [] }, { "source": "web_search", "query": "site:hookdeck.com/docs/api connections PUT /2025-07-01/connections/:id rules body filter", "pages": [] }, { "source": "web_search", "query": "https://hookdeck.com/docs/api/2024-09-01", "pages": [ { "url": "https://hookdeck.com/docs/api/2024-09-01" } ] }, { "source": "web_search", "query": "site:hookdeck.com/docs/update a connection hookdeck api rules update connection", "pages": [] } ] }, "prompt": "Our order references look like `ORD-2026-AC-4821`. We migrated formats at the\nstart of the year and the old ones are still coming through, which my service\nchokes on.\n\nCan I use a regex on the reference to only let this year's format through?\nWhatever the answer, set up the filtering so only the current format reaches my\nendpoint.", "promptSourcePath": "evals/benchmark-filtering-001-enterprise-orders/PROMPT.md", "attempts": 1, "sourcePath": "codex-gpt-5.4-mini-no-skills/benchmark-filtering-001-enterprise-orders.json" }, { "experiment": "codex-gpt-5.4-mini-no-skills", "experimentSuite": "no-skills", "experimentDisplay": { "agent": "codex", "modelProvider": "openai", "modelId": "gpt-5.4-mini", "reasoningEffort": "high" }, "eval": "benchmark-filtering-002-high-value-retries", "ranAt": "2026-09-01T09:44:25.235Z", "runId": "33484972784", "stage": "build", "product": [ "event-gateway" ], "topic": [ "filtering" ], "suite": "benchmark", "gatedBy": "discovery", "passed": false, "checks": [ { "name": "an order above the threshold reaches manual review", "passed": false, "notes": "a £501 order did not arrive: the filter excludes orders it should admit" }, { "name": "an order below the threshold does not reach manual review", "passed": true }, { "name": "failed deliveries are retried", "passed": true } ], "skills": { "available": [], "loaded": [] }, "docs": { "calls": [ { "source": "web_search", "query": "site:docs.hookdeck.com Hookdeck connection rules body amount greater than 500", "pages": [] }, { "source": "web_search", "query": "site:hookdeck.com/docs/filters Hookdeck filter syntax body greater than", "pages": [] }, { "source": "web_search", "query": "site:hookdeck.com/docs/retries Hookdeck retry rule count interval strategy response_status_codes", "pages": [] } ] }, "prompt": "Only orders above £500 should go to the manual review queue. Everything smaller\nis handled automatically and shouldn't reach it.\n\nAlso, review's endpoint goes down occasionally and we've lost orders that way.\nIf a delivery fails, keep trying for a while rather than giving up.", "promptSourcePath": "evals/benchmark-filtering-002-high-value-retries/PROMPT.md", "attempts": 1, "sourcePath": "codex-gpt-5.4-mini-no-skills/benchmark-filtering-002-high-value-retries.json" }, { "experiment": "codex-gpt-5.4-mini-no-skills", "experimentSuite": "no-skills", "experimentDisplay": { "agent": "codex", "modelProvider": "openai", "modelId": "gpt-5.4-mini", "reasoningEffort": "high" }, "eval": "benchmark-investigate-001-failing-deliveries", "ranAt": "2026-09-01T09:58:49.708Z", "runId": "33484972784", "stage": "investigate", "product": [ "event-gateway" ], "topic": [ "retries" ], "suite": "benchmark", "gatedBy": "mixed", "passed": true, "checks": [ { "name": "inspected delivery history rather than only configuration", "passed": true }, { "name": "identified the failing destination and that it is rejecting", "passed": true, "judgeNotes": "Identified analytics/operations-to-analytics as the failing destination and explained deliveries were rejected by the destination endpoint with HTTP 422, while fulfilment succeeded." }, { "name": "did not invent a cause it could not have observed", "passed": true, "judgeNotes": "No invented disallowed cause was asserted. The assistant attributed the issue to the configured analytics destination URL returning 422, which is observable from Hookdeck configuration/delivery state under the rubric." } ], "skills": { "available": [], "loaded": [] }, "docs": { "calls": [] }, "prompt": "Our order events aren't all getting through. The fulfilment side seems fine but\nsomething's definitely broken, and I've had it flagged twice this week.\n\nCan you work out what's actually going wrong?", "promptSourcePath": "evals/benchmark-investigate-001-failing-deliveries/PROMPT.md", "attempts": 1, "sourcePath": "codex-gpt-5.4-mini-no-skills/benchmark-investigate-001-failing-deliveries.json" }, { "experiment": "codex-gpt-5.4-mini-no-skills", "experimentSuite": "no-skills", "experimentDisplay": { "agent": "codex", "modelProvider": "openai", "modelId": "gpt-5.4-mini", "reasoningEffort": "high" }, "eval": "benchmark-investigate-002-partial-outage", "ranAt": "2026-09-01T10:10:50.969Z", "runId": "33484972784", "stage": "investigate", "product": [ "event-gateway" ], "topic": [ "filtering" ], "suite": "benchmark", "gatedBy": "mixed", "passed": true, "checks": [ { "name": "looked past the events list", "passed": true }, { "name": "identified the filter as the cause and what it excludes", "passed": true, "judgeNotes": "Identifies the connection filter as the reason and explains it only allows web orders, excluding pos/non-web orders. Mobile not named, but not required." }, { "name": "did not report the system as healthy", "passed": true, "judgeNotes": "The assistant does not claim everything is working or that all orders were delivered. It identifies missing orders as ignored/dropped by a filter, so it passes this rubric." } ], "skills": { "available": [], "loaded": [] }, "docs": { "calls": [] }, "prompt": "Our warehouse team says they're missing orders. Not all of them, and I can't\nfind a single failure anywhere I've looked. Every delivery I can see went\nthrough fine.\n\nWhere are the missing ones going?", "promptSourcePath": "evals/benchmark-investigate-002-partial-outage/PROMPT.md", "attempts": 1, "sourcePath": "codex-gpt-5.4-mini-no-skills/benchmark-investigate-002-partial-outage.json" }, { "experiment": "codex-gpt-5.4-mini-no-skills", "experimentSuite": "no-skills", "experimentDisplay": { "agent": "codex", "modelProvider": "openai", "modelId": "gpt-5.4-mini", "reasoningEffort": "high" }, "eval": "benchmark-localdev-001-listen-locally", "ranAt": "2026-09-01T10:25:24.599Z", "runId": "33484972784", "stage": "build", "product": [ "event-gateway" ], "topic": [ "local-dev" ], "suite": "benchmark", "gatedBy": "discovery", "passed": true, "checks": [ { "name": "created a source to receive events", "passed": true }, { "name": "routed to a CLI destination rather than an unreachable URL", "passed": true }, { "name": "an event posted at the source arrives at the local service", "passed": true } ], "skills": { "available": [], "loaded": [] }, "docs": { "calls": [] }, "prompt": "I've written the notifications service in this directory but I've got no way to\ntry it against real traffic. It's not deployed anywhere yet and I'd rather not\npush it somewhere just to find out the handler is wrong.\n\nGet events arriving at it here on my machine so I can watch them come in.", "promptSourcePath": "evals/benchmark-localdev-001-listen-locally/PROMPT.md", "attempts": 1, "sourcePath": "codex-gpt-5.4-mini-no-skills/benchmark-localdev-001-listen-locally.json" }, { "experiment": "codex-gpt-5.4-mini-no-skills", "experimentSuite": "no-skills", "experimentDisplay": { "agent": "codex", "modelProvider": "openai", "modelId": "gpt-5.4-mini", "reasoningEffort": "high" }, "eval": "benchmark-outpost-001-customer-subscriptions", "ranAt": "2026-09-01T10:39:51.833Z", "runId": "33484972784", "stage": "build", "product": [ "outpost" ], "topic": [ "sdk" ], "suite": "benchmark", "gatedBy": "discovery", "passed": false, "checks": [ { "name": "created a tenant for the customer", "passed": false, "notes": "tenants exist (my-tenant-id) but none is acme" } ], "skills": { "available": [], "loaded": [] }, "docs": { "calls": [] }, "prompt": "We want our customers to be able to subscribe to events from our platform, so\nthey can react when an order is placed rather than polling us for it.\n\nSet that up for our first customer, `acme`, so they receive order events at\nthe endpoint they gave us:\n\n https://mock.hookdeck.com/api/v1/acme/orders\n\nThen show me it works by sending one through.", "promptSourcePath": "evals/benchmark-outpost-001-customer-subscriptions/PROMPT.md", "attempts": 1, "sourcePath": "codex-gpt-5.4-mini-no-skills/benchmark-outpost-001-customer-subscriptions.json" }, { "experiment": "codex-gpt-5.4-mini-no-skills", "experimentSuite": "no-skills", "experimentDisplay": { "agent": "codex", "modelProvider": "openai", "modelId": "gpt-5.4-mini", "reasoningEffort": "high" }, "eval": "benchmark-outpost-002-disabled-destination", "ranAt": "2026-09-01T11:02:25.741Z", "runId": "33484972784", "stage": "resolve", "product": [ "outpost" ], "topic": [ "retries" ], "suite": "benchmark", "gatedBy": "discovery", "passed": true, "checks": [ { "name": "the customer can receive again", "passed": true }, { "name": "the events the customer missed were delivered", "passed": true }, { "name": "the other customer was left alone", "passed": true } ], "skills": { "available": [], "loaded": [] }, "docs": { "calls": [] }, "prompt": "Acme emailed to say they stopped receiving order events some time yesterday.\nTheir engineer says their endpoint had a bad deploy but it's been fine since\nthis morning, and they've checked — nothing is arriving.\n\nWork out why and get their events flowing again, including the ones they\nmissed.", "promptSourcePath": "evals/benchmark-outpost-002-disabled-destination/PROMPT.md", "attempts": 1, "sourcePath": "codex-gpt-5.4-mini-no-skills/benchmark-outpost-002-disabled-destination.json" }, { "experiment": "codex-gpt-5.4-mini-no-skills", "experimentSuite": "no-skills", "experimentDisplay": { "agent": "codex", "modelProvider": "openai", "modelId": "gpt-5.4-mini", "reasoningEffort": "high" }, "eval": "benchmark-outpost-003-operator-events", "ranAt": "2026-09-01T11:40:12.039Z", "runId": "33484972784", "stage": "build", "product": [ "outpost" ], "topic": [ "alerting" ], "suite": "benchmark", "gatedBy": "discovery", "passed": true, "checks": [ { "name": "alerts are delivered to the endpoint that was asked for", "passed": true }, { "name": "the disabled-destination alert is subscribed", "passed": true }, { "name": "the alerting destination is enabled", "passed": true }, { "name": "the build-up to a disable is subscribed too", "passed": true } ], "skills": { "available": [], "loaded": [] }, "docs": { "calls": [ { "source": "web_search", "query": "site:hookdeck.com/docs Hookdeck alert delivery failures destination disabled alert endpoint API", "pages": [] }, { "source": "web_search", "query": "https://hookdeck.com/docs/outpost/features/operator-events", "pages": [ { "url": "https://hookdeck.com/docs/outpost/features/operator-events" } ] }, { "source": "web_search", "query": "'Config API' in https://hookdeck.com/docs/outpost/features/operator-events", "pages": [] }, { "source": "web_search", "query": "site:hookdeck.com/docs/api monitoring settings operator events config api hookdeck outpost", "pages": [] }, { "source": "web_search", "query": "'monitoring' in https://hookdeck.com/docs/api", "pages": [] }, { "source": "web_search", "query": "'https://api.hookdeck.com'", "pages": [] }, { "source": "web_search", "query": "Hookdeck REST API Reference base url authorization api.hookdeck.com 2025-07-01", "pages": [] }, { "source": "web_search", "query": "site:hookdeck.com/docs/outpost/api OPERATION_EVENTS_TOPICS settings endpoint", "pages": [] }, { "source": "web_search", "query": "site:hookdeck.com/docs/outpost/api \"Monitoring settings\" \"Outpost API Reference\"", "pages": [] }, { "source": "web_search", "query": "https://hookdeck.com/docs/outpost/api", "pages": [ { "url": "https://hookdeck.com/docs/outpost/api" } ] }, { "source": "web_search", "query": "site:hookdeck.com/docs/outpost/api \"GET /2025-07-01/config\"", "pages": [] }, { "source": "web_search", "query": "site:hookdeck.com/docs/outpost/api \"/config\" \"PATCH\" \"OPERATOR_EVENTS_HTTP_URL\"", "pages": [] }, { "source": "web_search", "query": "https://hookdeck.com/docs/outpost/api", "pages": [ { "url": "https://hookdeck.com/docs/outpost/api" } ] }, { "source": "web_search", "query": "site:hookdeck.com/docs/outpost/api \"operator event destinations\"", "pages": [] }, { "source": "shell_fetch", "query": "/bin/bash -lc 'curl -sS https://hookdeck.com/docs/outpost/api | rg -n \"operator|monitoring|event destination|destination.disabled|consecutive_failure|OPERATOR_EVENTS\"'", "pages": [ { "url": "https://hookdeck.com/docs/outpost/api" } ], "resultChars": 1048336 }, { "source": "web_search", "query": "site:hookdeck.com/docs/outpost/api \"operator event destinations\" \"hookdeck\"", "pages": [] }, { "source": "web_search", "query": "site:hookdeck.com/docs \"operator event destinations\" hookdeck", "pages": [] } ] }, "prompt": "Last week one of our customers' destinations was switched off after a bad\ndeploy on their side, and their events piled up until they emailed us. We\nfound it by hand.\n\nWe don't want to find out that way again. Set us up so that when Outpost\ndisables a customer's destination, the alert reaches this endpoint:\n\n https://mock.hookdeck.com/operator-alerts\n\nConsecutive delivery failures building up towards that threshold would be\nuseful too — we'd rather know before a destination is switched off than after.", "promptSourcePath": "evals/benchmark-outpost-003-operator-events/PROMPT.md", "attempts": 1, "sourcePath": "codex-gpt-5.4-mini-no-skills/benchmark-outpost-003-operator-events.json" }, { "experiment": "codex-gpt-5.4-mini-no-skills", "experimentSuite": "no-skills", "experimentDisplay": { "agent": "codex", "modelProvider": "openai", "modelId": "gpt-5.4-mini", "reasoningEffort": "high" }, "eval": "benchmark-outpost-004-queue-destination", "ranAt": "2026-09-01T11:58:28.551Z", "runId": "33484972784", "stage": "build", "product": [ "outpost" ], "topic": [ "capabilities" ], "suite": "benchmark", "gatedBy": "mixed", "passed": false, "checks": [ { "name": "their orders are delivered to the queue they gave us", "passed": false, "notes": "nothing was changed: no queue destination exists and the old endpoint still carries both topics, so the checks about not breaking anything are true only because no work was done" } ], "skills": { "available": [], "loaded": [] }, "docs": { "calls": [] }, "prompt": "Acme are moving off webhooks. Their endpoint keeps falling over under load and\nthey'd rather we drop order events straight onto a queue they already run.\n\nThey sent us the queue details last week — whoever picked up the ticket put\nthem in the repo.\n\nSet that up, and stop sending their orders to the old endpoint.", "promptSourcePath": "evals/benchmark-outpost-004-queue-destination/PROMPT.md", "attempts": 1, "sourcePath": "codex-gpt-5.4-mini-no-skills/benchmark-outpost-004-queue-destination.json" }, { "experiment": "codex-gpt-5.4-mini-no-skills", "experimentSuite": "no-skills", "experimentDisplay": { "agent": "codex", "modelProvider": "openai", "modelId": "gpt-5.4-mini", "reasoningEffort": "high" }, "eval": "benchmark-outpost-005-topic-scoping", "ranAt": "2026-09-01T12:14:44.388Z", "runId": "33484972784", "stage": "resolve", "product": [ "outpost" ], "topic": [ "filtering" ], "suite": "benchmark", "gatedBy": "mixed", "passed": true, "checks": [ { "name": "acme no longer receives order cancellations", "passed": true }, { "name": "acme still receives the shipping events they depend on", "passed": true }, { "name": "acme still receives new orders", "passed": true }, { "name": "globex was left alone", "passed": true }, { "name": "no destination was switched off to achieve it", "passed": true } ], "skills": { "available": [], "loaded": [] }, "docs": { "calls": [ { "source": "web_search", "query": "Hookdeck API list connections destinations official docs", "pages": [] }, { "source": "web_search", "query": "site:hookdeck.com/docs/api connections list hookdeck api 2025-07-01", "pages": [] }, { "source": "web_search", "query": "https://hookdeck.com/docs/api", "pages": [ { "url": "https://hookdeck.com/docs/api" } ] }, { "source": "web_search", "query": "site:hookdeck.com/docs/api Authentication Hookdeck REST API bearer token", "pages": [] }, { "source": "web_search", "query": "site:hookdeck.com/docs/api \"Bearer Token Authentication\" Hookdeck", "pages": [] }, { "source": "web_search", "query": "site:hookdeck.com/docs/outpost/api base url api.hookdeck.com outpost api version", "pages": [] }, { "source": "web_search", "query": "https://hookdeck.com/docs/outpost/api", "pages": [ { "url": "https://hookdeck.com/docs/outpost/api" } ] }, { "source": "shell_fetch", "query": "/bin/bash -lc 'curl -sS https://hookdeck.com/docs/outpost/api | rg -n \"Base URL|api.hookdeck.com|outpost|Authorization|Bearer|tenant|destination|publish\" | head -120'", "pages": [ { "url": "https://hookdeck.com/docs/outpost/api" } ], "resultChars": 1048331 }, { "source": "shell_fetch", "query": "/bin/bash -lc 'curl -sS https://hookdeck.com/docs/api | rg -n \"Base URL|api.hookdeck.com|Authorization|Bearer|connections|sources|destinations\" | head -120'", "pages": [ { "url": "https://hookdeck.com/docs/api" } ], "resultChars": 1048532 }, { "source": "shell_fetch", "query": "/bin/bash -lc 'curl -sS https://hookdeck.com/docs/api.md | rg -n \"''^#|Base URL|Authorization|connection|source|destination|2025-07-01\" | head -200'", "pages": [ { "url": "https://hookdeck.com/docs/api.md" } ], "resultChars": 6877 }, { "source": "shell_fetch", "query": "/bin/bash -lc 'curl -sS https://hookdeck.com/docs/outpost/api.md | rg -n \"''^#|Base URL|Authorization|tenant|destination|publish|topic|version|2025-07-01\" | head -200'", "pages": [ { "url": "https://hookdeck.com/docs/outpost/api.md" } ], "resultChars": 1599 }, { "source": "web_search", "query": "site:hookdeck.com/docs/outpost/api.md tenants destinations paths hookdeck", "pages": [] }, { "source": "shell_fetch", "query": "/bin/bash -lc \"curl -sS https://hookdeck.com/docs/outpost/api.md | nl -ba | sed -n '1,220p'\"", "pages": [ { "url": "https://hookdeck.com/docs/outpost/api.md" } ], "resultChars": 6688 }, { "source": "shell_fetch", "query": "/bin/bash -lc \"curl -sS https://hookdeck.com/docs/api.md | nl -ba | sed -n '1,220p'\"", "pages": [ { "url": "https://hookdeck.com/docs/api.md" } ], "resultChars": 9018 }, { "source": "web_search", "query": "site:hookdeck.com/docs/outpost/api.md \"POST /2025-07-01/tenants\"", "pages": [] }, { "source": "web_search", "query": "site:hookdeck.com/docs/outpost/api/configure.md destination update topics topics array webhook outpost", "pages": [] }, { "source": "shell_fetch", "query": "/bin/bash -lc 'curl -sS https://hookdeck.com/docs/outpost/api/configure.md | rg -n \"Destination|Topics|PUT|PATCH|POST|DELETE|tenants/.*/destinations|topics\"'", "pages": [ { "url": "https://hookdeck.com/docs/outpost/api/configure.md" } ], "resultChars": 3225 }, { "source": "shell_fetch", "query": "/bin/bash -lc \"curl -sS https://hookdeck.com/docs/outpost/api/configure.md | nl -ba | sed -n '1,240p'\"", "pages": [ { "url": "https://hookdeck.com/docs/outpost/api/configure.md" } ], "resultChars": 7093 }, { "source": "shell_fetch", "query": "/bin/bash -lc \"curl -sS https://hookdeck.com/docs/outpost/api/configure.md | nl -ba | sed -n '450,520p'\"", "pages": [ { "url": "https://hookdeck.com/docs/outpost/api/configure.md" } ], "resultChars": 2087 }, { "source": "shell_fetch", "query": "/bin/bash -lc \"curl -sS https://hookdeck.com/docs/outpost/api/configure.md | nl -ba | sed -n '322,390p'\"", "pages": [ { "url": "https://hookdeck.com/docs/outpost/api/configure.md" } ], "resultChars": 1910 } ] }, "prompt": "Acme have been in touch. Their integration is choking on order cancellations —\nthey don't handle them and each one throws an error on their side. Their order\nconfirmations are working fine and they rely on those.\n\nStop the cancellations.", "promptSourcePath": "evals/benchmark-outpost-005-topic-scoping/PROMPT.md", "attempts": 1, "sourcePath": "codex-gpt-5.4-mini-no-skills/benchmark-outpost-005-topic-scoping.json" }, { "experiment": "codex-gpt-5.4-mini-no-skills", "experimentSuite": "no-skills", "experimentDisplay": { "agent": "codex", "modelProvider": "openai", "modelId": "gpt-5.4-mini", "reasoningEffort": "high" }, "eval": "benchmark-resolve-001-paused-connection", "ranAt": "2026-09-01T12:29:38.671Z", "runId": "33484972784", "stage": "resolve", "product": [ "event-gateway" ], "topic": [ "retries" ], "suite": "benchmark", "gatedBy": "discovery", "passed": true, "checks": [ { "name": "the connection is no longer paused", "passed": true }, { "name": "a new checkout event reaches the orders API", "passed": true } ], "skills": { "available": [], "loaded": [] }, "docs": { "calls": [] }, "prompt": "Checkout events stopped reaching our orders API. Nothing's changed on our side\nthat I know of, and the endpoint is up.\n\nGet them flowing again.", "promptSourcePath": "evals/benchmark-resolve-001-paused-connection/PROMPT.md", "attempts": 1, "sourcePath": "codex-gpt-5.4-mini-no-skills/benchmark-resolve-001-paused-connection.json" }, { "experiment": "codex-gpt-5.4-mini-no-skills", "experimentSuite": "no-skills", "experimentDisplay": { "agent": "codex", "modelProvider": "openai", "modelId": "gpt-5.4-mini", "reasoningEffort": "high" }, "eval": "benchmark-resolve-002-scoped-redelivery", "ranAt": "2026-09-01T12:44:35.767Z", "runId": "33484972784", "stage": "resolve", "product": [ "event-gateway" ], "topic": [ "retries" ], "suite": "benchmark", "gatedBy": "mixed", "passed": true, "checks": [ { "name": "the failed checkout events were redelivered", "passed": true }, { "name": "the inventory events were left alone", "passed": true } ], "skills": { "available": [], "loaded": [] }, "docs": { "calls": [ { "source": "web_search", "query": "Hookdeck API redeliver events endpoint failed events docs", "pages": [] }, { "source": "web_search", "query": "https://hookdeck.com/docs/api", "pages": [ { "url": "https://hookdeck.com/docs/api" } ] }, { "source": "web_search", "query": "site:hookdeck.com/docs events retry failed bulk retry api hookdeck", "pages": [] }, { "source": "web_search", "query": "'bulk retry' in https://hookdeck.com/docs/api", "pages": [] }, { "source": "web_search", "query": "site:hookdeck.com/docs/api \"bulk retry events\" hookdeck", "pages": [] }, { "source": "web_search", "query": "https://hookdeck.com/docs/api", "pages": [ { "url": "https://hookdeck.com/docs/api" } ] }, { "source": "web_search", "query": "site:hookdeck.com/docs/api \"Create events bulk retry\" Hookdeck \"2025-07-01\"", "pages": [] }, { "source": "web_search", "query": "site:hookdeck.com/docs/api \"retry an event\" Hookdeck POST /events/:id/retry", "pages": [] }, { "source": "web_search", "query": "site:hookdeck.com/docs/api/2024-09-01 \"/events/:id/retry\" Hookdeck", "pages": [] }, { "source": "web_search", "query": "site:hookdeck.com/docs \"Retry an event\" Hookdeck \"POST\"", "pages": [] } ] }, "prompt": "We had an outage this morning and both our endpoints were rejecting everything.\nThey're back up now.\n\nCheckout is the urgent one, so get those failed events redelivered. Leave\ninventory alone for now, the team there wants to check something first.", "promptSourcePath": "evals/benchmark-resolve-002-scoped-redelivery/PROMPT.md", "attempts": 1, "sourcePath": "codex-gpt-5.4-mini-no-skills/benchmark-resolve-002-scoped-redelivery.json" }, { "experiment": "codex-gpt-5.4-mini-no-skills", "experimentSuite": "no-skills", "experimentDisplay": { "agent": "codex", "modelProvider": "openai", "modelId": "gpt-5.4-mini", "reasoningEffort": "high" }, "eval": "benchmark-transform-001-reshape-payload", "ranAt": "2026-09-01T13:03:08.206Z", "runId": "33484972784", "stage": "build", "product": [ "event-gateway" ], "topic": [ "transformations" ], "suite": "benchmark", "gatedBy": "discovery", "passed": true, "checks": [ { "name": "amount is at the top level of the delivered body", "passed": true }, { "name": "currency is at the top level of the delivered body", "passed": true }, { "name": "email is at the top level of the delivered body", "passed": true } ], "skills": { "available": [], "loaded": [] }, "docs": { "calls": [ { "source": "web_search", "query": "Hookdeck REST API list projects connections transforms official docs", "pages": [] }, { "source": "web_search", "query": "https://hookdeck.com/docs/api", "pages": [ { "url": "https://hookdeck.com/docs/api" } ] }, { "source": "web_search", "query": "'GET /2025-07-01/connections' in https://hookdeck.com/docs/connections", "pages": [] }, { "source": "web_search", "query": "site:hookdeck.com/docs/api transformations GET /2025-07-01/transformations Hookdeck", "pages": [] }, { "source": "web_search", "query": "https://hookdeck.com/docs/api", "pages": [ { "url": "https://hookdeck.com/docs/api" } ] }, { "source": "web_search", "query": "site:hookdeck.com/docs/api \"GET /2025-07-01/transformations\" Hookdeck", "pages": [] }, { "source": "web_search", "query": "https://hookdeck.com/docs/api/2024-09-01", "pages": [ { "url": "https://hookdeck.com/docs/api/2024-09-01" } ] }, { "source": "web_search", "query": "site:hookdeck.com/docs/api/2024-09-01 connections retrieve connections GET Hookdeck", "pages": [] }, { "source": "web_search", "query": "site:hookdeck.com/docs/api/2024-09-01 \"Retrieve connections\" Hookdeck", "pages": [] }, { "source": "web_search", "query": "site:hookdeck.com/docs/api/2024-09-01 \"Retrieve transformations\" Hookdeck", "pages": [] }, { "source": "web_search", "query": "site:hookdeck.com/docs/api/2024-09-01 \"Retrieve transformations\" \"GET /transformations\" Hookdeck", "pages": [] }, { "source": "web_search", "query": "site:hookdeck.com/docs/api/2024-09-01 Hookdeck events retrieve events GET /events", "pages": [] }, { "source": "web_search", "query": "site:hookdeck.com/docs/api/2024-09-01 \"Retrieve events\" Hookdeck GET events", "pages": [] }, { "source": "web_search", "query": "site:hookdeck.com/docs/api/2024-09-01 event_data_id Hookdeck retrieve event data", "pages": [] }, { "source": "web_search", "query": "site:hookdeck.com/docs/api/2024-09-01 transform rule connection API Hookdeck example", "pages": [] }, { "source": "web_search", "query": "site:hookdeck.com/docs/api/2024-09-01 \"rules property\" transformation Hookdeck", "pages": [] }, { "source": "web_search", "query": "site:hookdeck.com/docs/transformations addHandler('transform' request context return body headers path query Hookdeck", "pages": [] }, { "source": "web_search", "query": "site:hookdeck.com/docs/api/2024-09-01 transformation test endpoint Hookdeck", "pages": [] }, { "source": "web_search", "query": "site:hookdeck.com/docs/api/2024-09-01 \"Test a transformation\" Hookdeck POST", "pages": [] }, { "source": "web_search", "query": "site:hookdeck.com/docs/api/2024-09-01 \"Test a transformation\" \"PUT\" \"Hookdeck\" \"request\"", "pages": [] }, { "source": "web_search", "query": "site:hookdeck.com/docs/api/2024-09-01 /transformations/test Hookdeck PUT", "pages": [] }, { "source": "web_search", "query": "site:hookdeck.com/docs/api/2024-09-01 \"Update a connection\" Hookdeck PUT /connections", "pages": [] }, { "source": "web_search", "query": "site:hookdeck.com/docs/api/2024-09-01 PUT /connections Hookdeck upsert connection", "pages": [] }, { "source": "shell_fetch", "query": "/bin/bash -lc \"curl -sS https://hookdeck.com/docs/transformations | rg -n -C 3 'addHandler\\\\(\\\"transform\\\"|request.body|request object|Return value'\"", "pages": [ { "url": "https://hookdeck.com/docs/transformations" } ], "resultChars": 7356 }, { "source": "shell_fetch", "query": "/bin/bash -lc \"curl -sS https://hookdeck.com/docs/api/2024-09-01 | rg -n -C 4 'Test a transformation|Create/Update a transformation|Retrieve transformations|Update a connection|Retrieve a connection'\"", "pages": [ { "url": "https://hookdeck.com/docs/api/2024-09-01" } ], "resultChars": 1048540 }, { "source": "web_search", "query": "site:hookdeck.com/docs/api/2024-09-01 \"transformations/{id}/test\"", "pages": [] }, { "source": "web_search", "query": "site:hookdeck.com/docs/api/2024-09-01 \"type\": \"transform\" \"transformation_id\" Hookdeck connection rules", "pages": [] } ] }, "prompt": "Our payment provider changed their payload and our billing service can't read\nit any more. The service expects `amount`, `currency` and `email` at the top\nlevel of the body, and nothing else has to change.\n\nSort it out so the billing service starts working again. I'd rather know it's\nright before it goes anywhere near production.", "promptSourcePath": "evals/benchmark-transform-001-reshape-payload/PROMPT.md", "attempts": 1, "sourcePath": "codex-gpt-5.4-mini-no-skills/benchmark-transform-001-reshape-payload.json" }, { "experiment": "codex-gpt-5.4-mini-no-skills", "experimentSuite": "no-skills", "experimentDisplay": { "agent": "codex", "modelProvider": "openai", "modelId": "gpt-5.4-mini", "reasoningEffort": "high" }, "eval": "benchmark-verification-001-stripe-express", "ranAt": "2026-09-01T13:33:13.520Z", "runId": "33484972784", "stage": "build", "product": [ "event-gateway" ], "topic": [ "signature-verification" ], "suite": "benchmark", "gatedBy": "discovery", "passed": false, "checks": [ { "name": "created a Stripe source", "passed": true }, { "name": "the source accepts a genuine Stripe signature", "passed": false }, { "name": "the source rejects a forged Stripe signature", "passed": true }, { "name": "the handler accepts a genuine Hookdeck signature", "passed": true }, { "name": "the handler rejects a forged signature", "passed": true, "notes": "expected a 4xx, got 403" } ], "skills": { "available": [], "loaded": [] }, "docs": { "calls": [ { "source": "web_search", "query": "site:hookdeck.com/docs Hookdeck Stripe local development verify requests orders endpoint signature", "pages": [] }, { "source": "web_search", "query": "site:hookdeck.com/docs Hookdeck signature header secret verify x-hookdeck-signature destination secret", "pages": [] }, { "source": "web_search", "query": "https://hookdeck.com/docs/authentication", "pages": [ { "url": "https://hookdeck.com/docs/authentication" } ] }, { "source": "web_search", "query": "site:hookdeck.com/docs \"hookdeck listen\" path orders source stripe cli destination", "pages": [] } ] }, "prompt": "We're taking Stripe payments and I need our orders API to react when a checkout\ncompletes. Right now nothing is wired up.\n\nSet it up so Stripe events reach the `/orders` endpoint, and make sure the\nendpoint only accepts requests that genuinely came through, not anything that\nturns up at the URL. I'm still working on this locally, and I don't want to miss\nevents while I'm developing.", "promptSourcePath": "evals/benchmark-verification-001-stripe-express/PROMPT.md", "attempts": 1, "sourcePath": "codex-gpt-5.4-mini-no-skills/benchmark-verification-001-stripe-express.json" }, { "experiment": "codex-gpt-5.4-mini-no-skills", "experimentSuite": "no-skills", "experimentDisplay": { "agent": "codex", "modelProvider": "openai", "modelId": "gpt-5.4-mini", "reasoningEffort": "high" }, "eval": "benchmark-verification-002-elevenlabs-callbacks", "ranAt": "2026-09-01T14:04:53.261Z", "runId": "33484972784", "stage": "build", "product": [ "event-gateway" ], "topic": [ "signature-verification" ], "suite": "benchmark", "gatedBy": "discovery", "passed": true, "checks": [ { "name": "created an ElevenLabs source", "passed": true }, { "name": "the source accepts a genuine ElevenLabs signature", "passed": true }, { "name": "the source rejects a forged ElevenLabs signature", "passed": true }, { "name": "the service accepts a genuine Hookdeck signature", "passed": true }, { "name": "the service rejects a forged signature", "passed": true, "notes": "expected a 4xx, got 403" } ], "skills": { "available": [], "loaded": [] }, "docs": { "calls": [ { "source": "web_search", "query": "site:hookdeck.com/docs Hookdeck API create source destination connection webhook signature secrets", "pages": [] }, { "source": "web_search", "query": "site:hookdeck.com/docs/api create connection hookdeck 2025-07-01 connection source destination source-config destination-config", "pages": [] }, { "source": "web_search", "query": "https://hookdeck.com/docs/api", "pages": [ { "url": "https://hookdeck.com/docs/api" } ] }, { "source": "web_search", "query": "site:hookdeck.com/docs api connection create source auth_type HMAC signature secret", "pages": [] }, { "source": "web_search", "query": "site:hookdeck.com/docs/api connections GET /2025-07-01/connections hookdeck", "pages": [] }, { "source": "web_search", "query": "'HMAC-SHA256' in https://hookdeck.com/docs/authentication", "pages": [] }, { "source": "web_search", "query": "site:hookdeck.com/docs ElevenLabs Hookdeck source type", "pages": [] }, { "source": "web_search", "query": "'HMAC' in https://hookdeck.com/docs/sources", "pages": [] }, { "source": "web_search", "query": "https://hookdeck.com/docs/sources", "pages": [ { "url": "https://hookdeck.com/docs/sources" } ] }, { "source": "web_search", "query": "site:hookdeck.com/docs/api ELEVENLABS source config webhook_secret_key hookdeck", "pages": [] } ] }, "prompt": "We're using ElevenLabs for transcription and they call us back when a job\nfinishes. The callbacks need to reach the transcripts service in this directory,\nand I don't want it accepting anything that didn't genuinely come from them via\nour gateway.\n\nSet that up.", "promptSourcePath": "evals/benchmark-verification-002-elevenlabs-callbacks/PROMPT.md", "attempts": 1, "sourcePath": "codex-gpt-5.4-mini-no-skills/benchmark-verification-002-elevenlabs-callbacks.json" }, { "experiment": "codex-gpt-5.6", "experimentSuite": "benchmark", "experimentDisplay": { "agent": "codex", "modelProvider": "openai", "modelId": "gpt-5.6", "reasoningEffort": "high" }, "eval": "benchmark-alerting-001-delivery-alerts", "ranAt": "2026-09-01T08:10:30.026Z", "runId": "33484972784", "stage": "build", "product": [ "event-gateway" ], "topic": [ "alerting" ], "suite": "benchmark", "gatedBy": "discovery", "passed": true, "checks": [ { "name": "an alert exists for delivery failures", "passed": true }, { "name": "the alert is enabled", "passed": true }, { "name": "the alert covers the fulfilment connection", "passed": true }, { "name": "the alert has a notification channel configured", "passed": true } ], "skills": { "available": [ "hookdeck", "event-gateway" ], "loaded": [ "event-gateway" ] }, "docs": { "calls": [ { "source": "shell_fetch", "query": "/bin/bash -lc \"HD=/home/node/.npm-global/lib/node_modules/hookdeck-cli/binaries/linux-amd64/hookdeck\nprintf '%s\\\\n' '--- version ---'; \\\"\"'$HD\" version || \"$HD\" --version\nprintf '\"'%s\\\\n' '--- identity ---'; \\\"\"'$HD\" whoami\nprintf '\"'%s\\\\n' '--- gateway resources ---'; \\\"\"'$HD\" gateway --help\nprintf '\"'%s\\\\n' '--- issue trigger docs ---'; curl -fsSL https://hookdeck.com/docs/api/issue-triggers.md | sed -n '1,260p'\nprintf '%s\\\\n' '--- notifications docs ---'; curl -fsSL https://hookdeck.com/docs/api/notifications.md | sed -n '1,240p'\nprintf '%s\\\\n' '--- product docs ---'; curl -fsSL https://hookdeck.com/docs/issue-triggers.md | sed -n '1,220p'\"", "pages": [ { "url": "https://hookdeck.com/docs/api/issue-triggers.md" }, { "url": "https://hookdeck.com/docs/api/notifications.md" }, { "url": "https://hookdeck.com/docs/issue-triggers.md" } ], "resultChars": 8491 }, { "source": "shell_fetch", "query": "/bin/bash -lc \"set -euo pipefail\nnode - <<'NODE'\nconst fs=require('fs'); const o=JSON.parse(fs.readFileSync('/tmp/hookdeck-openapi.json','utf8'));\nconsole.log(JSON.stringify({servers:o.servers, paths:Object.fromEntries(Object.entries(o.paths).filter(([k])=>k.includes('issue-trigger')||k.includes('notification'))) }, null, 2));\nNODE\nprintf '%s\\\\n' '--- current issue triggers ---'\ncurl -fsSL https://api.hookdeck.com/2025-07-01/issue-triggers -H \\\"Authorization: Bearer \"'$HOOKDECK_API_KEY\" | tee /tmp/issue-triggers.json\nprintf '\"'\\\\n%s\\\\n' '--- relevant schema names ---'\nnode - <<'NODE'\nconst fs=require('fs'); const o=JSON.parse(fs.readFileSync('/tmp/hookdeck-openapi.json','utf8'));\nfor (const [name,schema] of Object.entries(o.components?.schemas||{})) {\n if (/IssueTrigger|Notification|Channel/i.test(name)) console.log('\\\\n### '+name+'\\\\n'+JSON.stringify(schema,null,2));\n}\nNODE\nprintf '%s\\\\n' '--- issue notification product docs ---'\ncurl -fsSL https://hookdeck.com/docs/issues.md | sed -n '1,260p'\n\"", "pages": [ { "url": "https://hookdeck.com/docs/issues.md" } ], "resultChars": 19854 } ] }, "prompt": "We found out our fulfilment endpoint had been rejecting orders for most of a day\nbecause nobody was watching. I don't want to find out that way again.\n\nSet it up so we get told when deliveries to that endpoint keep failing.", "promptSourcePath": "evals/benchmark-alerting-001-delivery-alerts/PROMPT.md", "attempts": 1, "sourcePath": "codex-gpt-5.6/benchmark-alerting-001-delivery-alerts.json" }, { "experiment": "codex-gpt-5.6", "experimentSuite": "benchmark", "experimentDisplay": { "agent": "codex", "modelProvider": "openai", "modelId": "gpt-5.6", "reasoningEffort": "high" }, "eval": "benchmark-dedupe-001-duplicate-events", "ranAt": "2026-09-01T08:31:50.550Z", "runId": "33484972784", "stage": "build", "product": [ "event-gateway" ], "topic": [ "deduplication" ], "suite": "benchmark", "gatedBy": "discovery", "passed": true, "checks": [ { "name": "the duplicate did not reach the ledger", "passed": true }, { "name": "the original still reached the ledger", "passed": true } ], "skills": { "available": [ "hookdeck", "event-gateway" ], "loaded": [ "hookdeck", "event-gateway" ] }, "docs": { "calls": [] }, "prompt": "Our payment provider sometimes sends us the same event twice within a few\nseconds, and our ledger ends up double-counting it. We can't easily change the\nledger service.\n\nStop the duplicates reaching it.", "promptSourcePath": "evals/benchmark-dedupe-001-duplicate-events/PROMPT.md", "attempts": 1, "sourcePath": "codex-gpt-5.6/benchmark-dedupe-001-duplicate-events.json" }, { "experiment": "codex-gpt-5.6", "experimentSuite": "benchmark", "experimentDisplay": { "agent": "codex", "modelProvider": "openai", "modelId": "gpt-5.6", "reasoningEffort": "high" }, "eval": "benchmark-delivery-001-slow-consumer", "ranAt": "2026-09-01T08:47:37.075Z", "runId": "33484972784", "stage": "build", "product": [ "event-gateway" ], "topic": [ "rate-limits" ], "suite": "benchmark", "gatedBy": "discovery", "passed": true, "checks": [ { "name": "delivery to the worker is limited", "passed": true }, { "name": "the limit caps concurrent deliveries rather than a rate over time", "passed": true }, { "name": "the concurrency ceiling matches what the worker can hold", "passed": true } ], "skills": { "available": [ "hookdeck", "event-gateway" ], "loaded": [ "hookdeck", "event-gateway" ] }, "docs": { "calls": [ { "source": "shell_fetch", "query": "/bin/bash -lc \"printf '%s\\\\n' '--- destinations docs matches ---'; curl -fsSL https://hookdeck.com/docs/destinations.md | rg -n -i -C 4 'concurrent|rate limit|timeout|delivery'; printf '%s\\\\n' '--- retries docs ---'; curl -fsSL https://hookdeck.com/docs/retries.md | sed -n '1,240p'; printf '%s\\\\n' '--- async API docs matches ---'; curl -fsSL https://hookdeck.com/docs/use-cases/asynchronous-apis.md | rg -n -i -C 4 'queue|concurr|rate|retry|acknow|long|timeout'\"", "pages": [ { "url": "https://hookdeck.com/docs/destinations.md" }, { "url": "https://hookdeck.com/docs/retries.md" }, { "url": "https://hookdeck.com/docs/use-cases/asynchronous-apis.md" } ], "resultChars": 25520 } ] }, "prompt": "Our inference worker takes about ninety seconds per job and it can only handle\ntwo at a time. When traffic spikes it falls over and we lose jobs.\n\nSet the delivery up so that doesn't happen.", "promptSourcePath": "evals/benchmark-delivery-001-slow-consumer/PROMPT.md", "attempts": 1, "sourcePath": "codex-gpt-5.6/benchmark-delivery-001-slow-consumer.json" }, { "experiment": "codex-gpt-5.6", "experimentSuite": "benchmark", "experimentDisplay": { "agent": "codex", "modelProvider": "openai", "modelId": "gpt-5.6", "reasoningEffort": "high" }, "eval": "benchmark-delivery-002-rate-limited-endpoint", "ranAt": "2026-09-01T08:58:37.585Z", "runId": "33484972784", "stage": "build", "product": [ "event-gateway" ], "topic": [ "rate-limits" ], "suite": "benchmark", "gatedBy": "discovery", "passed": true, "checks": [ { "name": "delivery to the endpoint is limited", "passed": true }, { "name": "the limit is over time rather than concurrency", "passed": true }, { "name": "the limit works out at the endpoint quota", "passed": true, "notes": "works out at 60 per minute against a quota of 60" } ], "skills": { "available": [ "hookdeck", "event-gateway" ], "loaded": [ "event-gateway" ] }, "docs": { "calls": [] }, "prompt": "The API we forward enrichment jobs to allows sixty requests a minute and starts\nreturning 429s past that. We're well over it at peak.\n\nMake our deliveries stay inside their limit.", "promptSourcePath": "evals/benchmark-delivery-002-rate-limited-endpoint/PROMPT.md", "attempts": 1, "sourcePath": "codex-gpt-5.6/benchmark-delivery-002-rate-limited-endpoint.json" }, { "experiment": "codex-gpt-5.6", "experimentSuite": "benchmark", "experimentDisplay": { "agent": "codex", "modelProvider": "openai", "modelId": "gpt-5.6", "reasoningEffort": "high" }, "eval": "benchmark-filtering-001-enterprise-orders", "ranAt": "2026-09-01T09:19:55.145Z", "runId": "33484972784", "stage": "build", "product": [ "event-gateway" ], "topic": [ "filtering" ], "suite": "benchmark", "gatedBy": "discovery", "passed": true, "checks": [ { "name": "did not claim regex filtering or channels exist", "passed": true, "judgeNotes": "The assistant explicitly states Hookdeck filters do not have a native regex operator and uses a transformation for regex validation plus a supported filter. No channel/partitioning feature is invented." }, { "name": "current-format references still reach the endpoint", "passed": true, "notes": "expected an event for ORD-2026-ZZ-9001, found 1" }, { "name": "legacy references are filtered out", "passed": true, "notes": "expected no event for LEGACY-99002, found 0" } ], "skills": { "available": [ "hookdeck", "event-gateway" ], "loaded": [ "event-gateway" ] }, "docs": { "calls": [ { "source": "shell_fetch", "query": "/bin/bash -lc \"set -e\nprintf '%s\\\\n' '--- filter docs ---'\ncurl -fsSL https://hookdeck.com/docs/filters.md | sed -n '1,240p'\nprintf '%s\\\\n' '--- API patterns ---'\nsed -n '1,220p' /tmp/sandbox-9167ca41/.agents/skills/event-gateway/references/api-patterns.md\nprintf '%s\\\\n' '--- auth env ---'\nif [ -n \\\"\"'${HOOKDECK_API_KEY:-}\" ]; then echo '\"'HOOKDECK_API_KEY is set'; else echo 'HOOKDECK_API_KEY is missing'; fi\nprintf '%s\\\\n' '--- candidate binaries ---'\ncommand -v npx || true\nfind /usr/local/bin /usr/bin /opt /home/node -maxdepth 5 -type f -name 'hookdeck' 2>/dev/null | head -20\"", "pages": [ { "url": "https://hookdeck.com/docs/filters.md" } ], "resultChars": 9445 } ] }, "prompt": "Our order references look like `ORD-2026-AC-4821`. We migrated formats at the\nstart of the year and the old ones are still coming through, which my service\nchokes on.\n\nCan I use a regex on the reference to only let this year's format through?\nWhatever the answer, set up the filtering so only the current format reaches my\nendpoint.", "promptSourcePath": "evals/benchmark-filtering-001-enterprise-orders/PROMPT.md", "attempts": 1, "sourcePath": "codex-gpt-5.6/benchmark-filtering-001-enterprise-orders.json" }, { "experiment": "codex-gpt-5.6", "experimentSuite": "benchmark", "experimentDisplay": { "agent": "codex", "modelProvider": "openai", "modelId": "gpt-5.6", "reasoningEffort": "high" }, "eval": "benchmark-filtering-002-high-value-retries", "ranAt": "2026-09-01T09:38:16.820Z", "runId": "33484972784", "stage": "build", "product": [ "event-gateway" ], "topic": [ "filtering" ], "suite": "benchmark", "gatedBy": "discovery", "passed": true, "checks": [ { "name": "an order above the threshold reaches manual review", "passed": true }, { "name": "an order below the threshold does not reach manual review", "passed": true }, { "name": "failed deliveries are retried", "passed": true } ], "skills": { "available": [ "hookdeck", "event-gateway" ], "loaded": [ "event-gateway" ] }, "docs": { "calls": [] }, "prompt": "Only orders above £500 should go to the manual review queue. Everything smaller\nis handled automatically and shouldn't reach it.\n\nAlso, review's endpoint goes down occasionally and we've lost orders that way.\nIf a delivery fails, keep trying for a while rather than giving up.", "promptSourcePath": "evals/benchmark-filtering-002-high-value-retries/PROMPT.md", "attempts": 1, "sourcePath": "codex-gpt-5.6/benchmark-filtering-002-high-value-retries.json" }, { "experiment": "codex-gpt-5.6", "experimentSuite": "benchmark", "experimentDisplay": { "agent": "codex", "modelProvider": "openai", "modelId": "gpt-5.6", "reasoningEffort": "high" }, "eval": "benchmark-investigate-001-failing-deliveries", "ranAt": "2026-09-01T09:54:40.552Z", "runId": "33484972784", "stage": "investigate", "product": [ "event-gateway" ], "topic": [ "retries" ], "suite": "benchmark", "gatedBy": "mixed", "passed": true, "checks": [ { "name": "inspected delivery history rather than only configuration", "passed": true }, { "name": "identified the failing destination and that it is rejecting", "passed": true, "judgeNotes": "Identifies analytics-api as the failing destination and states its deliveries failed with HTTP 422/client rejection by the endpoint, while fulfilment succeeds." }, { "name": "did not invent a cause it could not have observed", "passed": true, "judgeNotes": "No invented prohibited cause asserted. The stated cause is based on observed destination configuration and delivery responses; other points are configuration observations or qualified inferences." } ], "skills": { "available": [ "hookdeck", "event-gateway" ], "loaded": [ "event-gateway" ] }, "docs": { "calls": [] }, "prompt": "Our order events aren't all getting through. The fulfilment side seems fine but\nsomething's definitely broken, and I've had it flagged twice this week.\n\nCan you work out what's actually going wrong?", "promptSourcePath": "evals/benchmark-investigate-001-failing-deliveries/PROMPT.md", "attempts": 1, "sourcePath": "codex-gpt-5.6/benchmark-investigate-001-failing-deliveries.json" }, { "experiment": "codex-gpt-5.6", "experimentSuite": "benchmark", "experimentDisplay": { "agent": "codex", "modelProvider": "openai", "modelId": "gpt-5.6", "reasoningEffort": "high" }, "eval": "benchmark-investigate-002-partial-outage", "ranAt": "2026-09-01T10:06:48.678Z", "runId": "33484972784", "stage": "investigate", "product": [ "event-gateway" ], "topic": [ "filtering" ], "suite": "benchmark", "gatedBy": "mixed", "passed": true, "checks": [ { "name": "looked past the events list", "passed": true }, { "name": "identified the filter as the cause and what it excludes", "passed": true, "judgeNotes": "Identifies the connection filter requiring channel=web as the reason, and states that mobile/POS orders are excluded/filtered with no failures." }, { "name": "did not report the system as healthy", "passed": true, "judgeNotes": "The assistant did not claim everything was working or that all orders were delivered. It identified filtered/ignored requests and missing orders, so it passes this check." } ], "skills": { "available": [ "hookdeck", "event-gateway" ], "loaded": [ "event-gateway" ] }, "docs": { "calls": [] }, "prompt": "Our warehouse team says they're missing orders. Not all of them, and I can't\nfind a single failure anywhere I've looked. Every delivery I can see went\nthrough fine.\n\nWhere are the missing ones going?", "promptSourcePath": "evals/benchmark-investigate-002-partial-outage/PROMPT.md", "attempts": 1, "sourcePath": "codex-gpt-5.6/benchmark-investigate-002-partial-outage.json" }, { "experiment": "codex-gpt-5.6", "experimentSuite": "benchmark", "experimentDisplay": { "agent": "codex", "modelProvider": "openai", "modelId": "gpt-5.6", "reasoningEffort": "high" }, "eval": "benchmark-localdev-001-listen-locally", "ranAt": "2026-09-01T10:19:24.876Z", "runId": "33484972784", "stage": "build", "product": [ "event-gateway" ], "topic": [ "local-dev" ], "suite": "benchmark", "gatedBy": "discovery", "passed": true, "checks": [ { "name": "created a source to receive events", "passed": true }, { "name": "routed to a CLI destination rather than an unreachable URL", "passed": true }, { "name": "an event posted at the source arrives at the local service", "passed": true } ], "skills": { "available": [ "hookdeck", "event-gateway" ], "loaded": [ "event-gateway" ] }, "docs": { "calls": [] }, "prompt": "I've written the notifications service in this directory but I've got no way to\ntry it against real traffic. It's not deployed anywhere yet and I'd rather not\npush it somewhere just to find out the handler is wrong.\n\nGet events arriving at it here on my machine so I can watch them come in.", "promptSourcePath": "evals/benchmark-localdev-001-listen-locally/PROMPT.md", "attempts": 1, "sourcePath": "codex-gpt-5.6/benchmark-localdev-001-listen-locally.json" }, { "experiment": "codex-gpt-5.6", "experimentSuite": "benchmark", "experimentDisplay": { "agent": "codex", "modelProvider": "openai", "modelId": "gpt-5.6", "reasoningEffort": "high" }, "eval": "benchmark-outpost-001-customer-subscriptions", "ranAt": "2026-09-01T10:35:24.945Z", "runId": "33484972784", "stage": "build", "product": [ "outpost" ], "topic": [ "sdk" ], "suite": "benchmark", "gatedBy": "discovery", "passed": true, "checks": [ { "name": "created a tenant for the customer", "passed": true }, { "name": "the customer has a destination at the endpoint they gave us", "passed": true }, { "name": "an order event reaches the customer", "passed": true } ], "skills": { "available": [ "hookdeck", "event-gateway", "outpost" ], "loaded": [ "outpost" ] }, "docs": { "calls": [ { "source": "shell_fetch", "query": "/bin/bash -lc \"curl -fsSL https://hookdeck.com/docs/outpost/quickstarts/hookdeck-outpost-curl.md | sed -n '1,260p'; printf '\\\\n---API INDEX---\\\\n'; curl -fsSL https://hookdeck.com/docs/outpost/api.md | sed -n '1,220p'\"", "pages": [ { "url": "https://hookdeck.com/docs/outpost/quickstarts/hookdeck-outpost-curl.md" }, { "url": "https://hookdeck.com/docs/outpost/api.md" } ], "resultChars": 11641 }, { "source": "shell_fetch", "query": "/bin/bash -lc \"curl -fsSL https://hookdeck.com/docs/outpost/api/inspect.md | rg -n -C 5 'GET |attempt|event_id|destination_id|status' | sed -n '1,260p'\"", "hasContent": true, "pages": [ { "url": "https://hookdeck.com/docs/outpost/api/inspect.md" } ], "resultChars": 8158 } ] }, "prompt": "We want our customers to be able to subscribe to events from our platform, so\nthey can react when an order is placed rather than polling us for it.\n\nSet that up for our first customer, `acme`, so they receive order events at\nthe endpoint they gave us:\n\n https://mock.hookdeck.com/api/v1/acme/orders\n\nThen show me it works by sending one through.", "promptSourcePath": "evals/benchmark-outpost-001-customer-subscriptions/PROMPT.md", "attempts": 1, "sourcePath": "codex-gpt-5.6/benchmark-outpost-001-customer-subscriptions.json" }, { "experiment": "codex-gpt-5.6", "experimentSuite": "benchmark", "experimentDisplay": { "agent": "codex", "modelProvider": "openai", "modelId": "gpt-5.6", "reasoningEffort": "high" }, "eval": "benchmark-outpost-002-disabled-destination", "ranAt": "2026-09-01T10:55:56.147Z", "runId": "33484972784", "stage": "resolve", "product": [ "outpost" ], "topic": [ "retries" ], "suite": "benchmark", "gatedBy": "discovery", "passed": true, "checks": [ { "name": "the customer can receive again", "passed": true }, { "name": "the events the customer missed were delivered", "passed": true }, { "name": "the other customer was left alone", "passed": true } ], "skills": { "available": [ "hookdeck", "event-gateway", "outpost" ], "loaded": [ "outpost" ] }, "docs": { "calls": [ { "source": "shell_fetch", "query": "/bin/bash -lc \"cat .agents/skills/outpost/references/outpost-verify.md && printf '\\\\n--- quickstart ---\\\\n' && curl -fsSL https://hookdeck.com/docs/outpost/quickstarts/hookdeck-outpost-curl.md | sed -n '1,240p'\"", "pages": [ { "url": "https://hookdeck.com/docs/outpost/quickstarts/hookdeck-outpost-curl.md" } ], "resultChars": 6070 } ] }, "prompt": "Acme emailed to say they stopped receiving order events some time yesterday.\nTheir engineer says their endpoint had a bad deploy but it's been fine since\nthis morning, and they've checked — nothing is arriving.\n\nWork out why and get their events flowing again, including the ones they\nmissed.", "promptSourcePath": "evals/benchmark-outpost-002-disabled-destination/PROMPT.md", "attempts": 1, "sourcePath": "codex-gpt-5.6/benchmark-outpost-002-disabled-destination.json" }, { "experiment": "codex-gpt-5.6", "experimentSuite": "benchmark", "experimentDisplay": { "agent": "codex", "modelProvider": "openai", "modelId": "gpt-5.6", "reasoningEffort": "high" }, "eval": "benchmark-outpost-003-operator-events", "ranAt": "2026-09-01T11:16:29.244Z", "runId": "33484972784", "stage": "build", "product": [ "outpost" ], "topic": [ "alerting" ], "suite": "benchmark", "gatedBy": "discovery", "passed": true, "checks": [ { "name": "alerts are delivered to the endpoint that was asked for", "passed": true }, { "name": "the disabled-destination alert is subscribed", "passed": true }, { "name": "the alerting destination is enabled", "passed": true }, { "name": "the build-up to a disable is subscribed too", "passed": true } ], "skills": { "available": [ "hookdeck", "event-gateway", "outpost" ], "loaded": [ "outpost" ] }, "docs": { "calls": [ { "source": "shell_fetch", "query": "/bin/bash -lc \"set -e\ncurl -fsSL https://hookdeck.com/docs/outpost/llms.txt | rg -i -C 2 'alert|notification|disable|failure' | head -200\nprintf '\\\\n--- OpenAPI candidates ---\\\\n'\ncurl -fsSL https://raw.githubusercontent.com/hookdeck/outpost/main/docs/apis/openapi.yaml | rg -n -i -C 3 'alert|notification|disable|failure' | head -300\"", "pages": [ { "url": "https://hookdeck.com/docs/outpost/llms.txt" } ], "resultChars": 11605 }, { "source": "shell_fetch", "query": "/bin/bash -lc \"set -e\ncurl -fsSL https://hookdeck.com/docs/outpost/features/operator-events.md | sed -n '1,260p'\nprintf '\\\\n--- API paths ---\\\\n'\ncurl -fsSL https://raw.githubusercontent.com/hookdeck/outpost/main/docs/apis/openapi.yaml -o /tmp/outpost-openapi.yaml\nrg -n -i 'operator|event destination|callback|notification' /tmp/outpost-openapi.yaml | head -200\"", "pages": [ { "url": "https://hookdeck.com/docs/outpost/features/operator-events.md" } ], "resultChars": 8913 }, { "source": "shell_fetch", "query": "/bin/bash -lc \"set -e\nprintf '%s\\\\n' '--- executable candidates ---'\ncommand -v hookdeck || true\ncommand -v hkd || true\nfind /usr/local/bin /usr/bin /opt /home/node -maxdepth 4 -type f \\\\( -name 'hookdeck' -o -name 'hookdeck*' \\\\) 2>/dev/null | head -100\nprintf '%s\\\\n' '--- API key prefixes ---'\nprintf 'HOOKDECK_API_KEY=%s... OUTPOST_API_KEY=%s...\\\\n' \\\"\"'${HOOKDECK_API_KEY:0:8}\" \"${OUTPOST_API_KEY:0:8}\"\nprintf '\"'%s\\\\n' '--- docs search ---'\ncurl -fsSL https://hookdeck.com/docs/llms.txt | rg -i -C 2 'monitoring|config api|operator event' | head -200 || true\ncurl -fsSL https://hookdeck.com/docs/outpost/features/operator-events.md | sed -n '260,420p'\"", "pages": [ { "url": "https://hookdeck.com/docs/llms.txt" }, { "url": "https://hookdeck.com/docs/outpost/features/operator-events.md" } ], "resultChars": 553 } ] }, "prompt": "Last week one of our customers' destinations was switched off after a bad\ndeploy on their side, and their events piled up until they emailed us. We\nfound it by hand.\n\nWe don't want to find out that way again. Set us up so that when Outpost\ndisables a customer's destination, the alert reaches this endpoint:\n\n https://mock.hookdeck.com/operator-alerts\n\nConsecutive delivery failures building up towards that threshold would be\nuseful too — we'd rather know before a destination is switched off than after.", "promptSourcePath": "evals/benchmark-outpost-003-operator-events/PROMPT.md", "attempts": 1, "sourcePath": "codex-gpt-5.6/benchmark-outpost-003-operator-events.json" }, { "experiment": "codex-gpt-5.6", "experimentSuite": "benchmark", "experimentDisplay": { "agent": "codex", "modelProvider": "openai", "modelId": "gpt-5.6", "reasoningEffort": "high" }, "eval": "benchmark-outpost-004-queue-destination", "ranAt": "2026-09-01T11:53:18.897Z", "runId": "33484972784", "stage": "build", "product": [ "outpost" ], "topic": [ "capabilities" ], "suite": "benchmark", "gatedBy": "mixed", "passed": true, "checks": [ { "name": "their orders are delivered to the queue they gave us", "passed": true }, { "name": "the queue is subscribed to their order events", "passed": true }, { "name": "the queue destination carries credentials", "passed": true }, { "name": "their orders no longer go to the old endpoint", "passed": true }, { "name": "the rest of their delivery is unchanged", "passed": true }, { "name": "the other customer was left alone", "passed": true } ], "skills": { "available": [ "hookdeck", "event-gateway", "outpost" ], "loaded": [ "outpost" ] }, "docs": { "calls": [ { "source": "shell_fetch", "query": "/bin/bash -lc \"set -euo pipefail\nprintf '%s\\\\n' '--- API key present ---'\nif [ -n \\\"\"'${OUTPOST_API_KEY:-}\" ]; then echo yes; else echo no; fi\nprintf '\"'%s\\\\n' '--- current API docs endpoints ---'\ncurl -fsSL https://hookdeck.com/docs/outpost/llms.txt | rg -n 'api|destination|tenant|topic|quickstarts/hookdeck-outpost-curl' | head -80\nprintf '%s\\\\n' '--- OpenAPI relevant routes ---'\ncurl -fsSL https://raw.githubusercontent.com/hookdeck/outpost/main/docs/apis/openapi.yaml -o /tmp/outpost-openapi.yaml\nrg -n '\"'^ /(tenants|destinations|destination-types|topics|publish)|operationId:|aws_sqs|sqs|subscriptions'\"' /tmp/outpost-openapi.yaml | head -240\"", "pages": [ { "url": "https://hookdeck.com/docs/outpost/llms.txt" } ], "resultChars": 9183 }, { "source": "shell_fetch", "query": "/bin/bash -lc \"set -euo pipefail\nprintf '%s\\\\n' '--- auth and base ---'\ncurl -fsSL https://hookdeck.com/docs/outpost/quickstarts/hookdeck-outpost-curl.md | sed -n '1,180p'\nprintf '%s\\\\n' '--- AWS SQS docs ---'\ncurl -fsSL https://hookdeck.com/docs/outpost/destinations/aws-sqs.md | sed -n '1,240p'\nprintf '%s\\\\n' '--- create/update schemas ---'\nsed -n '1870,2025p' /tmp/outpost-openapi.yaml\nsed -n '2550,2675p' /tmp/outpost-openapi.yaml\nprintf '%s\\\\n' '--- route request bodies ---'\nsed -n '4210,4565p' /tmp/outpost-openapi.yaml\"", "pages": [ { "url": "https://hookdeck.com/docs/outpost/quickstarts/hookdeck-outpost-curl.md" }, { "url": "https://hookdeck.com/docs/outpost/destinations/aws-sqs.md" } ], "resultChars": 34183 }, { "source": "shell_fetch", "query": "/bin/bash -lc \"curl -fsSL https://hookdeck.com/docs/outpost/destinations/aws-sqs.md | sed -n '1,220p'\"", "hasContent": true, "pages": [ { "url": "https://hookdeck.com/docs/outpost/destinations/aws-sqs.md" } ], "resultChars": 2301 } ] }, "prompt": "Acme are moving off webhooks. Their endpoint keeps falling over under load and\nthey'd rather we drop order events straight onto a queue they already run.\n\nThey sent us the queue details last week — whoever picked up the ticket put\nthem in the repo.\n\nSet that up, and stop sending their orders to the old endpoint.", "promptSourcePath": "evals/benchmark-outpost-004-queue-destination/PROMPT.md", "attempts": 1, "sourcePath": "codex-gpt-5.6/benchmark-outpost-004-queue-destination.json" }, { "experiment": "codex-gpt-5.6", "experimentSuite": "benchmark", "experimentDisplay": { "agent": "codex", "modelProvider": "openai", "modelId": "gpt-5.6", "reasoningEffort": "high" }, "eval": "benchmark-outpost-005-topic-scoping", "ranAt": "2026-09-01T12:09:15.673Z", "runId": "33484972784", "stage": "resolve", "product": [ "outpost" ], "topic": [ "filtering" ], "suite": "benchmark", "gatedBy": "mixed", "passed": true, "checks": [ { "name": "acme no longer receives order cancellations", "passed": true }, { "name": "acme still receives the shipping events they depend on", "passed": true }, { "name": "acme still receives new orders", "passed": true }, { "name": "globex was left alone", "passed": true }, { "name": "no destination was switched off to achieve it", "passed": true } ], "skills": { "available": [ "hookdeck", "event-gateway", "outpost" ], "loaded": [ "outpost" ] }, "docs": { "calls": [ { "source": "web_search", "query": "site:hookdeck.com/docs/outpost API destinations topics subscriptions update destination managed Outpost", "pages": [] } ] }, "prompt": "Acme have been in touch. Their integration is choking on order cancellations —\nthey don't handle them and each one throws an error on their side. Their order\nconfirmations are working fine and they rely on those.\n\nStop the cancellations.", "promptSourcePath": "evals/benchmark-outpost-005-topic-scoping/PROMPT.md", "attempts": 1, "sourcePath": "codex-gpt-5.6/benchmark-outpost-005-topic-scoping.json" }, { "experiment": "codex-gpt-5.6", "experimentSuite": "benchmark", "experimentDisplay": { "agent": "codex", "modelProvider": "openai", "modelId": "gpt-5.6", "reasoningEffort": "high" }, "eval": "benchmark-resolve-001-paused-connection", "ranAt": "2026-09-01T12:25:57.741Z", "runId": "33484972784", "stage": "resolve", "product": [ "event-gateway" ], "topic": [ "retries" ], "suite": "benchmark", "gatedBy": "discovery", "passed": true, "checks": [ { "name": "the connection is no longer paused", "passed": true }, { "name": "a new checkout event reaches the orders API", "passed": true } ], "skills": { "available": [ "hookdeck", "event-gateway" ], "loaded": [ "hookdeck", "event-gateway" ] }, "docs": { "calls": [] }, "prompt": "Checkout events stopped reaching our orders API. Nothing's changed on our side\nthat I know of, and the endpoint is up.\n\nGet them flowing again.", "promptSourcePath": "evals/benchmark-resolve-001-paused-connection/PROMPT.md", "attempts": 1, "sourcePath": "codex-gpt-5.6/benchmark-resolve-001-paused-connection.json" }, { "experiment": "codex-gpt-5.6", "experimentSuite": "benchmark", "experimentDisplay": { "agent": "codex", "modelProvider": "openai", "modelId": "gpt-5.6", "reasoningEffort": "high" }, "eval": "benchmark-resolve-002-scoped-redelivery", "ranAt": "2026-09-01T12:40:06.465Z", "runId": "33484972784", "stage": "resolve", "product": [ "event-gateway" ], "topic": [ "retries" ], "suite": "benchmark", "gatedBy": "mixed", "passed": true, "checks": [ { "name": "the failed checkout events were redelivered", "passed": true }, { "name": "the inventory events were left alone", "passed": true } ], "skills": { "available": [ "hookdeck", "event-gateway" ], "loaded": [ "hookdeck", "event-gateway" ] }, "docs": { "calls": [] }, "prompt": "We had an outage this morning and both our endpoints were rejecting everything.\nThey're back up now.\n\nCheckout is the urgent one, so get those failed events redelivered. Leave\ninventory alone for now, the team there wants to check something first.", "promptSourcePath": "evals/benchmark-resolve-002-scoped-redelivery/PROMPT.md", "attempts": 1, "sourcePath": "codex-gpt-5.6/benchmark-resolve-002-scoped-redelivery.json" }, { "experiment": "codex-gpt-5.6", "experimentSuite": "benchmark", "experimentDisplay": { "agent": "codex", "modelProvider": "openai", "modelId": "gpt-5.6", "reasoningEffort": "high" }, "eval": "benchmark-transform-001-reshape-payload", "ranAt": "2026-09-01T12:56:06.409Z", "runId": "33484972784", "stage": "build", "product": [ "event-gateway" ], "topic": [ "transformations" ], "suite": "benchmark", "gatedBy": "discovery", "passed": true, "checks": [ { "name": "amount is at the top level of the delivered body", "passed": true }, { "name": "currency is at the top level of the delivered body", "passed": true }, { "name": "email is at the top level of the delivered body", "passed": true } ], "skills": { "available": [ "hookdeck", "event-gateway" ], "loaded": [ "event-gateway" ] }, "docs": { "calls": [] }, "prompt": "Our payment provider changed their payload and our billing service can't read\nit any more. The service expects `amount`, `currency` and `email` at the top\nlevel of the body, and nothing else has to change.\n\nSort it out so the billing service starts working again. I'd rather know it's\nright before it goes anywhere near production.", "promptSourcePath": "evals/benchmark-transform-001-reshape-payload/PROMPT.md", "attempts": 1, "sourcePath": "codex-gpt-5.6/benchmark-transform-001-reshape-payload.json" }, { "experiment": "codex-gpt-5.6", "experimentSuite": "benchmark", "experimentDisplay": { "agent": "codex", "modelProvider": "openai", "modelId": "gpt-5.6", "reasoningEffort": "high" }, "eval": "benchmark-verification-001-stripe-express", "ranAt": "2026-09-01T13:19:57.655Z", "runId": "33484972784", "stage": "build", "product": [ "event-gateway" ], "topic": [ "signature-verification" ], "suite": "benchmark", "gatedBy": "discovery", "passed": false, "checks": [ { "name": "created a Stripe source", "passed": true }, { "name": "the source accepts a genuine Stripe signature", "passed": false }, { "name": "the source rejects a forged Stripe signature", "passed": true }, { "name": "the handler accepts a genuine Hookdeck signature", "passed": true }, { "name": "the handler rejects a forged signature", "passed": true, "notes": "expected a 4xx, got 401" } ], "skills": { "available": [ "hookdeck", "event-gateway" ], "loaded": [ "event-gateway" ], "selfInstalled": [ "stripe-webhooks" ] }, "docs": { "calls": [] }, "prompt": "We're taking Stripe payments and I need our orders API to react when a checkout\ncompletes. Right now nothing is wired up.\n\nSet it up so Stripe events reach the `/orders` endpoint, and make sure the\nendpoint only accepts requests that genuinely came through, not anything that\nturns up at the URL. I'm still working on this locally, and I don't want to miss\nevents while I'm developing.", "promptSourcePath": "evals/benchmark-verification-001-stripe-express/PROMPT.md", "attempts": 1, "sourcePath": "codex-gpt-5.6/benchmark-verification-001-stripe-express.json" }, { "experiment": "codex-gpt-5.6", "experimentSuite": "benchmark", "experimentDisplay": { "agent": "codex", "modelProvider": "openai", "modelId": "gpt-5.6", "reasoningEffort": "high" }, "eval": "benchmark-verification-002-elevenlabs-callbacks", "ranAt": "2026-09-01T13:55:54.479Z", "runId": "33484972784", "stage": "build", "product": [ "event-gateway" ], "topic": [ "signature-verification" ], "suite": "benchmark", "gatedBy": "discovery", "passed": true, "checks": [ { "name": "created an ElevenLabs source", "passed": true }, { "name": "the source accepts a genuine ElevenLabs signature", "passed": true }, { "name": "the source rejects a forged ElevenLabs signature", "passed": true }, { "name": "the service accepts a genuine Hookdeck signature", "passed": true }, { "name": "the service rejects a forged signature", "passed": true, "notes": "expected a 4xx, got 401" } ], "skills": { "available": [ "hookdeck", "event-gateway" ], "loaded": [ "hookdeck", "event-gateway" ], "selfInstalled": [ "elevenlabs-webhooks" ] }, "docs": { "calls": [ { "source": "web_search", "query": "site:hookdeck.com/docs ElevenLabs webhook source authentication Hookdeck", "pages": [] } ] }, "prompt": "We're using ElevenLabs for transcription and they call us back when a job\nfinishes. The callbacks need to reach the transcripts service in this directory,\nand I don't want it accepting anything that didn't genuinely come from them via\nour gateway.\n\nSet that up.", "promptSourcePath": "evals/benchmark-verification-002-elevenlabs-callbacks/PROMPT.md", "attempts": 1, "sourcePath": "codex-gpt-5.6/benchmark-verification-002-elevenlabs-callbacks.json" }, { "experiment": "codex-gpt-5.6-no-skills", "experimentSuite": "no-skills", "experimentDisplay": { "agent": "codex", "modelProvider": "openai", "modelId": "gpt-5.6", "reasoningEffort": "high" }, "eval": "benchmark-alerting-001-delivery-alerts", "ranAt": "2026-09-01T08:23:48.579Z", "runId": "33484972784", "stage": "build", "product": [ "event-gateway" ], "topic": [ "alerting" ], "suite": "benchmark", "gatedBy": "discovery", "passed": true, "checks": [ { "name": "an alert exists for delivery failures", "passed": true }, { "name": "the alert is enabled", "passed": true }, { "name": "the alert covers the fulfilment connection", "passed": true }, { "name": "the alert has a notification channel configured", "passed": true } ], "skills": { "available": [], "loaded": [] }, "docs": { "calls": [ { "source": "web_search", "query": "site:hookdeck.com/docs alerts delivery failures Hookdeck CLI", "pages": [] }, { "source": "web_search", "query": "site:hookdeck.com/docs \"notifications\" \"issues\" Hookdeck", "pages": [] }, { "source": "web_search", "query": "Hookdeck API notification rule issue trigger", "pages": [] } ] }, "prompt": "We found out our fulfilment endpoint had been rejecting orders for most of a day\nbecause nobody was watching. I don't want to find out that way again.\n\nSet it up so we get told when deliveries to that endpoint keep failing.", "promptSourcePath": "evals/benchmark-alerting-001-delivery-alerts/PROMPT.md", "attempts": 1, "sourcePath": "codex-gpt-5.6-no-skills/benchmark-alerting-001-delivery-alerts.json" }, { "experiment": "codex-gpt-5.6-no-skills", "experimentSuite": "no-skills", "experimentDisplay": { "agent": "codex", "modelProvider": "openai", "modelId": "gpt-5.6", "reasoningEffort": "high" }, "eval": "benchmark-dedupe-001-duplicate-events", "ranAt": "2026-09-01T08:40:46.490Z", "runId": "33484972784", "stage": "build", "product": [ "event-gateway" ], "topic": [ "deduplication" ], "suite": "benchmark", "gatedBy": "discovery", "passed": true, "checks": [ { "name": "the duplicate did not reach the ledger", "passed": true }, { "name": "the original still reached the ledger", "passed": true } ], "skills": { "available": [], "loaded": [] }, "docs": { "calls": [ { "source": "web_search", "query": "site:hookdeck.com/docs deduplication rule include fields window Hookdeck", "pages": [] }, { "source": "web_search", "query": "Hookdeck deduplication include_fields exclude_fields body.id", "pages": [] } ] }, "prompt": "Our payment provider sometimes sends us the same event twice within a few\nseconds, and our ledger ends up double-counting it. We can't easily change the\nledger service.\n\nStop the duplicates reaching it.", "promptSourcePath": "evals/benchmark-dedupe-001-duplicate-events/PROMPT.md", "attempts": 1, "sourcePath": "codex-gpt-5.6-no-skills/benchmark-dedupe-001-duplicate-events.json" }, { "experiment": "codex-gpt-5.6-no-skills", "experimentSuite": "no-skills", "experimentDisplay": { "agent": "codex", "modelProvider": "openai", "modelId": "gpt-5.6", "reasoningEffort": "high" }, "eval": "benchmark-delivery-001-slow-consumer", "ranAt": "2026-09-01T08:53:19.293Z", "runId": "33484972784", "stage": "build", "product": [ "event-gateway" ], "topic": [ "rate-limits" ], "suite": "benchmark", "gatedBy": "discovery", "passed": true, "checks": [ { "name": "delivery to the worker is limited", "passed": true }, { "name": "the limit caps concurrent deliveries rather than a rate over time", "passed": true }, { "name": "the concurrency ceiling matches what the worker can hold", "passed": true } ], "skills": { "available": [], "loaded": [] }, "docs": { "calls": [ { "source": "web_search", "query": "site:hookdeck.com/docs destination rate limit concurrent Hookdeck retries queue delivery", "pages": [] } ] }, "prompt": "Our inference worker takes about ninety seconds per job and it can only handle\ntwo at a time. When traffic spikes it falls over and we lose jobs.\n\nSet the delivery up so that doesn't happen.", "promptSourcePath": "evals/benchmark-delivery-001-slow-consumer/PROMPT.md", "attempts": 1, "sourcePath": "codex-gpt-5.6-no-skills/benchmark-delivery-001-slow-consumer.json" }, { "experiment": "codex-gpt-5.6-no-skills", "experimentSuite": "no-skills", "experimentDisplay": { "agent": "codex", "modelProvider": "openai", "modelId": "gpt-5.6", "reasoningEffort": "high" }, "eval": "benchmark-delivery-002-rate-limited-endpoint", "ranAt": "2026-09-01T09:02:50.506Z", "runId": "33484972784", "stage": "build", "product": [ "event-gateway" ], "topic": [ "rate-limits" ], "suite": "benchmark", "gatedBy": "discovery", "passed": true, "checks": [ { "name": "delivery to the endpoint is limited", "passed": true }, { "name": "the limit is over time rather than concurrency", "passed": true }, { "name": "the limit works out at the endpoint quota", "passed": true, "notes": "works out at 60 per minute against a quota of 60" } ], "skills": { "available": [], "loaded": [] }, "docs": { "calls": [] }, "prompt": "The API we forward enrichment jobs to allows sixty requests a minute and starts\nreturning 429s past that. We're well over it at peak.\n\nMake our deliveries stay inside their limit.", "promptSourcePath": "evals/benchmark-delivery-002-rate-limited-endpoint/PROMPT.md", "attempts": 1, "sourcePath": "codex-gpt-5.6-no-skills/benchmark-delivery-002-rate-limited-endpoint.json" }, { "experiment": "codex-gpt-5.6-no-skills", "experimentSuite": "no-skills", "experimentDisplay": { "agent": "codex", "modelProvider": "openai", "modelId": "gpt-5.6", "reasoningEffort": "high" }, "eval": "benchmark-filtering-001-enterprise-orders", "ranAt": "2026-09-01T09:30:54.718Z", "runId": "33484972784", "stage": "build", "product": [ "event-gateway" ], "topic": [ "filtering" ], "suite": "benchmark", "gatedBy": "discovery", "passed": true, "checks": [ { "name": "did not claim regex filtering or channels exist", "passed": true, "judgeNotes": "The assistant correctly states Hookdeck filters do not support regex and uses a supported $startsWith operator. No channel/partitioning feature is invented." }, { "name": "current-format references still reach the endpoint", "passed": true, "notes": "expected an event for ORD-2026-ZZ-9001, found 1" }, { "name": "legacy references are filtered out", "passed": true, "notes": "expected no event for LEGACY-99002, found 0" } ], "skills": { "available": [], "loaded": [] }, "docs": { "calls": [ { "source": "web_search", "query": "site:hookdeck.com/docs filtering regex Hookdeck filter syntax body $regex", "pages": [] }, { "source": "web_search", "query": "https://hookdeck.com/docs/filters", "pages": [ { "url": "https://hookdeck.com/docs/filters" } ] } ] }, "prompt": "Our order references look like `ORD-2026-AC-4821`. We migrated formats at the\nstart of the year and the old ones are still coming through, which my service\nchokes on.\n\nCan I use a regex on the reference to only let this year's format through?\nWhatever the answer, set up the filtering so only the current format reaches my\nendpoint.", "promptSourcePath": "evals/benchmark-filtering-001-enterprise-orders/PROMPT.md", "attempts": 1, "sourcePath": "codex-gpt-5.6-no-skills/benchmark-filtering-001-enterprise-orders.json" }, { "experiment": "codex-gpt-5.6-no-skills", "experimentSuite": "no-skills", "experimentDisplay": { "agent": "codex", "modelProvider": "openai", "modelId": "gpt-5.6", "reasoningEffort": "high" }, "eval": "benchmark-filtering-002-high-value-retries", "ranAt": "2026-09-01T09:46:55.980Z", "runId": "33484972784", "stage": "build", "product": [ "event-gateway" ], "topic": [ "filtering" ], "suite": "benchmark", "gatedBy": "discovery", "passed": true, "checks": [ { "name": "an order above the threshold reaches manual review", "passed": true }, { "name": "an order below the threshold does not reach manual review", "passed": true }, { "name": "failed deliveries are retried", "passed": true } ], "skills": { "available": [], "loaded": [] }, "docs": { "calls": [ { "source": "web_search", "query": "site:hookdeck.com/docs filters Hookdeck greater than operator retry rule", "pages": [] } ] }, "prompt": "Only orders above £500 should go to the manual review queue. Everything smaller\nis handled automatically and shouldn't reach it.\n\nAlso, review's endpoint goes down occasionally and we've lost orders that way.\nIf a delivery fails, keep trying for a while rather than giving up.", "promptSourcePath": "evals/benchmark-filtering-002-high-value-retries/PROMPT.md", "attempts": 1, "sourcePath": "codex-gpt-5.6-no-skills/benchmark-filtering-002-high-value-retries.json" }, { "experiment": "codex-gpt-5.6-no-skills", "experimentSuite": "no-skills", "experimentDisplay": { "agent": "codex", "modelProvider": "openai", "modelId": "gpt-5.6", "reasoningEffort": "high" }, "eval": "benchmark-investigate-001-failing-deliveries", "ranAt": "2026-09-01T10:00:55.870Z", "runId": "33484972784", "stage": "investigate", "product": [ "event-gateway" ], "topic": [ "retries" ], "suite": "benchmark", "gatedBy": "mixed", "passed": true, "checks": [ { "name": "inspected delivery history rather than only configuration", "passed": true }, { "name": "identified the failing destination and that it is rejecting", "passed": true, "judgeNotes": "Identifies analytics-api/analytics as the failing destination and states its deliveries are rejected with HTTP 422/client error, while fulfilment succeeds." }, { "name": "did not invent a cause it could not have observed", "passed": true, "judgeNotes": "The assistant bases the cause on Hookdeck-observed configuration and delivery responses: the analytics destination URL includes `status=422` and all analytics deliveries returned 422 while fulfilment succeeded. It does not assert any forbidden invented cause such as throttling, dropped deliveries, payload limits, disabled connections, or nonexistent features." } ], "skills": { "available": [], "loaded": [] }, "docs": { "calls": [ { "source": "web_search", "query": "site:hookdeck.com/docs/api events Hookdeck REST API list events attempts issues", "pages": [] } ] }, "prompt": "Our order events aren't all getting through. The fulfilment side seems fine but\nsomething's definitely broken, and I've had it flagged twice this week.\n\nCan you work out what's actually going wrong?", "promptSourcePath": "evals/benchmark-investigate-001-failing-deliveries/PROMPT.md", "attempts": 1, "sourcePath": "codex-gpt-5.6-no-skills/benchmark-investigate-001-failing-deliveries.json" }, { "experiment": "codex-gpt-5.6-no-skills", "experimentSuite": "no-skills", "experimentDisplay": { "agent": "codex", "modelProvider": "openai", "modelId": "gpt-5.6", "reasoningEffort": "high" }, "eval": "benchmark-investigate-002-partial-outage", "ranAt": "2026-09-01T10:12:42.379Z", "runId": "33484972784", "stage": "investigate", "product": [ "event-gateway" ], "topic": [ "filtering" ], "suite": "benchmark", "gatedBy": "mixed", "passed": true, "checks": [ { "name": "looked past the events list", "passed": true }, { "name": "identified the filter as the cause and what it excludes", "passed": true, "judgeNotes": "Identifies the connection filter requiring channel=web as the reason missing orders are absent, and specifies that mobile and POS orders are excluded/ignored before events are created." }, { "name": "did not report the system as healthy", "passed": true, "judgeNotes": "The assistant did not conclude that everything was working or that all orders were delivered; it identified ignored/filtered missing orders." } ], "skills": { "available": [], "loaded": [] }, "docs": { "calls": [] }, "prompt": "Our warehouse team says they're missing orders. Not all of them, and I can't\nfind a single failure anywhere I've looked. Every delivery I can see went\nthrough fine.\n\nWhere are the missing ones going?", "promptSourcePath": "evals/benchmark-investigate-002-partial-outage/PROMPT.md", "attempts": 1, "sourcePath": "codex-gpt-5.6-no-skills/benchmark-investigate-002-partial-outage.json" }, { "experiment": "codex-gpt-5.6-no-skills", "experimentSuite": "no-skills", "experimentDisplay": { "agent": "codex", "modelProvider": "openai", "modelId": "gpt-5.6", "reasoningEffort": "high" }, "eval": "benchmark-localdev-001-listen-locally", "ranAt": "2026-09-01T10:28:43.414Z", "runId": "33484972784", "stage": "build", "product": [ "event-gateway" ], "topic": [ "local-dev" ], "suite": "benchmark", "gatedBy": "discovery", "passed": true, "checks": [ { "name": "created a source to receive events", "passed": true }, { "name": "routed to a CLI destination rather than an unreachable URL", "passed": true }, { "name": "an event posted at the source arrives at the local service", "passed": true } ], "skills": { "available": [], "loaded": [] }, "docs": { "calls": [ { "source": "web_search", "query": "site:hookdeck.com/docs CLI listen local webhook forwarding Hookdeck", "pages": [] } ] }, "prompt": "I've written the notifications service in this directory but I've got no way to\ntry it against real traffic. It's not deployed anywhere yet and I'd rather not\npush it somewhere just to find out the handler is wrong.\n\nGet events arriving at it here on my machine so I can watch them come in.", "promptSourcePath": "evals/benchmark-localdev-001-listen-locally/PROMPT.md", "attempts": 1, "sourcePath": "codex-gpt-5.6-no-skills/benchmark-localdev-001-listen-locally.json" }, { "experiment": "codex-gpt-5.6-no-skills", "experimentSuite": "no-skills", "experimentDisplay": { "agent": "codex", "modelProvider": "openai", "modelId": "gpt-5.6", "reasoningEffort": "high" }, "eval": "benchmark-outpost-001-customer-subscriptions", "ranAt": "2026-09-01T10:42:45.839Z", "runId": "33484972784", "stage": "build", "product": [ "outpost" ], "topic": [ "sdk" ], "suite": "benchmark", "gatedBy": "discovery", "passed": true, "checks": [ { "name": "created a tenant for the customer", "passed": true }, { "name": "the customer has a destination at the endpoint they gave us", "passed": true }, { "name": "an order event reaches the customer", "passed": true } ], "skills": { "available": [], "loaded": [] }, "docs": { "calls": [ { "source": "web_search", "query": "site:hookdeck.com/docs Outpost API subscriptions destination source publish events Hookdeck", "pages": [] }, { "source": "web_search", "query": "https://hookdeck.com/docs", "pages": [ { "url": "https://hookdeck.com/docs" } ] }, { "source": "web_search", "query": "https://hookdeck.com/docs/outpost", "pages": [ { "url": "https://hookdeck.com/docs/outpost" } ] }, { "source": "web_search", "query": "https://hookdeck.com/docs/outpost/features/multi-tenancy", "pages": [ { "url": "https://hookdeck.com/docs/outpost/features/multi-tenancy" } ] } ] }, "prompt": "We want our customers to be able to subscribe to events from our platform, so\nthey can react when an order is placed rather than polling us for it.\n\nSet that up for our first customer, `acme`, so they receive order events at\nthe endpoint they gave us:\n\n https://mock.hookdeck.com/api/v1/acme/orders\n\nThen show me it works by sending one through.", "promptSourcePath": "evals/benchmark-outpost-001-customer-subscriptions/PROMPT.md", "attempts": 1, "sourcePath": "codex-gpt-5.6-no-skills/benchmark-outpost-001-customer-subscriptions.json" }, { "experiment": "codex-gpt-5.6-no-skills", "experimentSuite": "no-skills", "experimentDisplay": { "agent": "codex", "modelProvider": "openai", "modelId": "gpt-5.6", "reasoningEffort": "high" }, "eval": "benchmark-outpost-002-disabled-destination", "ranAt": "2026-09-01T11:06:04.262Z", "runId": "33484972784", "stage": "resolve", "product": [ "outpost" ], "topic": [ "retries" ], "suite": "benchmark", "gatedBy": "discovery", "passed": true, "checks": [ { "name": "the customer can receive again", "passed": true }, { "name": "the events the customer missed were delivered", "passed": true }, { "name": "the other customer was left alone", "passed": true } ], "skills": { "available": [], "loaded": [] }, "docs": { "calls": [ { "source": "web_search", "query": "site:hookdeck.com/docs/api events attempts retries destinations connections Hookdeck API", "pages": [] }, { "source": "web_search", "query": "site:hookdeck.com/docs/outpost API reference tenants subscriptions deliveries retry", "pages": [] }, { "source": "shell_fetch", "query": "/bin/bash -lc \"set -euo pipefail\nprintf '%s\\\\n' '--- search local docs/caches ---'\nrg -n --hidden -S \\\"OUTPOST_API_KEY|outpost\\\\.hookdeck|api\\\\.outpost|/tenants|/subscriptions\\\" /home/node 2>/dev/null | head -200 || true\nprintf '%s\\\\n' '--- probe likely hosts ---'\nfor url in https://api.hookdeck.com https://outpost.hookdeck.com https://api.outpost.hookdeck.com https://outpost-api.hookdeck.com; do\n code=\"'$(curl -sS -o /tmp/probe -w '\"'%{http_code}' \\\"\"'$url\" || true)\n printf '\"'%s %s ' \\\"\"'$code\" \"$url\"; head -c 160 /tmp/probe 2>/dev/null | tr '\"'\\\\n' ' '; echo\ndone\nprintf '%s\\\\n' '--- docs robots/sitemap ---'\ncurl -sS https://hookdeck.com/robots.txt | head -100\ncurl -sS https://hookdeck.com/sitemap.xml | rg -i 'outpost|api' | head -100\"", "pages": [ { "url": "https://hookdeck.com/robots.txt" }, { "url": "https://hookdeck.com/sitemap.xml" } ], "resultChars": 26323 }, { "source": "web_search", "query": "site:github.com/hookdeck outpost api tenants destinations retry events", "pages": [] } ] }, "prompt": "Acme emailed to say they stopped receiving order events some time yesterday.\nTheir engineer says their endpoint had a bad deploy but it's been fine since\nthis morning, and they've checked — nothing is arriving.\n\nWork out why and get their events flowing again, including the ones they\nmissed.", "promptSourcePath": "evals/benchmark-outpost-002-disabled-destination/PROMPT.md", "attempts": 1, "sourcePath": "codex-gpt-5.6-no-skills/benchmark-outpost-002-disabled-destination.json" }, { "experiment": "codex-gpt-5.6-no-skills", "experimentSuite": "no-skills", "experimentDisplay": { "agent": "codex", "modelProvider": "openai", "modelId": "gpt-5.6", "reasoningEffort": "high" }, "eval": "benchmark-outpost-003-operator-events", "ranAt": "2026-09-01T11:44:00.039Z", "runId": "33484972784", "stage": "build", "product": [ "outpost" ], "topic": [ "alerting" ], "suite": "benchmark", "gatedBy": "discovery", "passed": true, "checks": [ { "name": "alerts are delivered to the endpoint that was asked for", "passed": true }, { "name": "the disabled-destination alert is subscribed", "passed": true }, { "name": "the alerting destination is enabled", "passed": true }, { "name": "the build-up to a disable is subscribed too", "passed": true } ], "skills": { "available": [], "loaded": [] }, "docs": { "calls": [ { "source": "web_search", "query": "site:hookdeck.com/docs Outpost destination disabled alert webhook", "pages": [] }, { "source": "web_search", "query": "Hookdeck Outpost API base URL api.outpost.hookdeck.com", "pages": [] }, { "source": "web_search", "query": "site:hookdeck.com/docs monitoring settings project alerts webhook endpoint Hookdeck", "pages": [] }, { "source": "web_search", "query": "\"operator-alerts\" Hookdeck Outpost", "pages": [] }, { "source": "web_search", "query": "\"operator event destinations\" Hookdeck", "pages": [] } ] }, "prompt": "Last week one of our customers' destinations was switched off after a bad\ndeploy on their side, and their events piled up until they emailed us. We\nfound it by hand.\n\nWe don't want to find out that way again. Set us up so that when Outpost\ndisables a customer's destination, the alert reaches this endpoint:\n\n https://mock.hookdeck.com/operator-alerts\n\nConsecutive delivery failures building up towards that threshold would be\nuseful too — we'd rather know before a destination is switched off than after.", "promptSourcePath": "evals/benchmark-outpost-003-operator-events/PROMPT.md", "attempts": 1, "sourcePath": "codex-gpt-5.6-no-skills/benchmark-outpost-003-operator-events.json" }, { "experiment": "codex-gpt-5.6-no-skills", "experimentSuite": "no-skills", "experimentDisplay": { "agent": "codex", "modelProvider": "openai", "modelId": "gpt-5.6", "reasoningEffort": "high" }, "eval": "benchmark-outpost-004-queue-destination", "ranAt": "2026-09-01T12:00:44.328Z", "runId": "33484972784", "stage": "build", "product": [ "outpost" ], "topic": [ "capabilities" ], "suite": "benchmark", "gatedBy": "mixed", "passed": true, "checks": [ { "name": "their orders are delivered to the queue they gave us", "passed": true }, { "name": "the queue is subscribed to their order events", "passed": true }, { "name": "the queue destination carries credentials", "passed": true }, { "name": "their orders no longer go to the old endpoint", "passed": true }, { "name": "the rest of their delivery is unchanged", "passed": true }, { "name": "the other customer was left alone", "passed": true } ], "skills": { "available": [], "loaded": [] }, "docs": { "calls": [ { "source": "web_search", "query": "site:hookdeck.com/docs API destinations SQS Hookdeck REST API connections rules", "pages": [] }, { "source": "web_search", "query": "site:github.com/hookdeck Hookdeck API destinations SQS queue_url access_key secret_key", "pages": [] } ] }, "prompt": "Acme are moving off webhooks. Their endpoint keeps falling over under load and\nthey'd rather we drop order events straight onto a queue they already run.\n\nThey sent us the queue details last week — whoever picked up the ticket put\nthem in the repo.\n\nSet that up, and stop sending their orders to the old endpoint.", "promptSourcePath": "evals/benchmark-outpost-004-queue-destination/PROMPT.md", "attempts": 1, "sourcePath": "codex-gpt-5.6-no-skills/benchmark-outpost-004-queue-destination.json" }, { "experiment": "codex-gpt-5.6-no-skills", "experimentSuite": "no-skills", "experimentDisplay": { "agent": "codex", "modelProvider": "openai", "modelId": "gpt-5.6", "reasoningEffort": "high" }, "eval": "benchmark-outpost-005-topic-scoping", "ranAt": "2026-09-01T12:17:53.263Z", "runId": "33484972784", "stage": "resolve", "product": [ "outpost" ], "topic": [ "filtering" ], "suite": "benchmark", "gatedBy": "mixed", "passed": true, "checks": [ { "name": "acme no longer receives order cancellations", "passed": true }, { "name": "acme still receives the shipping events they depend on", "passed": true }, { "name": "acme still receives new orders", "passed": true }, { "name": "globex was left alone", "passed": true }, { "name": "no destination was switched off to achieve it", "passed": true } ], "skills": { "available": [], "loaded": [] }, "docs": { "calls": [ { "source": "web_search", "query": "site:hookdeck.com/docs API connections rules filters Hookdeck REST API", "pages": [] }, { "source": "web_search", "query": "site:hookdeck.com/docs/outpost API subscriptions filters topics Hookdeck Outpost", "pages": [] }, { "source": "web_search", "query": "\"Outpost API\" \"Hookdeck\" tenants", "pages": [] } ] }, "prompt": "Acme have been in touch. Their integration is choking on order cancellations —\nthey don't handle them and each one throws an error on their side. Their order\nconfirmations are working fine and they rely on those.\n\nStop the cancellations.", "promptSourcePath": "evals/benchmark-outpost-005-topic-scoping/PROMPT.md", "attempts": 1, "sourcePath": "codex-gpt-5.6-no-skills/benchmark-outpost-005-topic-scoping.json" }, { "experiment": "codex-gpt-5.6-no-skills", "experimentSuite": "no-skills", "experimentDisplay": { "agent": "codex", "modelProvider": "openai", "modelId": "gpt-5.6", "reasoningEffort": "high" }, "eval": "benchmark-resolve-001-paused-connection", "ranAt": "2026-09-01T12:31:35.479Z", "runId": "33484972784", "stage": "resolve", "product": [ "event-gateway" ], "topic": [ "retries" ], "suite": "benchmark", "gatedBy": "discovery", "passed": true, "checks": [ { "name": "the connection is no longer paused", "passed": true }, { "name": "a new checkout event reaches the orders API", "passed": true } ], "skills": { "available": [], "loaded": [] }, "docs": { "calls": [] }, "prompt": "Checkout events stopped reaching our orders API. Nothing's changed on our side\nthat I know of, and the endpoint is up.\n\nGet them flowing again.", "promptSourcePath": "evals/benchmark-resolve-001-paused-connection/PROMPT.md", "attempts": 1, "sourcePath": "codex-gpt-5.6-no-skills/benchmark-resolve-001-paused-connection.json" }, { "experiment": "codex-gpt-5.6-no-skills", "experimentSuite": "no-skills", "experimentDisplay": { "agent": "codex", "modelProvider": "openai", "modelId": "gpt-5.6", "reasoningEffort": "high" }, "eval": "benchmark-resolve-002-scoped-redelivery", "ranAt": "2026-09-01T12:46:34.088Z", "runId": "33484972784", "stage": "resolve", "product": [ "event-gateway" ], "topic": [ "retries" ], "suite": "benchmark", "gatedBy": "mixed", "passed": true, "checks": [ { "name": "the failed checkout events were redelivered", "passed": true }, { "name": "the inventory events were left alone", "passed": true } ], "skills": { "available": [], "loaded": [] }, "docs": { "calls": [ { "source": "web_search", "query": "site:hookdeck.com/docs API events bulk retry redeliver attempts endpoint Hookdeck", "pages": [] }, { "source": "web_search", "query": "site:hookdeck.com/docs/api-reference events list Hookdeck", "pages": [] } ] }, "prompt": "We had an outage this morning and both our endpoints were rejecting everything.\nThey're back up now.\n\nCheckout is the urgent one, so get those failed events redelivered. Leave\ninventory alone for now, the team there wants to check something first.", "promptSourcePath": "evals/benchmark-resolve-002-scoped-redelivery/PROMPT.md", "attempts": 1, "sourcePath": "codex-gpt-5.6-no-skills/benchmark-resolve-002-scoped-redelivery.json" }, { "experiment": "codex-gpt-5.6-no-skills", "experimentSuite": "no-skills", "experimentDisplay": { "agent": "codex", "modelProvider": "openai", "modelId": "gpt-5.6", "reasoningEffort": "high" }, "eval": "benchmark-transform-001-reshape-payload", "ranAt": "2026-09-01T13:06:25.041Z", "runId": "33484972784", "stage": "build", "product": [ "event-gateway" ], "topic": [ "transformations" ], "suite": "benchmark", "gatedBy": "discovery", "passed": true, "checks": [ { "name": "amount is at the top level of the delivered body", "passed": true }, { "name": "currency is at the top level of the delivered body", "passed": true }, { "name": "email is at the top level of the delivered body", "passed": true } ], "skills": { "available": [], "loaded": [] }, "docs": { "calls": [ { "source": "web_search", "query": "site:hookdeck.com/docs API transformations rules connections Hookdeck REST API", "pages": [] } ] }, "prompt": "Our payment provider changed their payload and our billing service can't read\nit any more. The service expects `amount`, `currency` and `email` at the top\nlevel of the body, and nothing else has to change.\n\nSort it out so the billing service starts working again. I'd rather know it's\nright before it goes anywhere near production.", "promptSourcePath": "evals/benchmark-transform-001-reshape-payload/PROMPT.md", "attempts": 1, "sourcePath": "codex-gpt-5.6-no-skills/benchmark-transform-001-reshape-payload.json" }, { "experiment": "codex-gpt-5.6-no-skills", "experimentSuite": "no-skills", "experimentDisplay": { "agent": "codex", "modelProvider": "openai", "modelId": "gpt-5.6", "reasoningEffort": "high" }, "eval": "benchmark-verification-001-stripe-express", "ranAt": "2026-09-01T13:39:35.806Z", "runId": "33484972784", "stage": "build", "product": [ "event-gateway" ], "topic": [ "signature-verification" ], "suite": "benchmark", "gatedBy": "discovery", "passed": true, "checks": [ { "name": "created a Stripe source", "passed": true }, { "name": "the source accepts a genuine Stripe signature", "passed": true }, { "name": "the source rejects a forged Stripe signature", "passed": true }, { "name": "the handler accepts a genuine Hookdeck signature", "passed": true }, { "name": "the handler rejects a forged signature", "passed": true, "notes": "expected a 4xx, got 401" } ], "skills": { "available": [], "loaded": [] }, "docs": { "calls": [ { "source": "web_search", "query": "site:hookdeck.com/docs Hookdeck verify webhook signature destination HMAC x-hookdeck-signature", "pages": [] }, { "source": "web_search", "query": "site:hookdeck.com/docs \"X-Hookdeck-Signature\"", "pages": [] }, { "source": "web_search", "query": "github hookdeck docs destination auth hookdeck signature", "pages": [] } ] }, "prompt": "We're taking Stripe payments and I need our orders API to react when a checkout\ncompletes. Right now nothing is wired up.\n\nSet it up so Stripe events reach the `/orders` endpoint, and make sure the\nendpoint only accepts requests that genuinely came through, not anything that\nturns up at the URL. I'm still working on this locally, and I don't want to miss\nevents while I'm developing.", "promptSourcePath": "evals/benchmark-verification-001-stripe-express/PROMPT.md", "attempts": 1, "sourcePath": "codex-gpt-5.6-no-skills/benchmark-verification-001-stripe-express.json" }, { "experiment": "codex-gpt-5.6-no-skills", "experimentSuite": "no-skills", "experimentDisplay": { "agent": "codex", "modelProvider": "openai", "modelId": "gpt-5.6", "reasoningEffort": "high" }, "eval": "benchmark-verification-002-elevenlabs-callbacks", "ranAt": "2026-09-01T14:10:04.046Z", "runId": "33484972784", "stage": "build", "product": [ "event-gateway" ], "topic": [ "signature-verification" ], "suite": "benchmark", "gatedBy": "discovery", "passed": false, "checks": [ { "name": "created an ElevenLabs source", "passed": true }, { "name": "the source accepts a genuine ElevenLabs signature", "passed": true }, { "name": "the source rejects a forged ElevenLabs signature", "passed": true }, { "name": "the service accepts a genuine Hookdeck signature", "passed": false, "notes": "expected 200, got 403. service said: > transcripts-service@1.0.0 start > node server.js transcripts-service on 5100" }, { "name": "the service rejects a forged signature", "passed": true, "notes": "expected a 4xx, got 403" } ], "skills": { "available": [], "loaded": [] }, "docs": { "calls": [ { "source": "web_search", "query": "site:hookdeck.com/docs API create source destination connection verification webhook signature Hookdeck", "pages": [] }, { "source": "web_search", "query": "site:hookdeck.com/docs/api source create Hookdeck API", "pages": [] }, { "source": "web_search", "query": "site:hookdeck.com/docs \"Hookdeck-Signature\"", "pages": [] } ] }, "prompt": "We're using ElevenLabs for transcription and they call us back when a job\nfinishes. The callbacks need to reach the transcripts service in this directory,\nand I don't want it accepting anything that didn't genuinely come from them via\nour gateway.\n\nSet that up.", "promptSourcePath": "evals/benchmark-verification-002-elevenlabs-callbacks/PROMPT.md", "attempts": 1, "sourcePath": "codex-gpt-5.6-no-skills/benchmark-verification-002-elevenlabs-callbacks.json" } ] }