// Package eventforward forwards a connector's raw provider events to the customer's own URLs, // its event destinations (T46, AI-875; channels.md on connectors/planning, «Integration modes: // full platform, customizations, pass-through», modes B and C). // // The events route verifies a provider delivery with the provider app's secret and acts on it, // then hands it to Forward, which queues it in Postgres for each destination that takes it and // returns: the provider's ack never waits on a customer's URL. A worker on every router takes // the queued forwards and sends each one with the provider's body and verification headers as // they came, signed on top in the Standard Webhooks shape with the destination's own secret, // never with a deployment secret (https://www.standardwebhooks.com/). package eventforward import ( "bytes" "context" "crypto/sha256" "encoding/hex" "errors" "fmt" "io" "log/slog" "maps" "net/http" "strconv" "strings" "sync" "time" "github.com/GetStream/Vision-Agents/acceleration/internal/activity" "github.com/GetStream/Vision-Agents/acceleration/internal/auth" "github.com/GetStream/Vision-Agents/acceleration/internal/connectors/core" "github.com/GetStream/Vision-Agents/acceleration/internal/egress" "github.com/GetStream/Vision-Agents/acceleration/internal/stack" "github.com/GetStream/Vision-Agents/acceleration/internal/store" "github.com/GetStream/Vision-Agents/acceleration/internal/webhook" ) // The Standard Webhooks spec, opened October 6, 2026: // https://github.com/standard-webhooks/standard-webhooks/blob/main/spec/standard-webhooks.md // («Webhook headers», «Deliverability and reliability», «Request timeouts»). const ( headerID = "webhook-id" headerTimestamp = "webhook-timestamp" headerSignature = "webhook-signature" ) // attemptTimeout bounds one send: the spec's «A recommended request timeout value for webhooks // is somewhere between 15 and 30s», its low end, so one slow customer holds a worker slot the // least the spec allows. const attemptTimeout = 15 * time.Second // lease is how far a worker pushes a delivery's next attempt when it takes one, so no other // router takes it while it is sent: longer than one attempt's attemptTimeout, so a live worker // never loses one, and short enough that one a stopped router held is sent again within a // minute. A choice, not a spec value. // // It is also the longest a worker waits between looks for deliveries, even when none is queued // or the first is due later (sendDue): a forward another router queued or leased and then // stopped with is sent within about two leases, at 2 queries a lease from an idle router. const lease = time.Minute // defaultPoll is the shortest wait before a worker looks again for deliveries while some are // queued (sendDue), besides when Forward queues one, which it sends at once. The shortest wait // in defaultRetries is 5 s, so a retry goes out at most a second late. A choice, not a spec // value. const defaultPoll = time.Second // inFlight is how many sends one router runs at once: one slow destination, at attemptTimeout, // takes perDestination slots and leaves the rest to the others. A choice, not a spec value. // # unverified against real load const inFlight = 16 // perDestination is how many sends to one destination one router runs at once (AI-924). A // destination that never answers holds perDestination slots for attemptTimeout each and // leaves the other inFlight - perDestination, 14 of 16, to every other destination. Two, not // one, so one slow answer does not hold up a destination that is up. A choice, not a spec // value: the example the review of #774 gave. const perDestination = 2 // maxAnswerBytes is how much of a destination's answer is read: none of it is used, so it is // read only to let the connection be reused; 64 KiB is a choice. const maxAnswerBytes = 64 << 10 // defaultRetries are the waits before each attempt after the first: the spec's example retry // schedule («Example retry schedule»: immediately, 5 seconds, 5 minutes, 30 minutes, 2 hours), // its first five rows. After the fifth attempt fails, at 2 h 35 min 5 s, the forward is given // up and logged. The spec's later rows, 5 h to 24 h, go with a delivery log the customer can // read and with telling the customer by other means, which this router does not have yet; a // raw interactive event, such as a button click, is moot long before then. A choice. var defaultRetries = []time.Duration{5 * time.Second, 5 * time.Minute, 30 * time.Minute, 2 * time.Hour} // rotationOverlap is how long the secret a rotation replaced still signs beside the new one, // so a receiver can move to the new one without a delivery failing its check («the webhook is // signed both using the current key, and using an old key (for a set period of time)»). The // spec names no period; a day is a choice, and covers the last retry of defaultRetries. const rotationOverlap = 24 * time.Hour // errNotPublic is a destination URL egress refuses: not https, or a host that is or resolves // to a private, loopback or otherwise not public address. var errNotPublic = errors.New("eventforward: the destination is not a public https URL") // errRefused is a destination answering a forward with a status that is not sent again: a 3xx // or a 4xx other than 429. var errRefused = errors.New("eventforward: the destination refused the forward") // The codes a forward given up is counted under in the activity record, by why it was: the // destination said no, it never took one of the attempts, or it was never sent, such as a // signing secret that does not open. const ( codeRefused = "destination_refused" codeUnavailable = "destination_unavailable" codeNotSent = "forward_failed" ) // Options configures a Forwarder. Store and Secrets are required. type Options struct { Store *store.Store // Secrets seals and opens the destinations' signing secrets. Secrets *auth.Sealer // HTTP sends every forward. Nil is egress.NewClient, which dials public addresses alone. // Tests pass a client that reaches their local destinations. Redirects are never followed, // whatever client it is. HTTP *http.Client // PublicURL checks a destination URL before it is stored. Nil is // egress.ValidatePublicHTTPSURL. PublicURL func(ctx context.Context, raw string) error // Retries are the waits before each attempt after the first. Nil is defaultRetries. Retries []time.Duration // Poll is the shortest wait between looks for due deliveries while some are queued. Zero is // defaultPoll. Poll time.Duration // Lease is how far a claim pushes a delivery's next attempt, and the longest wait between // looks. Zero is lease. Tests shorten it. Lease time.Duration // Activity counts each forward sent or given up under its app, as area webhooks_out. Nil // counts none. Activity *activity.Recorder Logger *slog.Logger } // Forwarder queues provider deliveries for the customer's event destinations and sends them. type Forwarder struct { store *store.Store secrets *auth.Sealer http *http.Client publicURL func(ctx context.Context, raw string) error retries []time.Duration poll time.Duration lease time.Duration activity *activity.Recorder logger *slog.Logger // kick wakes the worker: a delivery was queued, or a slot freed. kick chan struct{} // slots holds one token for each send running. slots chan struct{} // sending is how many sends to each destination are running, by destination id. mu sync.Mutex sending map[string]int ctx context.Context cancel context.CancelFunc working sync.WaitGroup } // Event is one verified provider delivery of one customer's provider app. type Event struct { CustomerID string ConnectorID string // ID is the provider's own id for the delivery (core.ChannelRule.DeliveryEventID). Empty // keys the forward by its body (deliveryID). ID string // Headers are the provider's headers a receiver verifies the body with (ProviderHeaders). Headers map[string]string // HeadersUntil is when the provider's signature in Headers stops verifying at a receiver // that checks its timestamp (ProviderEvent). Zero when it does not age. HeadersUntil time.Time // Body is the raw request body, as verified. Body []byte // Handled is whether the router acted on the delivery: a signal, or a message an agent of // the customer answers. A destination that forwards unhandled takes it only when false. Handled bool } // Secret is a destination's signing secret, made for it and sealed bound to it. type Secret struct { // Plain is the whsec_ secret, shown to the customer once. Plain string Sealed []byte Version int } // New validates the options and returns a Forwarder. It sends nothing until Start. func New(options Options) (*Forwarder, error) { if options.Store == nil || options.Secrets == nil { return nil, stack.Wrap(errors.New("eventforward: a store and a sealer are required")) } client := options.HTTP if client == nil { client = egress.NewClient(attemptTimeout, nil) } // A copy, so the caller's client keeps its own policy. The spec: «3xx: Failure. Following // redirects causes unnecessary load on both the sender and the receiver». unredirected := *client unredirected.CheckRedirect = func(*http.Request, []*http.Request) error { return http.ErrUseLastResponse } publicURL := options.PublicURL if publicURL == nil { publicURL = egress.ValidatePublicHTTPSURL } retries := options.Retries if retries == nil { retries = defaultRetries } poll := options.Poll if poll <= 0 { poll = defaultPoll } leaseFor := options.Lease if leaseFor <= 0 { leaseFor = lease } logger := options.Logger if logger == nil { logger = slog.Default() } ctx, cancel := context.WithCancel(context.Background()) return &Forwarder{ store: options.Store, secrets: options.Secrets, http: &unredirected, publicURL: publicURL, retries: retries, poll: poll, lease: leaseFor, activity: options.Activity, logger: logger, kick: make(chan struct{}, 1), slots: make(chan struct{}, inFlight), sending: map[string]int{}, ctx: ctx, cancel: cancel, }, nil } // Start runs the worker until Close. It looks for deliveries at start, for the ones a router // queued before it stopped, then when Forward or a finished send wakes it, at the first one // due, and at least once a lease: with no deliveries it sends Postgres 2 queries a minute, not // one a second (AI-926). func (f *Forwarder) Start() { f.working.Add(1) go func() { defer f.working.Done() timer := time.NewTimer(0) defer timer.Stop() for { wait, again := f.sendDue() timer.Stop() if again { timer.Reset(wait) } select { case <-f.ctx.Done(): return case <-timer.C: case <-f.kick: } } }() } // Close stops the worker and cancels the sends in flight. A send cut short is not counted as // an attempt: its lease runs out and a router sends it again. func (f *Forwarder) Close() { f.cancel() f.working.Wait() } // Forward queues one delivery for each of the customer's destinations of the connector that // takes it, and wakes the worker. It touches Postgres alone and sends nothing, so it is safe on // the request path of a provider that waits a few seconds at most (Slack: «respond ... within // three seconds», https://docs.slack.dev/apis/events-api/). An error is the store failing. func (f *Forwarder) Forward(ctx context.Context, event Event) error { forwards := []string{store.ForwardAll} if !event.Handled { forwards = append(forwards, store.ForwardUnhandled) } return f.queue(ctx, event, forwards) } // ForwardUnanswered queues a delivery for the destinations that take what the router does not // handle, alone, and wakes the worker. It is for a delivery Forward counted as handled for a // message an agent was to answer, whose write into its thread channel failed after the ack // (channelbridge.Bridge.Deliver): the destinations of every event already have it. func (f *Forwarder) ForwardUnanswered(ctx context.Context, event Event) error { return f.queue(ctx, event, []string{store.ForwardUnhandled}) } // queue queues the event for the customer's destinations whose forward is one of forwards. func (f *Forwarder) queue(ctx context.Context, event Event, forwards []string) error { if event.CustomerID == "" { return nil } delivery := store.EventDelivery{ ID: deliveryID(event.ID, event.Body), Headers: event.Headers, Body: event.Body, NextAttemptAt: time.Now(), } if !event.HeadersUntil.IsZero() { delivery.ProviderHeadersUntil = &event.HeadersUntil } queued, err := f.store.QueueEventDeliveries(ctx, event.CustomerID, event.ConnectorID, forwards, delivery) if err != nil { return err } if queued > 0 { f.wake() } return nil } // CheckURL is errNotPublic for a destination URL a forward could not be sent to. func (f *Forwarder) CheckURL(ctx context.Context, raw string) error { if err := f.publicURL(ctx, raw); err != nil { return stack.Wrap(fmt.Errorf("%w: %w", errNotPublic, err)) } return nil } // NewSecret is a fresh signing secret for one destination, sealed with the destination's // customer, connector and id as AAD, so a sealed secret copied onto another row does not open. func (f *Forwarder) NewSecret(customerID, connectorID, id string) (Secret, error) { // 32 random bytes, inside the spec's «Between 24 bytes (192 bits) and 64 bytes (512 bits)», // base64 after whsec_, the form MCP Events subscriptions already use. plain, err := webhook.NewSecret() if err != nil { return Secret{}, stack.Wrap(err) } sealed, err := f.secrets.SealWithAAD(plain, secretAAD(customerID, connectorID, id)) if err != nil { return Secret{}, stack.Wrap(err) } return Secret{Plain: plain, Sealed: sealed, Version: f.secrets.CurrentVersion()}, nil } // RotationOverlap is how long a secret a rotation replaced still signs, from now. func RotationOverlap(now time.Time) time.Time { return now.Add(rotationOverlap) } // deliveryID is the webhook-id of a delivery: msg_ and the first 128 bits of the SHA-256 of // the provider's own id for it, eventID, or of the body when the provider names none. So one // provider event is one id, on every attempt and when the provider delivers it again, and two // events are two ids even when their bodies are the same. The spec: the id «remains the same // no matter how many times a webhook that has failed is retried», and it must hold no «.», // which hex never does and Slack's trigger_id does. The event\x00 prefix keeps an event id's // digest apart from a body's: no JSON or form body starts with "event" and a NUL. func deliveryID(eventID string, body []byte) string { sum := sha256.Sum256(body) if eventID != "" { sum = sha256.Sum256([]byte("event\x00" + eventID)) } return "msg_" + hex.EncodeToString(sum[:16]) } // ProviderHeaders are the request headers a destination is sent beside the raw body, as they // came: Content-Type, and the signature and timestamp headers the connector's channel.verifier // names (Slack: X-Slack-Signature and X-Slack-Request-Timestamp), so a receiver that holds the // app's own signing secret, such as Slack Bolt, verifies the body itself. Nothing else is // copied: a proxy's or a load balancer's headers say where the router runs, and a provider's // retry headers (X-Slack-Retry-Num) are about the router's answer, not the customer's; // webhook-id is what tells the receiver a delivery again. func ProviderHeaders(m core.Manifest, header http.Header) map[string]string { names := []string{"Content-Type"} if m.Channel != nil { names = append(names, m.Channel.Verifier.Header, m.Channel.Verifier.TimestampHeader) } kept := map[string]string{} for _, name := range names { if value := header.Get(name); name != "" && value != "" { kept[http.CanonicalHeaderKey(name)] = value } } return kept } // ProviderEvent is the Event of one verified delivery of a connector m: the provider's own id // for it, the provider's headers a receiver verifies it with (ProviderHeaders), and until when // those verify, its signed timestamp plus channel.verifier.max_age. For Slack that is // X-Slack-Request-Timestamp plus 5 minutes, the window Slack Bolt checks too: // «const requestTimestampMaxDeltaMin = 5;» in bolt-js src/receivers/verify-request.ts // (https://github.com/slackapi/bolt-js/blob/91c96e3e6d3495c1a976ecf6edfb2a6a5b236b47/src/receivers/verify-request.ts#L41, // opened October 7, 2026). The caller sets the customer and Handled. func ProviderEvent(m core.Manifest, header http.Header, body []byte) Event { event := Event{ConnectorID: m.ID, Headers: ProviderHeaders(m, header), Body: body} if m.Channel == nil { return event } event.ID = m.Channel.DeliveryEventID(body) rule := m.Channel.Verifier if rule.TimestampHeader == "" || rule.MaxAge <= 0 { return event } // Decimal Unix seconds, as the hmac_header verifier read it (hmacheader.Verifier.Verify). if seconds, err := strconv.ParseInt(header.Get(rule.TimestampHeader), 10, 64); err == nil { event.HeadersUntil = time.Unix(seconds, 0).Add(time.Duration(rule.MaxAge)) } return event } // sign is the webhook-signature header of one send: v1, and the base64 HMAC-SHA256 of // id.timestamp.body under each secret, space-separated, so a receiver holding either secret of // a rotation verifies it («Webhook headers»: «The signature header is a space delimited list of // signatures»). func sign(id string, at time.Time, body []byte, secrets ...string) (string, error) { signatures := make([]string, 0, len(secrets)) for _, secret := range secrets { signature, err := webhook.Sign(secret, id, at, body) if err != nil { return "", stack.Wrap(err) } signatures = append(signatures, signature) } return strings.Join(signatures, " "), nil } // Wake tells the worker that something outside Forward queued deliveries, such as the // policies' warning that an organization's credit is running out, so it sends them now rather // than at its next look. func (f *Forwarder) Wake() { f.wake() } // wake tells the worker there may be something to send, without waiting. func (f *Forwarder) wake() { select { case f.kick <- struct{}{}: default: } } // sendDue takes as many due deliveries as there are free slots, at most perDestination of // one destination's counting the ones it is still sending, and sends each on its own // goroutine, until none is due or no slot is free. // // It returns how long to wait before looking again, with again false when only a finished send // needs to wake the worker: every slot is busy. While deliveries are queued it waits until the // first is due, its retry or another router's lease running out, and at least the poll, so // rows another router is claiming at that moment are not looked for in a busy loop. It waits // a lease at most, also when none is queued: a router that queued or leased one and then // stopped wakes nobody else (the review of #778). func (f *Forwarder) sendDue() (wait time.Duration, again bool) { for { if f.ctx.Err() != nil { return 0, false } free := cap(f.slots) - len(f.slots) if free == 0 { return 0, false } now := time.Now() f.mu.Lock() sending := maps.Clone(f.sending) f.mu.Unlock() claimed, err := f.store.ClaimEventDeliveries(f.ctx, now, free, perDestination, sending, now.Add(f.lease)) if err != nil { if f.ctx.Err() == nil { f.logger.Error("could not take the event forwards due", "error", err) } return f.poll, true } for _, delivery := range claimed { f.slots <- struct{}{} f.mu.Lock() f.sending[delivery.DestinationID]++ f.mu.Unlock() f.working.Add(1) go func() { defer f.working.Done() defer func() { f.mu.Lock() if f.sending[delivery.DestinationID]--; f.sending[delivery.DestinationID] == 0 { delete(f.sending, delivery.DestinationID) } f.mu.Unlock() <-f.slots f.wake() }() f.attempt(delivery) }() } if len(claimed) < free { break } } next, queued, err := f.store.NextEventDeliveryAt(f.ctx) switch { case err != nil: if f.ctx.Err() == nil { f.logger.Error("could not find when the next event forward is due", "error", err) } return f.poll, true case !queued: return f.lease, true } return min(max(time.Until(next), f.poll), f.lease), true } // attempt sends a delivery once and records what came of it: gone when the destination took // it or refused it for good, else due again after the next wait, or gone when the waits ran // out. func (f *Forwarder) attempt(delivery store.ClaimedEventDelivery) { ctx, cancel := context.WithTimeout(f.ctx, attemptTimeout) err := f.send(ctx, delivery) cancel() if f.ctx.Err() != nil { // Close cut it short: the lease runs out and a router sends it again. return } writing, done := context.WithTimeout(context.WithoutCancel(f.ctx), attemptTimeout) defer done() log := f.logger.With("connector", delivery.ConnectorID, "customer", delivery.CustomerID, "destination", delivery.DestinationID, "webhook_id", delivery.ID) var again retryable attempts := delivery.Attempts + 1 switch { case err == nil: f.count(delivery, "") err = f.store.FinishEventDelivery(writing, delivery.DestinationID, delivery.ID) case errors.As(err, &again) && attempts <= len(f.retries): log.Info("an event forward failed and is sent again", "attempts", attempts, "error", err) err = f.store.RetryEventDelivery(writing, delivery.DestinationID, delivery.ID, attempts, time.Now().Add(f.retries[attempts-1])) default: log.Warn("gave up an event forward", "attempts", attempts, "error", err) f.count(delivery, givenUpCode(err)) err = f.store.FinishEventDelivery(writing, delivery.DestinationID, delivery.ID) } if err != nil { log.Error("could not record an event forward's attempt", "error", err) } } // count counts a forward that ended in the activity record, once and not per attempt: sent, or // refused with code when it was given up. func (f *Forwarder) count(delivery store.ClaimedEventDelivery, code string) { if f.activity == nil { return } f.activity.Record(activity.Request{ AppID: delivery.CustomerID, Area: activity.AreaWebhooksOut, Client: activity.ClientRouter, At: time.Now(), Refused: code != "", Code: code, }) } // givenUpCode is the code a forward given up after err is counted under. func givenUpCode(err error) string { var again retryable switch { case errors.Is(err, errRefused): return codeRefused case errors.As(err, &again): return codeUnavailable default: return codeNotSent } } // send posts the raw body with the provider's headers and the Standard Webhooks ones. A 2xx is // sent. No answer, a 5xx or a 429 is retryable: a 429 is «Too Many Requests», which the spec // says to throttle, not drop (RFC 6585 section 4). Any other answer, a 3xx or a 4xx, is the // destination refusing it for good, and is not retried. The spec counts a 4xx as a failure to // retry; a destination that answers 400 to a body it cannot read answers the same body the same // way every time, so this router does not send it again. func (f *Forwarder) send(ctx context.Context, delivery store.ClaimedEventDelivery) error { secrets, err := f.openSecrets(delivery) if err != nil { return err } at := time.Now() signature, err := sign(delivery.ID, at, delivery.Body, secrets...) if err != nil { return err } request, err := http.NewRequestWithContext(ctx, http.MethodPost, delivery.URL, bytes.NewReader(delivery.Body)) if err != nil { return stack.Wrap(err) } // Past the provider's own window its signature fails a receiver that checks it, Slack Bolt // among them (ProviderEvent), so an attempt sent then carries Content-Type alone of them, // and the receiver verifies webhook-signature: no stale signature that a receiver which // skips the timestamp check would take as Slack's, sent hours after Slack signed it. late := delivery.ProviderHeadersUntil != nil && at.After(*delivery.ProviderHeadersUntil) for name, value := range delivery.Headers { if late && name != "Content-Type" { continue } request.Header.Set(name, value) } request.Header.Set(headerID, delivery.ID) request.Header.Set(headerTimestamp, strconv.FormatInt(at.Unix(), 10)) request.Header.Set(headerSignature, signature) response, err := f.http.Do(request) if err != nil { return retryable{stack.Wrap(err)} } defer response.Body.Close() //nolint:errcheck // the answer is drained and dropped _, _ = io.Copy(io.Discard, io.LimitReader(response.Body, maxAnswerBytes)) switch { case response.StatusCode >= 200 && response.StatusCode <= 299: return nil case response.StatusCode >= http.StatusInternalServerError || response.StatusCode == http.StatusTooManyRequests: return retryable{stack.Wrap(fmt.Errorf("eventforward: the destination answered %d", response.StatusCode))} default: return stack.Wrap(fmt.Errorf("%w with %d", errRefused, response.StatusCode)) } } // openSecrets are the secrets a send is signed with: the destination's, and the one its last // rotation replaced while that still signs. func (f *Forwarder) openSecrets(delivery store.ClaimedEventDelivery) ([]string, error) { aad := secretAAD(delivery.CustomerID, delivery.ConnectorID, delivery.DestinationID) current, err := f.secrets.OpenWithAADVersion(delivery.SecretSealed, aad, delivery.KEKVersion) if err != nil { return nil, stack.Wrap(fmt.Errorf("eventforward: the signing secret of destination %s does not open: %w", delivery.DestinationID, err)) } secrets := []string{current} if len(delivery.PreviousSecretSealed) > 0 && delivery.PreviousUntil != nil && time.Now().Before(*delivery.PreviousUntil) { previous, err := f.secrets.OpenWithAADVersion(delivery.PreviousSecretSealed, aad, delivery.PreviousKEKVersion) if err != nil { return nil, stack.Wrap(fmt.Errorf("eventforward: the previous signing secret of destination %s does not open: %w", delivery.DestinationID, err)) } secrets = append(secrets, previous) } return secrets, nil } // retryable is a send a later attempt can get past: no answer, a 5xx or a 429. type retryable struct{ err error } func (r retryable) Error() string { return r.err.Error() } func (r retryable) Unwrap() error { return r.err } // secretAAD binds a sealed signing secret to its customer, connector and destination, so a blob // copied onto another destination's row does not open. Each part is length-prefixed, as // api.oauthClientAAD's are, with a prefix of its own, so no other secret's blob opens as this // one. v1 changes with the layout. func secretAAD(customerID, connectorID, id string) []byte { return fmt.Appendf(nil, "accelerate:connector-event-destination-secret:v1:%d:%s:%d:%s:%d:%s", len(customerID), customerID, len(connectorID), connectorID, len(id), id) }