page(); if (! $page->is_published || $page->archived_at !== null) { return; } $category = self::category($incident, $event); $endpoints = WebhookEndpoint::where('enabled', true)->get()->filter(fn (WebhookEndpoint $e) => $e->wants($category)); foreach ($endpoints as $endpoint) { $key = hash('sha256', implode(':', [$incident->id, $event, $incident->updates()->max('id') ?? 0, $incident->status->value, $incident->updated_at?->format('U.u')])); WebhookDelivery::firstOrCreate(['webhook_endpoint_id' => $endpoint->id, 'event_key' => $key], [ 'status_page_id' => $endpoint->status_page_id, 'event' => $category, // Read by someone else: always the installation zone. 'payload' => Clock::withInstallationZone(fn () => $this->payload($endpoint->format, $incident, $event)), 'next_attempt_at' => now(), ]); } } /** * Planned work: maintenance.scheduled, .started, .completed or .cancelled. * Only destinations that chose Maintenance receive it; ones saved before the * choice existed receive everything, as they always did. */ public function maintenanceChanged(Maintenance $maintenance, string $event): void { $page = app(PageContext::class)->page(); if (! $page->is_published || $page->archived_at !== null) { return; } foreach (WebhookEndpoint::where('enabled', true)->get()->filter(fn (WebhookEndpoint $e) => $e->wants('maintenance')) as $endpoint) { WebhookDelivery::firstOrCreate( ['webhook_endpoint_id' => $endpoint->id, 'event_key' => hash('sha256', 'maintenance:'.$maintenance->id.':'.$event)], [ 'status_page_id' => $endpoint->status_page_id, 'event' => 'maintenance', 'payload' => Clock::withInstallationZone(fn () => $this->maintenancePayload($endpoint->format, $maintenance, $event)), 'next_attempt_at' => now(), ], ); } } /** Which WebhookEndpoint::EVENTS entry an incident event belongs to. */ public static function category(Incident $incident, string $event): string { if ($event === 'incident.created') { return 'incident.opened'; } return $incident->resolved_at !== null ? 'incident.resolved' : 'incident.updated'; } /** A bounded batch fits inside the lock and shared-hosting execution limits. */ public function sendPending(): int { $lock = Cache::lock('pharos:webhook-outbox', 180); if (! $lock->get()) { return 0; } $sent = 0; try { $due = WebhookDelivery::whereNull('sent_at')->where('attempts', '<', 6) ->where('next_attempt_at', '<=', now())->oldest('id')->limit(10)->get(); foreach ($due as $delivery) { $sent += app(PageContext::class)->run($delivery->status_page_id, function () use ($delivery) { $page = app(PageContext::class)->page(); if (! $page->is_published || $page->archived_at !== null) { $delivery->update(['attempts' => 6, 'error' => 'Destination page is unpublished or archived']); return 0; } $delivery->load('endpoint'); $endpoint = $delivery->endpoint; if (! $endpoint?->enabled) { $delivery->update(['attempts' => 6, 'error' => 'Destination disabled or removed']); return 0; } $delivery->increment('attempts'); $ok = $this->deliver($endpoint, $delivery->payload); $status = $endpoint->last_status; $permanent = $status >= 400 && $status < 500 && ! in_array($status, [408, 429], true); $delivery->update([ 'sent_at' => $ok ? now() : null, 'attempts' => $permanent ? 6 : $delivery->attempts, 'last_status' => $status, 'error' => $endpoint->last_error, 'next_attempt_at' => now()->addSeconds($this->retryAfter ?? min(3600, 60 * (2 ** $delivery->attempts))), ]); return (int) $ok; }); } WebhookDelivery::where('created_at', '<', now()->subDays(30))->where(fn ($q) => $q->whereNotNull('sent_at')->orWhere('attempts', '>=', 6))->delete(); } finally { $lock->release(); } return $sent; } /** One fake incident, so an operator can prove the wiring before an outage does. */ public function test(WebhookEndpoint $endpoint): bool { $incident = new Incident([ 'name' => 'Test notification from Pharos', 'status' => IncidentStatus::Investigating, 'impact' => 'minor', 'occurred_at' => now(), ]); $incident->id = 0; $incident->setRelation('components', collect()); return $this->deliver($endpoint, Clock::withInstallationZone(fn () => $this->payload($endpoint->format, $incident, 'incident.test'))); } /** @return array */ protected function payload(string $format, Incident $incident, string $event): array { return match ($format) { 'slack' => $this->slack($incident, $event), 'teams' => $this->teams($incident, $event), 'discord' => ['content' => Str::limit(($incident->resolved_at ? 'Resolved: ' : 'Incident: ').$incident->name, 1800)."\n".PageUrls::route('status'), 'allowed_mentions' => ['parse' => []]], 'telegram' => ['text' => Str::limit($incident->status->label().': '.$incident->name, 1800)."\n".PageUrls::route('status')], 'signal' => ['message' => Str::limit(($incident->resolved_at ? 'Resolved: ' : 'Incident: ').$incident->name, 1800)."\n".PageUrls::route('status')], default => $this->generic($incident, $event), }; } /** @return array */ protected function maintenancePayload(string $format, Maintenance $maintenance, string $event): array { $headline = match ($event) { 'maintenance.started' => 'Maintenance started', 'maintenance.completed' => 'Maintenance completed', 'maintenance.cancelled' => 'Maintenance cancelled', default => 'Maintenance scheduled', }; $when = $maintenance->starts_at->format('j M H:i').' to '.$maintenance->ends_at->format('j M H:i T'); $affects = $maintenance->components->pluck('name')->join(', '); $line = Str::limit($headline.': '.$maintenance->title, 1800); $link = PageUrls::route('status'); return match ($format) { 'slack' => [ 'text' => $line, 'blocks' => [ ['type' => 'section', 'text' => ['type' => 'mrkdwn', 'text' => ':wrench: *'.$headline.'*: '.$maintenance->title]], ['type' => 'context', 'elements' => [['type' => 'mrkdwn', 'text' => implode(' · ', array_filter([$when, $affects !== '' ? 'Affects '.$affects : '', Setting::get('brand.name', 'Pharos')]))]]], ], ], 'teams' => ['type' => 'message', 'attachments' => [[ 'contentType' => 'application/vnd.microsoft.card.adaptive', 'content' => [ '$schema' => 'http://adaptivecards.io/schemas/adaptive-card.json', 'type' => 'AdaptiveCard', 'version' => '1.4', 'body' => [ ['type' => 'TextBlock', 'size' => 'Medium', 'weight' => 'Bolder', 'wrap' => true, 'color' => 'Accent', 'text' => $line], ['type' => 'FactSet', 'facts' => array_values(array_filter([ ['title' => 'When', 'value' => $when], $affects !== '' ? ['title' => 'Affected', 'value' => $affects] : null, ['title' => 'Status page', 'value' => Setting::get('brand.name', 'Pharos')], ]))], ], ], ]]], 'discord' => ['content' => $line."\n".$when."\n".$link, 'allowed_mentions' => ['parse' => []]], 'telegram' => ['text' => $line."\n".$when."\n".$link], 'signal' => ['message' => $line."\n".$when."\n".$link], default => [ 'event' => $event, 'maintenance' => [ 'id' => $maintenance->id, 'title' => $maintenance->title, 'message' => $maintenance->message, 'starts_at' => $maintenance->starts_at->toIso8601String(), 'ends_at' => $maintenance->ends_at->toIso8601String(), 'components' => $maintenance->components->pluck('name')->all(), ], ], }; } /** @return array */ protected function generic(Incident $incident, string $event): array { return [ 'event' => $event, 'incident' => [ 'id' => $incident->id, 'name' => $incident->name, 'status' => $incident->status->name, 'impact' => $incident->impact->value, 'occurred_at' => $incident->occurred_at?->toIso8601String(), 'resolved_at' => $incident->resolved_at?->toIso8601String(), 'components' => $incident->components->pluck('name')->all(), ], ]; } /** @return array */ protected function slack(Incident $incident, string $event): array { $resolved = $incident->resolved_at !== null; // text is the notification and the fallback; blocks are what is read. return [ 'text' => ($resolved ? 'Resolved: ' : 'Incident: ').$incident->name, 'blocks' => [ [ 'type' => 'section', 'text' => [ 'type' => 'mrkdwn', 'text' => ($resolved ? ':white_check_mark: *Resolved*: ' : ':rotating_light: *'.$incident->status->label().'*: ') .$incident->name, ], ], [ 'type' => 'context', 'elements' => [[ 'type' => 'mrkdwn', 'text' => implode(' · ', array_filter([ ucfirst($incident->impact->value).' impact', $this->componentLine($incident), Setting::get('brand.name', 'Pharos'), ])), ]], ], ], ]; } /** * The Adaptive Card envelope, which is what a Teams Workflow accepts. The * old Office 365 connector took a MessageCard instead; that route is being * retired, so this deliberately targets the one with a future. * * @return array */ protected function teams(Incident $incident, string $event): array { $resolved = $incident->resolved_at !== null; return [ 'type' => 'message', 'attachments' => [[ 'contentType' => 'application/vnd.microsoft.card.adaptive', 'content' => [ '$schema' => 'http://adaptivecards.io/schemas/adaptive-card.json', 'type' => 'AdaptiveCard', 'version' => '1.4', 'body' => [ [ 'type' => 'TextBlock', 'size' => 'Medium', 'weight' => 'Bolder', 'wrap' => true, 'color' => $resolved ? 'Good' : 'Attention', 'text' => ($resolved ? 'Resolved: ' : $incident->status->label().': ').$incident->name, ], [ 'type' => 'FactSet', 'facts' => array_values(array_filter([ ['title' => 'Impact', 'value' => ucfirst($incident->impact->value)], $incident->components->isNotEmpty() ? ['title' => 'Affected', 'value' => $incident->components->pluck('name')->join(', ')] : null, ['title' => 'Since', 'value' => $incident->occurred_at?->toDayDateTimeString() ?? 'Not set'], ['title' => 'Status page', 'value' => Setting::get('brand.name', 'Pharos')], ])), ], ], ], ]], ]; } protected function componentLine(Incident $incident): string { return $incident->components->isEmpty() ? '' : 'Affects '.$incident->components->pluck('name')->join(', '); } /** @param array $payload */ protected function deliver(WebhookEndpoint $endpoint, array $payload): bool { $this->retryAfter = null; if ($endpoint->format === 'signal') { $payload += ['number' => $endpoint->options['number'], 'recipients' => [$endpoint->options['recipient']]]; } if ($endpoint->format === 'telegram') { $payload['chat_id'] = $endpoint->options['chat_id']; } $body = json_encode($payload, JSON_UNESCAPED_SLASHES); $headers = ['Content-Type' => 'application/json']; // Only the generic shape is signed. Slack and Teams drop unknown headers, // and their URL is the credential in the first place. if ($endpoint->format === 'generic') { $secret = Setting::get('integrations.webhook_secret', ''); $headers['X-Pharos-Signature'] = hash_hmac('sha256', $body, $secret); } if ($endpoint->format === 'signal') { $headers['Authorization'] = 'Bearer '.$endpoint->options['token']; } try { // Vetted again here, not only when the endpoint was saved: a name can // be re-pointed at the metadata service in between (DNS rebinding). $response = $this->safe->toOwn($endpoint->url)->timeout(5)->connectTimeout(5)->withHeaders($headers) ->withBody($body, 'application/json')->post($endpoint->url); $successful = $response->successful() && ($endpoint->format !== 'telegram' || $response->json('ok') === true); $retry = $endpoint->format === 'telegram' ? ($response->json('parameters.retry_after') ?? $response->header('Retry-After')) : $response->header('Retry-After'); if ($retry !== '') { $seconds = is_numeric($retry) ? (int) $retry : max(0, (strtotime($retry) ?: time()) - time()); $this->retryAfter = min(86400, max(60, $seconds)); } $endpoint->forceFill([ 'last_status' => $response->status(), // The status is what diagnoses it. The body is the receiver's, // could be anything, and would be stored and shown in the admin. 'last_error' => $successful ? null : ($response->successful() ? 'Telegram rejected the message.' : "HTTP {$response->status()}"), 'last_attempt_at' => now(), ])->save(); return $successful; } catch (\Throwable $e) { Log::warning('Outgoing webhook failed', ['endpoint' => $endpoint->id, 'type' => get_class($e)]); $endpoint->forceFill([ 'last_status' => null, 'last_error' => str_contains($e->getMessage(), 'never allowed') ? 'Destination is never allowed.' : 'Connection failed; check the address, DNS, TLS and receiver.', 'last_attempt_at' => now(), ])->save(); return false; } } }