whereHas('component')->where('enabled', true)->get() as $check) { if (! $check->isDue($now) || ! $check->component?->enabled) { continue; } $this->runOne($check, $now); $ran++; } return $ran; } public function runOne(Check $check, ?\DateTimeInterface $now = null): ProbeResult { $now ??= now(); $result = app(ProbeQuorum::class)->combine($check, $this->probe->run($check)); if ($result->inconclusive) { if ($check->component->status !== ComponentStatus::UnderMaintenance) { $check->component->update(['status' => ComponentStatus::PerformanceIssues]); } if ($check->type !== CheckType::Heartbeat) { $check->update(['last_run_at' => $now]); } return $result; } if ($check->type === CheckType::Http) { $check->tls_expires_at = $result->tlsExpiresAt; $check->tls_warning = $result->tlsExpiresAt && $result->tlsExpiresAt <= now()->addDays(14) ? $result->message : null; } // Read before this run's own result is written: it is the reference for how // much wall time this evaluation stands for. $previousRun = CheckResult::where('component_id', $check->component_id)->max('checked_at'); CheckResult::create([ 'component_id' => $check->component_id, 'ok' => $result->ok, 'latency_ms' => $result->latencyMs, 'message' => $result->message, 'checked_at' => $now, ]); if ($result->ok) { $check->consecutive_successes++; $check->consecutive_failures = 0; } else { $check->consecutive_failures++; $check->consecutive_successes = 0; } // A heartbeat's last_run_at is owned by whoever pings in, not by the runner. if ($check->type !== CheckType::Heartbeat) { $check->last_run_at = $now; } $check->save(); $this->recordUptime($check, $result, $now, $previousRun); $this->applyStatus($check, $result, $now); return $result; } protected function recordUptime(Check $check, ProbeResult $result, \DateTimeInterface $now, ?string $previousRun): void { $now = Carbon::instance($now)->utc(); $dayStart = $now->copy()->startOfDay(); // Preserve the existing observation cap and UTC midnight boundary. $since = $previousRun ? Carbon::parse($previousRun, 'UTC') : $now->copy()->subSeconds($check->interval_seconds); $since = $since->max($dayStart)->max($now->copy()->subSeconds($check->interval_seconds)); $seconds = (int) max(0, $since->diffInSeconds($now)); $windows = Maintenance::whereNotNull('started_at')->where('starts_at', '<', $now)->where('ends_at', '>', $since) ->whereHas('components', fn ($q) => $q->whereKey($check->component_id)) ->with(['components' => fn ($q) => $q->whereKey($check->component_id)])->limit(2001)->get(); if ($windows->count() > 2000) { return; // Too many windows to establish accurate eligible credit; keep raw evidence only. } $excluded = MaintenanceIntervals::excluded($windows, $check->component_id, $since, $now); // A manually held maintenance state has no reliable earlier boundary. if ($check->component->status === ComponentStatus::UnderMaintenance && $excluded === 0) { return; } $seconds = max(0, $seconds - $excluded); if ($seconds === 0) { return; } $day = UptimeDay::firstOrCreate( ['component_id' => $check->component_id, 'day' => $dayStart], ['up_seconds' => 0, 'down_seconds' => 0, 'worst_status' => ComponentStatus::Operational->value], ); $column = $result->ok ? 'up_seconds' : 'down_seconds'; $day->increment($column, $seconds); if (! $result->ok) { $day->worst_status = ComponentStatus::MajorOutage; $day->save(); } } protected function applyStatus(Check $check, ProbeResult $result, \DateTimeInterface $now): void { $component = $check->component; // Planned work: a probe failing inside a running maintenance window is // expected, and must neither flip the component nor open an incident. if (! $result->ok && $component->status === ComponentStatus::UnderMaintenance && Maintenance::holds($component->id)) { return; } if (! $result->ok && $check->consecutive_failures >= $check->retries) { if ($component->status !== ComponentStatus::MajorOutage) { $component->update(['status' => ComponentStatus::MajorOutage]); $this->openIncident($check, $result, $now); } return; } if (! $result->ok) { return; } // A passing check owns the status — including a hand-set "Degraded", which // the form says it will overwrite. Under maintenance is the one deliberate // state a probe cannot judge, so that stays until someone clears it. if (! in_array($component->status, [ComponentStatus::Operational, ComponentStatus::UnderMaintenance], true)) { $component->update(['status' => $result->degraded ? ComponentStatus::PerformanceIssues : ComponentStatus::Operational]); } if ($result->degraded && $component->status === ComponentStatus::Operational) { $component->update(['status' => ComponentStatus::PerformanceIssues]); } // Checked independently of the component status: the first healthy result // already flipped the component back, so nesting this inside that branch // meant the incident never closed. if ($check->consecutive_successes >= $this->recoveryStreak($check)) { $this->resolveIncident($check, $now); } } /** * A heartbeat that calls in has proved it is alive; the streak is hysteresis * against a flapping poll, and applying it here would keep a nightly backup * red for two more days after it demonstrably ran. */ protected function recoveryStreak(Check $check): int { return $check->type === CheckType::Heartbeat ? 1 : self::RECOVERY_STREAK; } protected function openIncident(Check $check, ProbeResult $result, \DateTimeInterface $now): Incident { $incident = DB::transaction(function () use ($check, $result, $now) { $incident = Incident::create([ 'name' => __(':component unreachable', ['component' => $check->component->name], Localization::page()), 'status' => IncidentStatus::Investigating, 'impact' => 'major', 'source' => 'check', 'auto_resolve' => true, 'grouping_key' => 'check:'.$check->component_id, 'occurred_at' => $now, ]); $incident->components()->attach($check->component_id, [ 'status' => ComponentStatus::MajorOutage->value, ]); IncidentUpdate::create([ 'incident_id' => $incident->id, 'status' => IncidentStatus::Investigating, 'message' => $result->message ? __('Automatic check failed: :reason.', ['reason' => __($result->message, [], Localization::page())], Localization::page()) : __('Automatic check failed.', [], Localization::page()), 'automatic' => true, ]); return $incident; }); $this->webhook->incidentChanged($incident->fresh('components'), 'incident.created'); return $incident; } protected function resolveIncident(Check $check, \DateTimeInterface $now): void { $incident = Incident::where('grouping_key', 'check:'.$check->component_id) ->where('auto_resolve', true) ->whereNull('resolved_at') ->latest('occurred_at') ->first(); if (! $incident) { return; } DB::transaction(function () use ($incident, $now, $check) { $incident->update(['status' => IncidentStatus::Resolved, 'resolved_at' => $now]); IncidentUpdate::create([ 'incident_id' => $incident->id, 'status' => IncidentStatus::Resolved, 'message' => __('The component responded normally again for :count consecutive checks.', ['count' => $this->recoveryStreak($check)], Localization::page()), 'automatic' => true, ]); }); $this->webhook->incidentChanged($incident->fresh('components'), 'incident.updated'); } }