package health import ( "time" appsv1 "k8s.io/api/apps/v1" batchv1 "k8s.io/api/batch/v1" corev1 "k8s.io/api/core/v1" "github.com/skyhook-io/radar/pkg/cronsched" ) // Workload classifies a controller / batch / storage object into a canonical // Verdict. Kinds it doesn't handle return LevelUnknown so the caller can fall // back. This subsumes the four divergent replica/job/pvc classifiers that lived // in pkg/topology, pkg/packages, pkg/resourcecontext, and internal/k8s. // // The semantics align with the frontend's status badges (the most // operator-considered surface): intentional/lifecycle states are neutral, not // degraded or unknown. Notable reconciliations vs the old backend copies: // - desired==0 (scaled to zero) → neutral (was unknown/healthy, depending on copy) // - a running Job → neutral (in-progress, not "healthy"; was healthy) // - a Pending PVC → neutral (WaitForFirstConsumer is benign; topology said degraded) func Workload(obj any, now time.Time) Verdict { switch o := obj.(type) { case *appsv1.Deployment: return replicaVerdict(specReplicas(o.Spec.Replicas), o.Status.ReadyReplicas, o.Status.AvailableReplicas, true, converging(o.Generation, o.Status.ObservedGeneration)) case *appsv1.ReplicaSet: return replicaVerdict(specReplicas(o.Spec.Replicas), o.Status.ReadyReplicas, 0, false, converging(o.Generation, o.Status.ObservedGeneration)) case *appsv1.StatefulSet: return replicaVerdict(specReplicas(o.Spec.Replicas), o.Status.ReadyReplicas, 0, false, converging(o.Generation, o.Status.ObservedGeneration)) case *appsv1.DaemonSet: // DesiredNumberScheduled 0 means the selector matches no nodes — benign, // nothing to run, not unhealthy. return replicaVerdict(o.Status.DesiredNumberScheduled, o.Status.NumberReady, 0, false, converging(o.Generation, o.Status.ObservedGeneration)) case *batchv1.Job: return jobVerdict(o) case *batchv1.CronJob: return cronJobVerdict(o, now) case *corev1.PersistentVolumeClaim: return pvcVerdict(o) } return Verdict{Level: LevelUnknown} } // replicaVerdict grades a replica-based workload. requireAvailable additionally // demands available==desired (Deployment tracks availability; ReplicaSet / // StatefulSet / DaemonSet don't expose it the same way). func replicaVerdict(desired, ready, available int32, requireAvailable, converging bool) Verdict { if desired == 0 { return Verdict{Level: LevelNeutral, Reason: "ScaledToZero"} } if ready >= desired && (!requireAvailable || available >= desired) { return Verdict{Level: LevelHealthy} } // Convergence grace. Without it every ordinary rollout and every scale-up // grades as impaired for its whole duration — a Deployment scaled 3->10 // reads NoneReady/PartiallyReady the instant the spec changes, before the // controller has created a single pod, which is controller lag rather than // an outage. // // Neutral, never Healthy: we are declining to call it broken, not claiming // it is fine. And this only softens the WORKLOAD verdict — a pod that is // genuinely failing (ImagePullBackOff, CrashLoopBackOff) is classified // unhealthy on its own by Pod(), so real breakage during the window still // surfaces; it just surfaces on the pod rather than as a premature verdict // on the controller. if converging { return Verdict{Level: LevelNeutral, Reason: "Converging"} } if ready > 0 { return Verdict{Level: LevelDegraded, Reason: "PartiallyReady"} } return Verdict{Level: LevelUnhealthy, Reason: "NoneReady"} } // converging reports whether the controller has not yet acted on the workload's // current spec, so grading against that spec would report controller lag as an // outage. This is the scale-up case: spec says 10, the controller is still // working from generation N-1 and has created 3, so ready-vs-desired looks like // a 70% outage for as long as the reconcile takes. // // Deliberately NOT age-based. An earlier revision also granted grace to any // workload created within five minutes, which softened genuinely broken // workloads: an unschedulable Deployment at 0/3 read neutral while the object's // own issues said [critical] workload_degraded. Generation skew is evidence — // the controller has demonstrably not looked at this spec yet. "Created // recently" is only a guess, and it guesses wrong exactly when a deploy is // broken on arrival, which is when it matters most. func converging(generation, observedGeneration int64) bool { return observedGeneration > 0 && observedGeneration < generation } func jobVerdict(j *batchv1.Job) Verdict { // Terminal conditions dominate spec intent: a Job that has failed or completed // must not be masked by spec.suspend (a failed-then-suspended Job is still a // failure). Check the terminal conditions before the suspend branch. for _, c := range j.Status.Conditions { if c.Type == batchv1.JobFailed && c.Status == corev1.ConditionTrue { return Verdict{Level: LevelUnhealthy, Reason: "Failed"} } if c.Type == batchv1.JobComplete && c.Status == corev1.ConditionTrue { return Verdict{Level: LevelNeutral, Reason: "Completed"} } } if j.Spec.Suspend != nil && *j.Spec.Suspend { return Verdict{Level: LevelNeutral, Reason: "Suspended"} } if int(j.Status.Succeeded) >= jobDesiredCompletions(j) { return Verdict{Level: LevelNeutral, Reason: "Completed"} } // A failed batch with nothing active and nothing succeeded is a real failure. if j.Status.Failed > 0 && j.Status.Active == 0 && j.Status.Succeeded == 0 { return Verdict{Level: LevelUnhealthy, Reason: "Failed"} } // In-progress: running, not yet complete — intentional/transient, not "healthy". if j.Status.Active > 0 { return Verdict{Level: LevelNeutral, Reason: "Running"} } return Verdict{Level: LevelUnknown} } // jobDesiredCompletions is the number of successful completions a Job needs. // Only a nil completions defaults to 1 (the K8s default for non-indexed Jobs); an // explicit 0 is preserved — Kubernetes treats a zero-completion Job as already // complete (the controller never schedules pods for it), so Succeeded>=0 reading // complete is correct, not a false positive. func jobDesiredCompletions(j *batchv1.Job) int { if j.Spec.Completions != nil { return int(*j.Spec.Completions) } return 1 } func cronJobVerdict(cj *batchv1.CronJob, now time.Time) Verdict { if cj == nil { return Verdict{Level: LevelUnknown} } if cj.Spec.Suspend != nil && *cj.Spec.Suspend { return Verdict{Level: LevelNeutral, Reason: "Suspended"} } // Never run yet: benign for a freshly-created CronJob, but one that has existed // longer than its cadence and still never fired is stuck (bad schedule / broken // controller) — degraded, matching the never-scheduled problem detector. if cj.Status.LastScheduleTime == nil { if now.Sub(cj.CreationTimestamp.Time) > cronsched.StaleThreshold(cj.Spec.Schedule) { return Verdict{Level: LevelDegraded, Reason: "NeverScheduled"} } return Verdict{Level: LevelNeutral, Reason: "NeverRun"} } // Cadence-aware staleness: a job that has missed its schedule by more than the // cadence-relative grace window is degraded. if now.Sub(cj.Status.LastScheduleTime.Time) > cronsched.StaleThreshold(cj.Spec.Schedule) { return Verdict{Level: LevelDegraded, Reason: "Stale"} } // LastScheduleTime records a past scheduling attempt, not current schedule health. return Verdict{Level: LevelNeutral, Reason: "Enabled"} } func pvcVerdict(pvc *corev1.PersistentVolumeClaim) Verdict { switch pvc.Status.Phase { case corev1.ClaimBound: return Verdict{Level: LevelHealthy} case corev1.ClaimPending: // WaitForFirstConsumer / provisioning is benign — not a problem to alarm on. return Verdict{Level: LevelNeutral, Reason: "Pending"} case corev1.ClaimLost: return Verdict{Level: LevelUnhealthy, Reason: "Lost"} } return Verdict{Level: LevelUnknown} } func specReplicas(r *int32) int32 { if r != nil { return *r } return 1 }