package health import ( "strconv" "time" corev1 "k8s.io/api/core/v1" ) const NodeNetworkUnavailableGrace = 2 * time.Minute const NodeRemovalWarningAfter = 10 * time.Minute const NodeRemovalCriticalAfter = 30 * time.Minute // NodeFleetCounts partitions nodes into exclusive availability buckets. // RemovingUnhealthy is a subset of Removing, not an additional bucket. type NodeFleetCounts struct { Total int `json:"total"` Ready int `json:"ready"` NotReady int `json:"notReady"` Cordoned int `json:"cordoned"` Removing int `json:"removing"` RemovingUnhealthy int `json:"removingUnhealthy"` } func CountNodeFleet(nodes []*corev1.Node, now time.Time) NodeFleetCounts { var counts NodeFleetCounts for _, node := range nodes { if node == nil { continue } counts.Total++ lifecycle := NodeLifecycle(node, now) if lifecycle.Removing && !lifecycle.ReadinessFailed { counts.Removing++ if lifecycle.Level == LevelUnhealthy { counts.RemovingUnhealthy++ } } else if h := Node(node); h.Ready { if h.Unschedulable { counts.Cordoned++ } else { counts.Ready++ } } else { counts.NotReady++ } } return counts } // NodeLifecycleState keeps observed readiness separate from an intentional removal. type NodeLifecycleState struct { Label string `json:"label"` Level Level `json:"level"` Removing bool `json:"removing"` Actor string `json:"actor,omitempty"` StartedAt time.Time `json:"startedAt,omitzero"` ReadinessFailed bool `json:"readinessFailed"` Problems []string `json:"problems,omitempty"` Delayed bool `json:"delayed"` } // NodeLifecycle treats readiness loss as expected only when it follows a known // removal start; existing failures and uncertain timing remain actionable. func NodeLifecycle(node *corev1.Node, now time.Time) NodeLifecycleState { s := NodeLifecycleState{Label: "Unknown", Level: LevelUnknown} var ready *corev1.NodeCondition for i := range node.Status.Conditions { if node.Status.Conditions[i].Type == corev1.NodeReady { ready = &node.Status.Conditions[i] break } } validTime := func(t time.Time) time.Time { if t.IsZero() || t.After(now) || (!node.CreationTimestamp.IsZero() && t.Before(node.CreationTimestamp.Time)) { return time.Time{} } return t } if node.DeletionTimestamp != nil { s.Removing = true s.StartedAt = validTime(node.DeletionTimestamp.Time) } candidate := false for _, taint := range node.Spec.Taints { if taint.Key == "DeletionCandidateOfClusterAutoscaler" && taint.Effect == corev1.TaintEffectPreferNoSchedule { candidate = true } if taint.Effect != corev1.TaintEffectNoSchedule { continue } var actor string switch taint.Key { case "ToBeDeletedByClusterAutoscaler": actor = "cluster autoscaler" case "karpenter.sh/disrupted": actor = "Karpenter" default: continue } s.Removing, s.Actor = true, actor // NoSchedule taints have no automatic timeAdded. Only CA records a // start time in its value; deletionTimestamp dates actual deletion. if taint.Key == "ToBeDeletedByClusterAutoscaler" { if seconds, err := strconv.ParseUint(taint.Value, 10, 63); err == nil && seconds > 0 { start := validTime(time.Unix(int64(seconds), 0)) if !start.IsZero() && (s.StartedAt.IsZero() || start.Before(s.StartedAt)) { s.StartedAt = start } } } } s.ReadinessFailed = ready != nil && ready.Status != corev1.ConditionTrue if s.Removing { s.Label, s.Level = "Removing", LevelNeutral if s.Actor != "" { s.Label += " (" + s.Actor + ")" } if !s.StartedAt.IsZero() { s.Delayed = now.Sub(s.StartedAt) >= NodeRemovalWarningAfter } // A removal requested to repair an existing outage does not explain it. if s.ReadinessFailed && ready != nil && !s.StartedAt.IsZero() && !ready.LastTransitionTime.IsZero() && !ready.LastTransitionTime.Time.Before(s.StartedAt) && !ready.LastTransitionTime.Time.After(now) { s.ReadinessFailed = false } if s.Delayed { s.Level = LevelDegraded if now.Sub(s.StartedAt) >= NodeRemovalCriticalAfter { s.Level = LevelUnhealthy } } if s.ReadinessFailed { s.Level = LevelUnhealthy s.Label += " · NotReady" } } else if s.ReadinessFailed { s.Label, s.Level = "NotReady", LevelUnhealthy } else if ready != nil && ready.Status == corev1.ConditionTrue { s.Label, s.Level = "Ready", LevelHealthy if node.Spec.Unschedulable { s.Label, s.Level = "Cordoned", LevelDegraded } else if candidate { s.Label, s.Level = "Scale-down candidate", LevelNeutral } } for _, cond := range node.Status.Conditions { if cond.Status != corev1.ConditionTrue { continue } name := "" switch cond.Type { case corev1.NodeMemoryPressure: name = "Memory pressure" case corev1.NodeDiskPressure: name = "Disk pressure" case corev1.NodePIDPressure: name = "PID pressure" case corev1.NodeNetworkUnavailable: if !cond.LastTransitionTime.IsZero() && !cond.LastTransitionTime.After(now) && now.Sub(cond.LastTransitionTime.Time) < NodeNetworkUnavailableGrace { continue } name = "Network unavailable" } if name != "" { s.Problems = append(s.Problems, string(cond.Type)) s.Label += " · " + name if cond.Type == corev1.NodeNetworkUnavailable { s.Level = WorseOf(s.Level, LevelDegraded) } else { s.Level = LevelUnhealthy } } } return s }