package checks import ( "maps" "slices" "strconv" "strings" "k8s.io/apimachinery/pkg/api/resource" "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" "github.com/sophotechlabs/spinoza/internal/unstr" ) const ( hostnameKey = "kubernetes.io/hostname" enforceLabel = "pod-security.kubernetes.io/enforce" profileBaseline = "baseline" profileStrict = "restricted" containerLimits = "Container" doNotSchedule = "DoNotSchedule" ) var nodeTarget = target{group: "", resource: "nodes"} var namespaceTarget = target{group: "", resource: "namespaces"} var quotaTarget = target{group: "", resource: "resourcequotas"} var limitRangeTarget = target{group: "", resource: "limitranges"} func factChecks() []check { return []check{ { id: "node-selector-matches-nothing", title: "Scheduled onto a node that does not exist", category: categoryReliability, severity: severityHigh, needs: []target{nodeTarget}, wrong: "No node in the cluster carries the labels the pod asks for, so it can never be scheduled.", remedy: "Fix the label in nodeSelector, or label the nodes it is meant to land on.", find: overFacts(nodeSelectorMatchesNothing), }, { id: "tolerations-miss-the-taints", title: "Every matching node repels the pod", category: categoryReliability, severity: severityHigh, needs: []target{nodeTarget}, wrong: "The nodes whose labels match all carry a taint the pod does not tolerate, so nothing will accept it.", remedy: "Add the toleration the taint calls for, or take the taint off a node that should run this.", find: overFacts(tolerationsMissTaints), }, { id: "request-exceeds-largest-node", title: "Asks for more than any node has", category: categoryReliability, severity: severityHigh, needs: []target{nodeTarget}, wrong: "The request is larger than the allocatable capacity of the biggest node, so no amount of free space will let it schedule.", remedy: "Lower the request, or add a node large enough to hold it.", find: overFacts(requestExceedsLargestNode), }, { id: "spread-needs-more-domains", title: "Spread asks for more failure domains than exist", category: categoryReliability, severity: severityMedium, needs: []target{nodeTarget}, wrong: "A DoNotSchedule constraint cannot be satisfied with fewer domains than replicas, so the surplus replicas stay Pending.", remedy: "Use ScheduleAnyway, lower the replica count, or add nodes in more domains.", find: overFacts(spreadNeedsMoreDomains), }, { id: "anti-affinity-exceeds-nodes", title: "Anti-affinity asks for more nodes than exist", category: categoryReliability, severity: severityMedium, needs: []target{nodeTarget}, wrong: "A required anti-affinity on hostname allows one replica per node, and there are fewer nodes than replicas.", remedy: "Make the rule preferred, lower the replica count, or add nodes.", find: overFacts(antiAffinityExceedsNodes), }, { id: "outside-limit-range", title: "Resources outside the namespace's LimitRange", category: categoryReliability, severity: severityHigh, needs: []target{limitRangeTarget}, wrong: "The namespace's LimitRange rejects the pod at admission, so the controller creates it and never gets one.", remedy: "Bring the request and limit inside the range, or widen the LimitRange.", find: overFacts(outsideLimitRange), }, { id: "quota-nearly-exhausted", title: "Namespace quota almost spent", category: categoryReliability, severity: severityMedium, needs: []target{quotaTarget}, wrong: "Once a ResourceQuota is full the next pod is refused at admission, which shows up as a controller that cannot create its pods.", remedy: "Raise the quota, or lower what the workloads in this namespace reserve.", find: overFacts(quotaNearlyExhausted), }, { id: "pod-security-would-reject", enforced: true, title: "The namespace's Pod Security level would reject this", category: categorySecurity, severity: severityHigh, frameworks: []string{pssBaseline, pssRestricted}, needs: []target{namespaceTarget}, wrong: "The namespace enforces a Pod Security level this pod does not meet, so the pods are refused at admission.", remedy: "Fix what the level forbids, or move the workload to a namespace at a lower level.", find: enforcedLevelRejects, }, } } func nodeLabels(node *unstructured.Unstructured) map[string]string { return node.GetLabels() } func selectorOf(subject Subject) map[string]string { raw, ok := subject.Pod["nodeSelector"].(map[string]any) if !ok { return nil } out := map[string]string{} for key, value := range raw { text, isString := value.(string) if !isString { continue } out[key] = text } return out } func nodesMatching(subject Subject, held *corpus) []*unstructured.Unstructured { wanted := selectorOf(subject) out := []*unstructured.Unstructured{} for _, node := range held.of("", "nodes") { labels := nodeLabels(node) fits := true for key, value := range wanted { if labels[key] != value { fits = false break } } if fits { out = append(out, node) } } return out } func nodeSelectorMatchesNothing(subject Subject, held *corpus) (string, string) { wanted := selectorOf(subject) if len(wanted) == 0 || len(held.of("", "nodes")) == 0 { return "", "" } if len(nodesMatching(subject, held)) > 0 { return "", "" } pairs := make([]string, 0, len(wanted)) for key, value := range wanted { pairs = append(pairs, key+"="+value) } slices.Sort(pairs) return "no node carries " + strings.Join(pairs, ", "), "" } type taint struct { key string value string effect string } func taintsOf(node *unstructured.Unstructured) []taint { out := []taint{} for _, raw := range unstr.Slice(node, specField, "taints") { entry, ok := raw.(map[string]any) if !ok { continue } effect := stringAt(entry, "effect") if effect == "PreferNoSchedule" { continue } out = append(out, taint{ key: stringAt(entry, "key"), value: stringAt(entry, "value"), effect: effect, }) } return out } func tolerates(subject Subject, one taint) bool { listed, ok := subject.Pod["tolerations"].([]any) if !ok { return false } for _, raw := range listed { entry, isMap := raw.(map[string]any) if !isMap { continue } if !tolerationCovers(entry, one) { continue } return true } return false } func tolerationCovers(entry map[string]any, one taint) bool { effect := stringAt(entry, "effect") if effect != "" && effect != one.effect { return false } operator := stringAt(entry, "operator") key := stringAt(entry, "key") if operator == "Exists" { return key == "" || key == one.key } if key != one.key { return false } return stringAt(entry, "value") == one.value } func tolerationsMissTaints(subject Subject, held *corpus) (string, string) { nodes := nodesMatching(subject, held) if len(nodes) == 0 { return "", "" } blocking := "" for _, node := range nodes { repelled := false for _, one := range taintsOf(node) { if tolerates(subject, one) { continue } repelled = true blocking = one.key break } if !repelled { return "", "" } } return "every node it could use is tainted " + blocking + ", which it does not tolerate", "" } func allocatableOf(node *unstructured.Unstructured, name string) (resource.Quantity, bool) { raw, found, err := unstructured.NestedFieldNoCopy(node.Object, "status", "allocatable", name) if !found || err != nil { return resource.Quantity{}, false } return quantityFrom(raw) } func largestNode(held *corpus, name string, scale func(resource.Quantity) int64) int64 { var biggest int64 for _, node := range held.of("", "nodes") { value, ok := allocatableOf(node, name) if !ok { continue } if scaled := scale(value); scaled > biggest { biggest = scaled } } return biggest } func requestExceedsLargestNode(subject Subject, held *corpus) (string, string) { for _, entry := range []struct { name string unit string scale func(resource.Quantity) int64 }{ {name: cpuName, unit: "m", scale: milliOf}, {name: memoryName, unit: " bytes", scale: plainOf}, } { biggest := largestNode(held, entry.name, entry.scale) if biggest == 0 { continue } for _, container := range subject.Containers { asked, ok := quantityAt(container.Spec, requests, entry.name) if !ok { continue } if entry.scale(asked) <= biggest { continue } return "container " + container.Name + " requests " + asked.String() + " " + entry.name + " and the largest node has " + strconv.FormatInt(biggest, 10) + entry.unit, "" } } return "", "" } func spreadConstraints(subject Subject) []map[string]any { out := []map[string]any{} listed, ok := subject.Pod["topologySpreadConstraints"].([]any) if !ok { return out } for _, raw := range listed { entry, isMap := raw.(map[string]any) if !isMap { continue } out = append(out, entry) } return out } func domainsOf(held *corpus, key string) int { seen := map[string]bool{} for _, node := range held.of("", "nodes") { value, ok := nodeLabels(node)[key] if !ok { continue } seen[value] = true } return len(seen) } func spreadNeedsMoreDomains(subject Subject, held *corpus) (string, string) { if subject.Replicas < 2 || len(held.of("", "nodes")) == 0 { return "", "" } for _, entry := range spreadConstraints(subject) { if stringAt(entry, "whenUnsatisfiable") != doNotSchedule { continue } key := stringAt(entry, "topologyKey") if key == "" { continue } skew, hasSkew := numberAt(entry, "maxSkew") if !hasSkew { skew = 1 } domains := int64(domainsOf(held, key)) if domains == 0 || subject.Replicas <= domains*skew { continue } return strconv.FormatInt(subject.Replicas, 10) + " replicas spread over " + key + ", and the cluster has " + strconv.FormatInt(domains, 10) + " of those", "" } return "", "" } func requiredAntiAffinity(subject Subject) map[string]any { affinity, ok := subject.Pod["affinity"].(map[string]any) if !ok { return nil } anti, ok := affinity["podAntiAffinity"].(map[string]any) if !ok { return nil } listed, ok := anti["requiredDuringSchedulingIgnoredDuringExecution"].([]any) if !ok || len(listed) == 0 { return nil } entry, isMap := listed[0].(map[string]any) if !isMap { return nil } return entry } func antiAffinityExceedsNodes(subject Subject, held *corpus) (string, string) { rule := requiredAntiAffinity(subject) if rule == nil || subject.Replicas < 2 { return "", "" } if stringAt(rule, "topologyKey") != hostnameKey { return "", "" } nodes := int64(len(nodesMatching(subject, held))) if nodes == 0 || subject.Replicas <= nodes { return "", "" } return strconv.FormatInt(subject.Replicas, 10) + " replicas, one node each, and " + strconv.FormatInt(nodes, 10) + " nodes to put them on", "" } func containerRanges(held *corpus, namespace string) []map[string]any { out := []map[string]any{} for _, obj := range held.inNamespace("", "limitranges", namespace) { for _, raw := range unstr.Slice(obj, specField, "limits") { entry, ok := raw.(map[string]any) if !ok || stringAt(entry, "type") != containerLimits { continue } out = append(out, entry) } } return out } func outsideLimitRange(subject Subject, held *corpus) (string, string) { for _, entry := range containerRanges(held, subject.Ref.Namespace) { for _, container := range subject.Containers { if detail := breaksRange(container, entry); detail != "" { return "container " + container.Name + " " + detail, "" } } } return "", "" } func breaksRange(container Container, entry map[string]any) string { for _, name := range []string{cpuName, memoryName} { asked, ok := quantityAt(container.Spec, requests, name) if !ok { continue } if floor, has := boundOf(entry, "min", name); has && asked.Cmp(floor) < 0 { return "requests " + asked.String() + " " + name + ", below the LimitRange minimum " + floor.String() } if ceiling, has := boundOf(entry, "max", name); has && asked.Cmp(ceiling) > 0 { return "requests " + asked.String() + " " + name + ", above the LimitRange maximum " + ceiling.String() } } return "" } func boundOf(entry map[string]any, section, name string) (resource.Quantity, bool) { part, ok := entry[section].(map[string]any) if !ok { return resource.Quantity{}, false } return quantityFrom(part[name]) } func quotaNearlyExhausted(subject Subject, held *corpus) (string, string) { const nearly = 90 for _, quota := range held.inNamespace("", "resourcequotas", subject.Ref.Namespace) { hard, okHard, hardErr := unstructured.NestedFieldNoCopy(quota.Object, "status", "hard") used, okUsed, usedErr := unstructured.NestedFieldNoCopy(quota.Object, "status", "used") if !okHard || !okUsed || hardErr != nil || usedErr != nil { continue } name, share := fullestEntry(hard, used) if share < nearly { continue } return quota.GetName() + " is " + strconv.FormatInt(share, 10) + "% spent on " + name, "" } return "", "" } func fullestEntry(hard, used any) (name string, share int64) { hardMap, okHard := hard.(map[string]any) usedMap, okUsed := used.(map[string]any) if !okHard || !okUsed { return "", 0 } for _, key := range slices.Sorted(maps.Keys(hardMap)) { raw := hardMap[key] ceiling, ok := quantityFrom(raw) if !ok || ceiling.Value() == 0 { continue } spent, ok := quantityFrom(usedMap[key]) if !ok { continue } percent := spent.MilliValue() * 100 / ceiling.MilliValue() if percent > share { name, share = key, percent } } return name, share }