package audit import ( "slices" "strings" "testing" "time" appsv1 "k8s.io/api/apps/v1" batchv1 "k8s.io/api/batch/v1" corev1 "k8s.io/api/core/v1" storagev1 "k8s.io/api/storage/v1" "k8s.io/apimachinery/pkg/api/resource" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/types" ) func storageInput() *CheckInput { return &CheckInput{ PVCConsumerNamespaces: []string{"app"}, PVDeletionEventsComplete: true, Pods: []*corev1.Pod{}, Deployments: []*appsv1.Deployment{}, ReplicaSets: []*appsv1.ReplicaSet{}, StatefulSets: []*appsv1.StatefulSet{}, DaemonSets: []*appsv1.DaemonSet{}, Jobs: []*batchv1.Job{}, CronJobs: []*batchv1.CronJob{}, PersistentVolumeClaims: []*corev1.PersistentVolumeClaim{{ ObjectMeta: metav1.ObjectMeta{Name: "data-db-0", Namespace: "app", CreationTimestamp: metav1.NewTime(time.Now().Add(-48 * time.Hour))}, Spec: corev1.PersistentVolumeClaimSpec{StorageClassName: ptr("local"), VolumeName: "disk", Resources: corev1.VolumeResourceRequirements{Requests: corev1.ResourceList{corev1.ResourceStorage: resource.MustParse("8Gi")}}}, Status: corev1.PersistentVolumeClaimStatus{Phase: corev1.ClaimBound}, }}, PersistentVolumes: []*corev1.PersistentVolume{{ObjectMeta: metav1.ObjectMeta{Name: "disk", UID: "disk-uid", CreationTimestamp: metav1.NewTime(time.Now().Add(-72 * time.Hour))}, Spec: corev1.PersistentVolumeSpec{Capacity: corev1.ResourceList{corev1.ResourceStorage: resource.MustParse("8Gi")}, StorageClassName: "local", PersistentVolumeReclaimPolicy: corev1.PersistentVolumeReclaimRetain, ClaimRef: &corev1.ObjectReference{Namespace: "app", Name: "data-db-0"}}}}, StorageClasses: []*storagev1.StorageClass{{ObjectMeta: metav1.ObjectMeta{Name: "local"}, VolumeBindingMode: ptr(storagev1.VolumeBindingWaitForFirstConsumer)}}, Events: []*corev1.Event{}, } } func pvcPodSpec(name string) corev1.PodSpec { return corev1.PodSpec{Volumes: []corev1.Volume{{Name: "data", VolumeSource: corev1.VolumeSource{PersistentVolumeClaim: &corev1.PersistentVolumeClaimVolumeSource{ClaimName: name}}}}} } func TestPVCNoConsumer(t *testing.T) { for _, tc := range []struct { name string change func(*CheckInput) want bool }{ {"no consumer", func(*CheckInput) {}, true}, {"Pod", func(i *CheckInput) { i.Pods = append(i.Pods, &corev1.Pod{ObjectMeta: metav1.ObjectMeta{Namespace: "app"}, Spec: pvcPodSpec("data-db-0")}) }, false}, {"terminal Pod", func(i *CheckInput) { i.Pods = append(i.Pods, &corev1.Pod{ObjectMeta: metav1.ObjectMeta{Namespace: "app"}, Spec: pvcPodSpec("data-db-0"), Status: corev1.PodStatus{Phase: corev1.PodSucceeded}}) }, false}, {"generic ephemeral Pod", func(i *CheckInput) { i.Pods = append(i.Pods, &corev1.Pod{ObjectMeta: metav1.ObjectMeta{Namespace: "app", Name: "data-db"}, Spec: corev1.PodSpec{Volumes: []corev1.Volume{{Name: "0", VolumeSource: corev1.VolumeSource{Ephemeral: &corev1.EphemeralVolumeSource{}}}}}}) }, false}, {"StatefulSet direct claim", func(i *CheckInput) { i.StatefulSets = append(i.StatefulSets, &appsv1.StatefulSet{ObjectMeta: metav1.ObjectMeta{Namespace: "app"}, Spec: appsv1.StatefulSetSpec{Template: corev1.PodTemplateSpec{Spec: pvcPodSpec("data-db-0")}}}) }, false}, {"Deployment at zero", func(i *CheckInput) { i.Deployments = append(i.Deployments, &appsv1.Deployment{ObjectMeta: metav1.ObjectMeta{Namespace: "app"}, Spec: appsv1.DeploymentSpec{Replicas: ptr(int32(0)), Template: corev1.PodTemplateSpec{Spec: pvcPodSpec("data-db-0")}}}) }, false}, {"ReplicaSet", func(i *CheckInput) { i.ReplicaSets = append(i.ReplicaSets, &appsv1.ReplicaSet{ObjectMeta: metav1.ObjectMeta{Namespace: "app"}, Spec: appsv1.ReplicaSetSpec{Template: corev1.PodTemplateSpec{Spec: pvcPodSpec("data-db-0")}}}) }, false}, {"DaemonSet", func(i *CheckInput) { i.DaemonSets = append(i.DaemonSets, &appsv1.DaemonSet{ObjectMeta: metav1.ObjectMeta{Namespace: "app"}, Spec: appsv1.DaemonSetSpec{Template: corev1.PodTemplateSpec{Spec: pvcPodSpec("data-db-0")}}}) }, false}, {"StatefulSet generated at zero", func(i *CheckInput) { i.StatefulSets = append(i.StatefulSets, &appsv1.StatefulSet{ObjectMeta: metav1.ObjectMeta{Namespace: "app", Name: "db"}, Spec: appsv1.StatefulSetSpec{Replicas: ptr(int32(0)), VolumeClaimTemplates: []corev1.PersistentVolumeClaim{{ObjectMeta: metav1.ObjectMeta{Name: "data"}}}}}) }, false}, {"StatefulSet retained ordinal", func(i *CheckInput) { i.PersistentVolumeClaims[0].Name = "data-db-12" i.StatefulSets = append(i.StatefulSets, &appsv1.StatefulSet{ObjectMeta: metav1.ObjectMeta{Namespace: "app", Name: "db"}, Spec: appsv1.StatefulSetSpec{Replicas: ptr(int32(0)), VolumeClaimTemplates: []corev1.PersistentVolumeClaim{{ObjectMeta: metav1.ObjectMeta{Name: "data"}}}}}) }, false}, {"not a generated ordinal", func(i *CheckInput) { i.PersistentVolumeClaims[0].Name = "data-db-backup" i.StatefulSets = append(i.StatefulSets, &appsv1.StatefulSet{ObjectMeta: metav1.ObjectMeta{Namespace: "app", Name: "db"}, Spec: appsv1.StatefulSetSpec{VolumeClaimTemplates: []corev1.PersistentVolumeClaim{{ObjectMeta: metav1.ObjectMeta{Name: "data"}}}}}) }, true}, {"Job including complete", func(i *CheckInput) { i.Jobs = append(i.Jobs, &batchv1.Job{ObjectMeta: metav1.ObjectMeta{Namespace: "app"}, Spec: batchv1.JobSpec{Template: corev1.PodTemplateSpec{Spec: pvcPodSpec("data-db-0")}}, Status: batchv1.JobStatus{Conditions: []batchv1.JobCondition{{Type: batchv1.JobComplete, Status: corev1.ConditionTrue}}}}) }, false}, {"CronJob", func(i *CheckInput) { i.CronJobs = append(i.CronJobs, &batchv1.CronJob{ObjectMeta: metav1.ObjectMeta{Namespace: "app"}, Spec: batchv1.CronJobSpec{JobTemplate: batchv1.JobTemplateSpec{Spec: batchv1.JobSpec{Template: corev1.PodTemplateSpec{Spec: pvcPodSpec("data-db-0")}}}}}) }, false}, {"other namespace", func(i *CheckInput) { i.Pods = append(i.Pods, &corev1.Pod{ObjectMeta: metav1.ObjectMeta{Namespace: "other"}, Spec: pvcPodSpec("data-db-0")}) }, true}, {"unreadable Pods", func(i *CheckInput) { i.Pods = nil }, false}, {"unknown coverage", func(i *CheckInput) { i.PVCConsumerNamespaces = nil }, false}, {"CNPG controller owner", func(i *CheckInput) { i.PersistentVolumeClaims[0].OwnerReferences = []metav1.OwnerReference{{APIVersion: "postgresql.cnpg.io/v1", Kind: "Cluster", Name: "hibernated", Controller: ptr(true)}} }, false}, {"non-controller owner", func(i *CheckInput) { i.PersistentVolumeClaims[0].OwnerReferences = []metav1.OwnerReference{{APIVersion: "v1", Kind: "ConfigMap", Name: "metadata"}} }, true}, {"unreadable PVs", func(i *CheckInput) { i.PersistentVolumes = nil }, true}, {"PVs outside scope", func(i *CheckInput) { i.PersistentVolumes = []*corev1.PersistentVolume{} }, true}, {"Pending", func(i *CheckInput) { i.PersistentVolumeClaims[0].Status.Phase = corev1.ClaimPending }, false}, } { t.Run(tc.name, func(t *testing.T) { i := storageInput() tc.change(i) r := RunChecks(i) if got := findingNames(r.Findings, "pvcNoConsumer")[i.PersistentVolumeClaims[0].Name]; got != tc.want { t.Fatalf("finding=%v want=%v: %+v", got, tc.want, r.Findings) } if tc.want { for _, f := range r.Findings { if f.CheckID == "pvcNoConsumer" { for _, s := range []string{"Requested 8Gi", `storage class "local"`, "Bound PV disk", "PVC age", "reclaim policy"} { if !strings.Contains(f.Message, s) { t.Errorf("missing %q in %q", s, f.Message) } } } } } if tc.name == "unreadable PVs" || tc.name == "PVs outside scope" { // The claim is still evaluated; only releasedPV is skipped, and // only when the inventory is unavailable rather than out of scope. if !strings.Contains(r.Findings[0].Message, "reclaim policy not visible") || r.CheckCounts["pvcNoConsumer"].Evaluated != 1 { t.Fatalf("claim finding lost its retention caveat: %+v", r) } if slices.Contains(r.MissingInputs, "persistentvolumes") != (tc.name == "unreadable PVs") { t.Fatalf("PV missing input does not track skipped releasedPV: %v", r.MissingInputs) } } if tc.name == "unknown coverage" && (r.CheckCounts["pvcNoConsumer"].Evaluated != 0 || !slices.Contains(r.MissingInputs, "pvc-consumers")) { t.Fatalf("unknown counted as passing: %+v", r) } }) } } func TestPVCUnusedCondition(t *testing.T) { now := time.Date(2026, time.October, 12, 12, 0, 0, 0, time.UTC) recent := metav1.NewTime(now.Add(-2 * time.Hour)) old := metav1.NewTime(now.Add(-10 * 24 * time.Hour)) evidence := "Kubernetes reports it unused by any Pod since 2026-10-02T12:00:00Z (10d ago)." for _, tc := range []struct { name string status corev1.ConditionStatus since metav1.Time change func(*CheckInput) evaluated, passed int finding bool evidence string missingConsumers bool }{ {name: "False without observed consumer", status: corev1.ConditionFalse, since: recent, evaluated: 1, passed: 1}, {name: "False with unreadable Pods", status: corev1.ConditionFalse, change: func(i *CheckInput) { i.Pods = nil }, evaluated: 1, passed: 1}, {name: "False with unknown coverage", status: corev1.ConditionFalse, change: func(i *CheckInput) { i.PVCConsumerNamespaces = nil }, evaluated: 1, passed: 1}, {name: "True recently unused", status: corev1.ConditionTrue, since: recent}, {name: "True at age floor", status: corev1.ConditionTrue, since: metav1.NewTime(now.Add(-24 * time.Hour))}, {name: "True just past age floor", status: corev1.ConditionTrue, since: metav1.NewTime(now.Add(-24*time.Hour - time.Second)), evaluated: 1, finding: true, evidence: "Kubernetes reports it unused by any Pod since 2026-10-11T11:59:59Z (1d ago)."}, {name: "True future transition", status: corev1.ConditionTrue, since: metav1.NewTime(now.Add(time.Hour))}, {name: "True with unknown coverage during grace", status: corev1.ConditionTrue, since: recent, change: func(i *CheckInput) { i.PVCConsumerNamespaces = nil }}, {name: "True unused for ten days", status: corev1.ConditionTrue, since: old, evaluated: 1, finding: true, evidence: evidence}, {name: "True uses transition instead of creation", status: corev1.ConditionTrue, since: old, change: func(i *CheckInput) { i.PersistentVolumeClaims[0].CreationTimestamp = recent }, evaluated: 1, finding: true, evidence: evidence}, {name: "True zero transition", status: corev1.ConditionTrue, evaluated: 1, finding: true}, {name: "True zero transition on young claim", status: corev1.ConditionTrue, change: func(i *CheckInput) { i.PersistentVolumeClaims[0].CreationTimestamp = recent }}, {name: "True with zero replica StatefulSet template", status: corev1.ConditionTrue, since: old, change: func(i *CheckInput) { i.StatefulSets = []*appsv1.StatefulSet{{ObjectMeta: metav1.ObjectMeta{Namespace: "app", Name: "db"}, Spec: appsv1.StatefulSetSpec{Replicas: ptr(int32(0)), VolumeClaimTemplates: []corev1.PersistentVolumeClaim{{ObjectMeta: metav1.ObjectMeta{Name: "data"}}}}}} }, evaluated: 1, passed: 1}, {name: "True with observed Pod", status: corev1.ConditionTrue, since: old, change: func(i *CheckInput) { i.Pods = []*corev1.Pod{{ObjectMeta: metav1.ObjectMeta{Namespace: "app"}, Spec: pvcPodSpec("data-db-0")}} }, evaluated: 1, passed: 1}, {name: "True with unknown coverage", status: corev1.ConditionTrue, since: old, change: func(i *CheckInput) { i.PVCConsumerNamespaces = nil }, missingConsumers: true}, {name: "absent", evaluated: 1, finding: true}, {name: "Unknown", status: corev1.ConditionUnknown, since: recent, evaluated: 1, finding: true}, {name: "unrelated False condition", change: func(i *CheckInput) { i.PersistentVolumeClaims[0].Status.Conditions = []corev1.PersistentVolumeClaimCondition{{Type: corev1.PersistentVolumeClaimResizing, Status: corev1.ConditionFalse}} }, evaluated: 1, finding: true}, {name: "unrelated condition before Unused", status: corev1.ConditionFalse, change: func(i *CheckInput) { pvc := i.PersistentVolumeClaims[0] pvc.Status.Conditions = append([]corev1.PersistentVolumeClaimCondition{{Type: corev1.PersistentVolumeClaimResizing, Status: corev1.ConditionTrue}}, pvc.Status.Conditions...) }, evaluated: 1, passed: 1}, {name: "Pending ignores False", status: corev1.ConditionFalse, change: func(i *CheckInput) { i.PersistentVolumeClaims[0].Status.Phase = corev1.ClaimPending }, evaluated: 1, finding: true}, {name: "Pending ignores recent True", status: corev1.ConditionTrue, since: recent, change: func(i *CheckInput) { i.PersistentVolumeClaims[0].Status.Phase = corev1.ClaimPending }, evaluated: 1, finding: true}, } { t.Run(tc.name, func(t *testing.T) { i := storageInput() pvc := i.PersistentVolumeClaims[0] pvc.CreationTimestamp = metav1.NewTime(now.Add(-30 * 24 * time.Hour)) if tc.status != "" { pvc.Status.Conditions = []corev1.PersistentVolumeClaimCondition{{Type: corev1.PersistentVolumeClaimUnused, Status: tc.status, LastTransitionTime: tc.since}} } if tc.change != nil { tc.change(i) } tr := newEvalTracker() r := buildResults(checkStorage(tr, i, now), tr, nil) checkID := "pvcNoConsumer" if pvc.Status.Phase == corev1.ClaimPending { checkID = "pvcLongPending" } if got := findingNames(r.Findings, checkID)[pvc.Name]; got != tc.finding { t.Fatalf("finding=%v want=%v: %+v", got, tc.finding, r.Findings) } if count := r.CheckCounts[checkID]; count.Evaluated != tc.evaluated || count.Passed != tc.passed { t.Fatalf("counts=%+v want evaluated=%d passed=%d", count, tc.evaluated, tc.passed) } if slices.Contains(r.MissingInputs, "pvc-consumers") != tc.missingConsumers { t.Fatalf("missing consumer input=%v want=%v", r.MissingInputs, tc.missingConsumers) } for _, f := range r.Findings { if f.CheckID != checkID { continue } if tc.evidence != "" { if !strings.Contains(f.Message, tc.evidence) { t.Errorf("missing evidence %q in %q", tc.evidence, f.Message) } } else if strings.Contains(f.Message, "reports it unused") { t.Errorf("unexpected Unused evidence in %q", f.Message) } for _, fact := range []string{"Requested 8Gi", `storage class "local"`, "PVC age"} { if !strings.Contains(f.Message, fact) { t.Errorf("missing fact %q in %q", fact, f.Message) } } } }) } } func TestPVCFindingMessages(t *testing.T) { now := time.Date(2026, time.October, 12, 12, 0, 0, 0, time.UTC) for _, tc := range []struct { name string phase corev1.PersistentVolumeClaimPhase unused bool checkID string message string }{ {"Bound", corev1.ClaimBound, false, "pvcNoConsumer", `Requested 8Gi; storage class "local"; PVC age 2d. Bound PV disk; reclaim policy Retain.`}, {"Bound with Unused condition", corev1.ClaimBound, true, "pvcNoConsumer", `Kubernetes reports it unused by any Pod since 2026-10-10T12:00:00Z (2d ago). Requested 8Gi; storage class "local"; PVC age 2d. Bound PV disk; reclaim policy Retain.`}, {"Pending", corev1.ClaimPending, false, "pvcLongPending", `Pending since creation, 2d ago. StorageClass uses WaitForFirstConsumer. Requested 8Gi; storage class "local"; PVC age 2d.`}, } { t.Run(tc.name, func(t *testing.T) { i := storageInput() pvc := i.PersistentVolumeClaims[0] pvc.CreationTimestamp = metav1.NewTime(now.Add(-48 * time.Hour)) pvc.Status.Phase = tc.phase if tc.unused { pvc.Status.Conditions = []corev1.PersistentVolumeClaimCondition{{Type: corev1.PersistentVolumeClaimUnused, Status: corev1.ConditionTrue, LastTransitionTime: pvc.CreationTimestamp}} } fs := checkStorage(newEvalTracker(), i, now) if len(fs) != 1 || fs[0].CheckID != tc.checkID { t.Fatalf("expected one %s finding: %+v", tc.checkID, fs) } if fs[0].Message != tc.message { t.Errorf("message=%q want=%q", fs[0].Message, tc.message) } for _, caveat := range []string{"No consumer observed among", "may still reference this claim"} { if strings.Contains(fs[0].Message, caveat) { t.Errorf("unexpected caveat %q in %q", caveat, fs[0].Message) } } }) } } func TestPVCCloneConsumers(t *testing.T) { now := time.Now() for _, phase := range []corev1.PersistentVolumeClaimPhase{corev1.ClaimBound, corev1.ClaimPending} { for _, tc := range []struct { name string change func(*corev1.PersistentVolumeClaim) consumed bool }{ {"Pending dataSource", func(*corev1.PersistentVolumeClaim) {}, true}, {"explicit core group", func(p *corev1.PersistentVolumeClaim) { p.Spec.DataSource.APIGroup = ptr("") }, true}, {"owned destination", func(p *corev1.PersistentVolumeClaim) { p.OwnerReferences = []metav1.OwnerReference{{Kind: "Job", Name: "clone", Controller: ptr(true)}} p.CreationTimestamp = metav1.NewTime(now.Add(-48 * time.Hour)) }, true}, {"deleting destination", func(p *corev1.PersistentVolumeClaim) { p.DeletionTimestamp = ptr(metav1.NewTime(now)) p.CreationTimestamp = metav1.NewTime(now.Add(-48 * time.Hour)) }, true}, {"Bound destination", func(p *corev1.PersistentVolumeClaim) { p.Status.Phase = corev1.ClaimBound }, false}, {"dataSourceRef", func(p *corev1.PersistentVolumeClaim) { p.Spec.DataSource = nil p.Spec.DataSourceRef = &corev1.TypedObjectReference{Kind: "PersistentVolumeClaim", Name: "data-db-0"} }, true}, {"dataSourceRef explicit core and same namespace", func(p *corev1.PersistentVolumeClaim) { p.Spec.DataSource = nil p.Spec.DataSourceRef = &corev1.TypedObjectReference{APIGroup: ptr(""), Kind: "PersistentVolumeClaim", Name: "data-db-0", Namespace: ptr("app")} }, true}, {"dataSourceRef empty namespace", func(p *corev1.PersistentVolumeClaim) { p.Spec.DataSource = nil p.Spec.DataSourceRef = &corev1.TypedObjectReference{Kind: "PersistentVolumeClaim", Name: "data-db-0", Namespace: ptr("")} }, true}, {"dataSourceRef targets another namespace", func(p *corev1.PersistentVolumeClaim) { p.Spec.DataSource = nil p.Spec.DataSourceRef = &corev1.TypedObjectReference{Kind: "PersistentVolumeClaim", Name: "data-db-0", Namespace: ptr("other")} }, false}, {"destination in other namespace", func(p *corev1.PersistentVolumeClaim) { p.Namespace = "other" }, false}, {"cross namespace destination referencing source", func(p *corev1.PersistentVolumeClaim) { p.Namespace = "other" p.Spec.DataSource = nil p.Spec.DataSourceRef = &corev1.TypedObjectReference{Kind: "PersistentVolumeClaim", Name: "data-db-0", Namespace: ptr("app")} }, true}, {"other source name", func(p *corev1.PersistentVolumeClaim) { p.Spec.DataSource.Name = "other" }, false}, {"non-core PVC", func(p *corev1.PersistentVolumeClaim) { p.Spec.DataSource.APIGroup = ptr("storage.example.com") }, false}, {"VolumeSnapshot", func(p *corev1.PersistentVolumeClaim) { p.Spec.DataSource = &corev1.TypedLocalObjectReference{APIGroup: ptr("snapshot.storage.k8s.io"), Kind: "VolumeSnapshot", Name: "data-db-0"} }, false}, {"dataSourceRef VolumeSnapshot", func(p *corev1.PersistentVolumeClaim) { p.Spec.DataSource = nil p.Spec.DataSourceRef = &corev1.TypedObjectReference{APIGroup: ptr("snapshot.storage.k8s.io"), Kind: "VolumeSnapshot", Name: "data-db-0"} }, false}, {"dataSourceRef non-core PVC", func(p *corev1.PersistentVolumeClaim) { p.Spec.DataSource = nil p.Spec.DataSourceRef = &corev1.TypedObjectReference{APIGroup: ptr("storage.example.com"), Kind: "PersistentVolumeClaim", Name: "data-db-0"} }, false}, {"Bound dataSourceRef", func(p *corev1.PersistentVolumeClaim) { p.Status.Phase = corev1.ClaimBound p.Spec.DataSource = nil p.Spec.DataSourceRef = &corev1.TypedObjectReference{Kind: "PersistentVolumeClaim", Name: "data-db-0"} }, false}, } { t.Run(string(phase)+"/"+tc.name, func(t *testing.T) { i := storageInput() i.PersistentVolumeClaims[0].Status.Phase = phase clone := &corev1.PersistentVolumeClaim{ ObjectMeta: metav1.ObjectMeta{Name: "clone", Namespace: "app", CreationTimestamp: metav1.NewTime(now.Add(-time.Minute))}, Spec: corev1.PersistentVolumeClaimSpec{DataSource: &corev1.TypedLocalObjectReference{Kind: "PersistentVolumeClaim", Name: "data-db-0"}}, Status: corev1.PersistentVolumeClaimStatus{Phase: corev1.ClaimPending}, } tc.change(clone) i.PersistentVolumeClaims = append(i.PersistentVolumeClaims, clone) tr := newEvalTracker() r := buildResults(checkStorage(tr, i, now), tr, nil) checkID := "pvcNoConsumer" if phase == corev1.ClaimPending { checkID = "pvcLongPending" } if findingNames(r.Findings, checkID)["data-db-0"] == tc.consumed { t.Fatalf("source consumed=%v: %+v", tc.consumed, r.Findings) } wantPassed := 0 if tc.consumed { wantPassed = 1 } if count := r.CheckCounts[checkID]; count.Evaluated != 1 || count.Passed != wantPassed || len(r.MissingInputs) != 0 { t.Fatalf("clone consumer counts: %+v missing=%v", count, r.MissingInputs) } }) } } } func TestLongPendingPVC(t *testing.T) { for _, tc := range []struct { name string age time.Duration consumer, immediate, unknownClass, unknownConsumers bool want bool }{ {"WFFC no consumer", 48 * time.Hour, false, false, false, false, true}, {"WFFC with consumer", 48 * time.Hour, true, false, false, false, false}, {"WFFC with Pod", 48 * time.Hour, true, false, false, false, false}, {"Immediate with consumer", 48 * time.Hour, true, true, false, false, false}, {"young", time.Hour, false, false, false, false, false}, {"exact threshold", 24 * time.Hour, false, false, false, false, false}, {"unknown class with consumer", 48 * time.Hour, true, false, true, false, false}, {"unknown class no consumer", 48 * time.Hour, false, false, true, false, true}, {"unknown consumers", 48 * time.Hour, false, false, false, true, false}, } { t.Run(tc.name, func(t *testing.T) { i := storageInput() now := time.Now() i.PersistentVolumeClaims[0].Status.Phase = corev1.ClaimPending i.PersistentVolumeClaims[0].CreationTimestamp = metav1.NewTime(now.Add(-tc.age)) if tc.consumer { if tc.name == "WFFC with Pod" { i.Pods = append(i.Pods, &corev1.Pod{ObjectMeta: metav1.ObjectMeta{Namespace: "app"}, Spec: pvcPodSpec("data-db-0")}) } else { i.CronJobs = append(i.CronJobs, &batchv1.CronJob{ObjectMeta: metav1.ObjectMeta{Namespace: "app"}, Spec: batchv1.CronJobSpec{JobTemplate: batchv1.JobTemplateSpec{Spec: batchv1.JobSpec{Template: corev1.PodTemplateSpec{Spec: pvcPodSpec("data-db-0")}}}}}) } } if tc.immediate { i.StorageClasses[0].VolumeBindingMode = ptr(storagev1.VolumeBindingImmediate) } if tc.unknownClass { i.StorageClasses = nil } if tc.unknownConsumers { i.PVCConsumerNamespaces = nil } tr := newEvalTracker() fs := checkStorage(tr, i, now) if got := findingNames(fs, "pvcLongPending")["data-db-0"]; got != tc.want { t.Fatalf("finding=%v want=%v: %+v", got, tc.want, fs) } if tc.want { for _, fact := range []string{"Requested 8Gi", `storage class "local"`, "PVC age 2d"} { if !strings.Contains(fs[0].Message, fact) { t.Errorf("missing fact %q in %q", fact, fs[0].Message) } } bindingEvidence := "StorageClass uses WaitForFirstConsumer." if tc.unknownClass { bindingEvidence = "StorageClass binding mode not visible." } if !strings.Contains(fs[0].Message, bindingEvidence) { t.Errorf("missing binding evidence %q in %q", bindingEvidence, fs[0].Message) } if !strings.Contains(fs[0].Message, "Pending since creation, 2d ago") || strings.Contains(fs[0].Message, "bound PV") || strings.Contains(fs[0].Message, "reclaim policy") || strings.Contains(fs[0].Message, "snapshot does not establish") { t.Fatalf("inaccurate Pending detail: %+v", fs[0]) } } }) } } func TestPVCStorageEligibility(t *testing.T) { now := time.Now() for _, phase := range []corev1.PersistentVolumeClaimPhase{corev1.ClaimBound, corev1.ClaimPending} { for _, age := range []time.Duration{time.Hour, 24 * time.Hour, 25 * time.Hour} { t.Run(string(phase)+"/"+age.String(), func(t *testing.T) { i := storageInput() i.PersistentVolumeClaims[0].Status.Phase = phase i.PersistentVolumeClaims[0].CreationTimestamp = metav1.NewTime(now.Add(-age)) fs := checkStorage(newEvalTracker(), i, now) if (len(fs) > 0) != (age > 24*time.Hour) { t.Fatalf("age eligibility: %+v", fs) } i.PersistentVolumeClaims[0].OwnerReferences = []metav1.OwnerReference{{APIVersion: "postgresql.cnpg.io/v1", Kind: "Cluster", Name: "hibernated", Controller: ptr(true)}} tr := newEvalTracker() if fs := checkStorage(tr, i, now); len(fs) != 0 || tr.counts["pvcNoConsumer"]["app"] != 0 || tr.counts["pvcLongPending"]["app"] != 0 { t.Fatalf("controller-owned claim evaluated: %+v", fs) } }) } } } // Missing inputs name only gaps that left a subject unevaluated; gaps that // leave a finding in place are caveats in its message. func TestStorageMissingInputsOnlyForSkippedSubjects(t *testing.T) { storageKeys := []string{"persistentvolumeclaims", "persistentvolumes", "storageclasses", "pvc-consumers", "pvc-binding-mode", "pv-deletion-events", "pv-release-time"} for _, tc := range []struct { name string change func(*CheckInput) missing []string findings int caveat string }{ {"healthy", func(i *CheckInput) { i.Pods = []*corev1.Pod{{ObjectMeta: metav1.ObjectMeta{Namespace: "app"}, Spec: pvcPodSpec("data-db-0")}} i.PersistentVolumes[0].Status.Phase = corev1.VolumeBound }, nil, 0, ""}, {"complete evidence", func(*CheckInput) {}, nil, 2, ""}, {"unreadable claims", func(i *CheckInput) { i.PersistentVolumeClaims = nil }, []string{"persistentvolumeclaims"}, 1, ""}, {"unknown consumers", func(i *CheckInput) { i.PVCConsumerNamespaces = nil }, []string{"pvc-consumers"}, 1, ""}, {"unreadable PVs without claims", func(i *CheckInput) { i.PersistentVolumeClaims = []*corev1.PersistentVolumeClaim{} i.PersistentVolumes = nil }, []string{"persistentvolumes"}, 0, ""}, {"PVs outside scope", func(i *CheckInput) { i.PersistentVolumes = []*corev1.PersistentVolume{} }, nil, 1, "reclaim policy not visible"}, {"unreadable classes", func(i *CheckInput) { i.PersistentVolumeClaims[0].Status.Phase = corev1.ClaimPending i.StorageClasses = nil }, nil, 2, "StorageClass binding mode not visible"}, {"unreadable events after grace", func(i *CheckInput) { i.PersistentVolumes[0].Spec.PersistentVolumeReclaimPolicy = corev1.PersistentVolumeReclaimDelete i.Events = nil }, nil, 2, "Deletion warning events not fully visible"}, {"unreadable events in grace", func(i *CheckInput) { i.PersistentVolumes[0].Spec.PersistentVolumeReclaimPolicy = corev1.PersistentVolumeReclaimDelete i.PersistentVolumes[0].Status.LastPhaseTransitionTime = ptr(metav1.NewTime(time.Now().Add(-time.Minute))) i.Events = nil }, []string{"pv-deletion-events"}, 1, ""}, {"unknown release time", func(i *CheckInput) { i.PersistentVolumes[0].Spec.PersistentVolumeReclaimPolicy = corev1.PersistentVolumeReclaimDelete i.PersistentVolumes[0].Status.LastPhaseTransitionTime = nil }, []string{"pv-release-time"}, 1, ""}, {"Failed volume with unreadable events", func(i *CheckInput) { i.PersistentVolumes[0].Spec.PersistentVolumeReclaimPolicy = corev1.PersistentVolumeReclaimDelete i.PersistentVolumes[0].Status.Phase = corev1.VolumeFailed i.PersistentVolumes[0].Status.LastPhaseTransitionTime = ptr(metav1.NewTime(time.Now().Add(-time.Minute))) i.Events = nil }, nil, 2, "Deletion warning events not fully visible"}, } { t.Run(tc.name, func(t *testing.T) { i := storageInput() i.PersistentVolumes[0].Status.Phase = corev1.VolumeReleased i.PersistentVolumes[0].Status.LastPhaseTransitionTime = ptr(metav1.NewTime(time.Now().Add(-72 * time.Hour))) tc.change(i) r := RunChecks(i) var got []string for _, key := range storageKeys { if slices.Contains(r.MissingInputs, key) { got = append(got, key) } } if !slices.Equal(got, tc.missing) { t.Fatalf("missing inputs %v, want %v", got, tc.missing) } var storageFindings []Finding for _, f := range r.Findings { if f.CheckID == "pvcNoConsumer" || f.CheckID == "pvcLongPending" || f.CheckID == "releasedPV" { storageFindings = append(storageFindings, f) } } if len(storageFindings) != tc.findings { t.Fatalf("findings %+v, want %d", storageFindings, tc.findings) } if tc.caveat != "" && !slices.ContainsFunc(storageFindings, func(f Finding) bool { return strings.Contains(f.Message, tc.caveat) }) { t.Fatalf("evidence gap %q not stated in findings: %+v", tc.caveat, storageFindings) } if tc.caveat == "" { for _, f := range storageFindings { if strings.Contains(f.Message, "not visible") || strings.Contains(f.Message, "not fully visible") { t.Fatalf("unexpected evidence caveat: %+v", f) } } } }) } } // Bound volumes pass; Available and Pending volumes have no claim to have // outlived, so they are neither passes nor findings; Failed volumes never pass. func TestReleasedPVCounts(t *testing.T) { i := storageInput() i.PersistentVolumeClaims = []*corev1.PersistentVolumeClaim{} i.PersistentVolumes[0].Status.Phase = corev1.VolumeBound for name, phase := range map[string]corev1.PersistentVolumePhase{"available": corev1.VolumeAvailable, "pending": corev1.VolumePending, "failed": corev1.VolumeFailed} { i.PersistentVolumes = append(i.PersistentVolumes, &corev1.PersistentVolume{ObjectMeta: metav1.ObjectMeta{Name: name, UID: types.UID(name)}, Spec: corev1.PersistentVolumeSpec{PersistentVolumeReclaimPolicy: corev1.PersistentVolumeReclaimDelete}, Status: corev1.PersistentVolumeStatus{Phase: phase}}) } r := RunChecks(i) if count := r.CheckCounts["releasedPV"]; count.Evaluated != 2 || count.Passed != 1 { t.Fatalf("releasedPV counts: %+v", count) } if got := findingNames(r.Findings, "releasedPV"); len(got) != 1 || !got["failed"] { t.Fatalf("releasedPV findings: %+v", r.Findings) } } // Deleting volumes belong to stuckTerminating, like deleting claims: releasedPV // neither passes a deleting Bound volume nor double-reports a deleting // Released or Failed one. func TestReleasedPVLeavesDeletingVolumesToStuckTerminating(t *testing.T) { i := storageInput() i.PersistentVolumeClaims = []*corev1.PersistentVolumeClaim{} deleted := metav1.NewTime(time.Now().Add(-2 * time.Hour)) released := metav1.NewTime(time.Now().Add(-48 * time.Hour)) i.PersistentVolumes = nil for name, phase := range map[string]corev1.PersistentVolumePhase{"bound": corev1.VolumeBound, "released": corev1.VolumeReleased, "failed": corev1.VolumeFailed} { i.PersistentVolumes = append(i.PersistentVolumes, &corev1.PersistentVolume{ ObjectMeta: metav1.ObjectMeta{Name: name, UID: types.UID(name), DeletionTimestamp: &deleted, Finalizers: []string{"kubernetes.io/pv-protection"}}, Spec: corev1.PersistentVolumeSpec{PersistentVolumeReclaimPolicy: corev1.PersistentVolumeReclaimRetain}, Status: corev1.PersistentVolumeStatus{Phase: phase, LastPhaseTransitionTime: &released}, }) } if slices.ContainsFunc(i.PersistentVolumes, PVNeedsReclaimEvents) { t.Fatal("deleting volumes must not request reclaim events") } r := RunChecks(i) if count, ok := r.CheckCounts["releasedPV"]; ok && (count.Evaluated != 0 || count.Passed != 0) { t.Fatalf("releasedPV counts: %+v", count) } if got := findingNames(r.Findings, "releasedPV"); len(got) != 0 { t.Fatalf("releasedPV findings: %+v", got) } if got := findingNames(r.Findings, "stuckTerminating"); len(got) != 3 { t.Fatalf("stuckTerminating findings: %+v", got) } if slices.Contains(r.MissingInputs, "persistentvolumes") || slices.Contains(r.MissingInputs, "pv-deletion-events") || slices.Contains(r.MissingInputs, "pv-release-time") { t.Fatalf("missing inputs: %v", r.MissingInputs) } } func TestFailedPV(t *testing.T) { now := time.Now() failedAt := now.Add(-5 * time.Minute) for _, tc := range []struct { name string change func(*CheckInput) text []string absent []string caveat bool }{ {"Delete with warning", func(i *CheckInput) { i.Events = []*corev1.Event{deleteFailure(failedAt, "host_path deleter only supports /tmp/.+")} }, []string{"Failed PV: reclaiming the volume failed after its claim was deleted (reclaim policy Delete).", "Latest reclaim warning (5m ago): host_path deleter only supports /tmp/.+.", "Failed for 5m."}, []string{"Status message"}, false}, {"status message without events", func(*CheckInput) {}, []string{"Status message: status failure."}, []string{"Latest reclaim warning"}, false}, {"partial event coverage", func(i *CheckInput) { i.PVDeletionEventsComplete = false }, []string{"Status message: status failure."}, nil, true}, {"unreadable events", func(i *CheckInput) { i.Events = nil }, []string{"Failed PV"}, nil, true}, {"visible warning with partial coverage", func(i *CheckInput) { i.PVDeletionEventsComplete = false i.Events = []*corev1.Event{deleteFailure(failedAt, "visible failure")} }, []string{"Latest reclaim warning (5m ago): visible failure."}, nil, true}, {"no message anywhere", func(i *CheckInput) { i.PersistentVolumes[0].Status.Message = "" i.PVDeletionEventsComplete = false }, []string{"Failed PV", "Deletion warning events not fully visible.", "8Gi"}, []string{"Status message", "Latest reclaim warning"}, true}, {"warning before the Failed transition", func(i *CheckInput) { i.Events = []*corev1.Event{deleteFailure(failedAt.Add(-time.Hour), "previous release failure")} }, []string{"Status message: status failure."}, []string{"previous release failure"}, false}, {"Recycle", func(i *CheckInput) { i.PersistentVolumes[0].Spec.PersistentVolumeReclaimPolicy = corev1.PersistentVolumeReclaimRecycle e := deleteFailure(failedAt, "recycler pod failed") e.Reason = "VolumeFailedRecycle" i.Events = []*corev1.Event{e} }, []string{"(reclaim policy Recycle)", "Latest reclaim warning (5m ago): recycler pod failed."}, nil, false}, {"policy changed to Retain after the failure", func(i *CheckInput) { i.PersistentVolumes[0].Spec.PersistentVolumeReclaimPolicy = corev1.PersistentVolumeReclaimRetain }, []string{"(reclaim policy Retain)", "Status message: status failure."}, nil, false}, {"transition unavailable", func(i *CheckInput) { i.PersistentVolumes[0].Status.LastPhaseTransitionTime = nil }, []string{"; PV age "}, []string{"Failed for"}, false}, } { t.Run(tc.name, func(t *testing.T) { i := storageInput() i.PersistentVolumeClaims = []*corev1.PersistentVolumeClaim{} pv := i.PersistentVolumes[0] pv.Spec.PersistentVolumeReclaimPolicy = corev1.PersistentVolumeReclaimDelete pv.Status = corev1.PersistentVolumeStatus{Phase: corev1.VolumeFailed, Message: "status failure", LastPhaseTransitionTime: ptr(metav1.NewTime(failedAt))} tc.change(i) tr := newEvalTracker() r := buildResults(checkStorage(tr, i, now), tr, nil) // The phase establishes the failure, so incomplete events never // leave the volume unevaluated. if count := r.CheckCounts["releasedPV"]; count.Evaluated != 1 || count.Passed != 0 || len(r.MissingInputs) != 0 { t.Fatalf("Failed PV counted as passed or skipped: %+v missing=%v", count, r.MissingInputs) } if len(r.Findings) != 1 || r.Findings[0].CheckID != "releasedPV" || r.Findings[0].Kind != "PersistentVolume" { t.Fatalf("Failed PV not reported: %+v", r.Findings) } message := r.Findings[0].Message for _, text := range append(tc.text, "Capacity 8Gi", `storage class "local"`, "former claim app/data-db-0") { if !strings.Contains(message, text) { t.Errorf("message lacks %q: %s", text, message) } } for _, text := range tc.absent { if strings.Contains(message, text) { t.Errorf("message has %q: %s", text, message) } } if strings.Contains(message, "Deletion warning events not fully visible") != tc.caveat { t.Errorf("event coverage caveat want %v: %s", tc.caveat, message) } }) } } func TestReleasedPVIgnoresRecycleWarnings(t *testing.T) { now := time.Now() i := storageInput() i.PersistentVolumes[0].Status.Phase = corev1.VolumeReleased i.PersistentVolumes[0].Spec.PersistentVolumeReclaimPolicy = corev1.PersistentVolumeReclaimDelete i.PersistentVolumes[0].Status.LastPhaseTransitionTime = ptr(metav1.NewTime(now.Add(-time.Minute))) e := deleteFailure(now, "recycler pod failed") e.Reason = "VolumeFailedRecycle" i.Events = []*corev1.Event{e} if got := latestPVReclaimWarnings(i.PersistentVolumes, i.Events, now); len(got) != 0 { t.Fatalf("recycle warning read as a deletion failure: %+v", got) } } func TestReleasedPV(t *testing.T) { now := time.Now() for _, tc := range []struct { name string change func(*CheckInput) want, text string }{ {"Retain", func(*CheckInput) {}, "disk", "kept after its claim was deleted"}, {"Bound", func(i *CheckInput) { i.PersistentVolumes[0].Status.Phase = corev1.VolumeBound }, "", ""}, {"Delete no event", func(i *CheckInput) { i.PersistentVolumes[0].Spec.PersistentVolumeReclaimPolicy = corev1.PersistentVolumeReclaimDelete }, "disk", "deletion has not completed"}, {"Delete recent event", func(i *CheckInput) { i.PersistentVolumes[0].Spec.PersistentVolumeReclaimPolicy = corev1.PersistentVolumeReclaimDelete i.Events = []*corev1.Event{deleteFailure(now.Add(-time.Minute), "latest failure")} }, "disk", "deletion is failing: latest failure"}, {"Delete old event", func(i *CheckInput) { i.PersistentVolumes[0].Spec.PersistentVolumeReclaimPolicy = corev1.PersistentVolumeReclaimDelete i.Events = []*corev1.Event{deleteFailure(now.Add(-48*time.Hour), "stale failure")} }, "disk", "Latest deletion warning (2d ago): stale failure"}, {"Delete old UID", func(i *CheckInput) { i.PersistentVolumes[0].Spec.PersistentVolumeReclaimPolicy = corev1.PersistentVolumeReclaimDelete e := deleteFailure(now, "other object") e.InvolvedObject.UID = "old-uid" i.Events = []*corev1.Event{e} }, "disk", "deletion has not completed"}, {"Delete unreadable events", func(i *CheckInput) { i.PersistentVolumes[0].Spec.PersistentVolumeReclaimPolicy = corev1.PersistentVolumeReclaimDelete i.Events = nil }, "disk", "deletion has not completed"}, {"Retain unreadable events", func(i *CheckInput) { i.Events = nil }, "disk", "kept after its claim was deleted"}, {"transition time", func(i *CheckInput) { i.PersistentVolumes[0].Status.LastPhaseTransitionTime = ptr(metav1.NewTime(now.Add(-time.Hour))) }, "", ""}, {"Retain recent release", func(i *CheckInput) { i.PersistentVolumes[0].Status.LastPhaseTransitionTime = ptr(metav1.NewTime(now.Add(-23 * time.Hour))) }, "", ""}, {"Delete recent release", func(i *CheckInput) { i.PersistentVolumes[0].Spec.PersistentVolumeReclaimPolicy = corev1.PersistentVolumeReclaimDelete i.PersistentVolumes[0].Status.LastPhaseTransitionTime = ptr(metav1.NewTime(now.Add(-time.Minute))) }, "", ""}, {"Delete recent release with warning", func(i *CheckInput) { i.PersistentVolumes[0].Spec.PersistentVolumeReclaimPolicy = corev1.PersistentVolumeReclaimDelete i.PersistentVolumes[0].Status.LastPhaseTransitionTime = ptr(metav1.NewTime(now.Add(-time.Minute))) i.Events = []*corev1.Event{deleteFailure(now, "immediate failure")} }, "disk", "deletion is failing: immediate failure"}, {"Delete exact threshold", func(i *CheckInput) { i.PersistentVolumes[0].Spec.PersistentVolumeReclaimPolicy = corev1.PersistentVolumeReclaimDelete i.PersistentVolumes[0].Status.LastPhaseTransitionTime = ptr(metav1.NewTime(now.Add(-time.Hour))) }, "", ""}, {"Delete old release", func(i *CheckInput) { i.PersistentVolumes[0].Spec.PersistentVolumeReclaimPolicy = corev1.PersistentVolumeReclaimDelete i.PersistentVolumes[0].Status.LastPhaseTransitionTime = ptr(metav1.NewTime(now.Add(-2 * time.Hour))) }, "disk", "Released for 2h"}, {"latest event series", func(i *CheckInput) { i.PersistentVolumes[0].Spec.PersistentVolumeReclaimPolicy = corev1.PersistentVolumeReclaimDelete e := deleteFailure(now.Add(-48*time.Hour), "series failure") e.Series = &corev1.EventSeries{LastObservedTime: metav1.NewMicroTime(now.Add(-time.Minute))} i.Events = []*corev1.Event{deleteFailure(now.Add(-time.Hour), "earlier failure"), e} }, "disk", "series failure"}, } { t.Run(tc.name, func(t *testing.T) { i := storageInput() i.PersistentVolumes[0].Status.Phase = corev1.VolumeReleased i.PersistentVolumes[0].Status.LastPhaseTransitionTime = ptr(metav1.NewTime(now.Add(-72 * time.Hour))) tc.change(i) tr := newEvalTracker() fs := checkStorage(tr, i, now) if got := findingNames(fs, "releasedPV")["disk"]; got != (tc.want != "") { t.Fatalf("finding=%v want=%q: %+v", got, tc.want, fs) } for _, f := range fs { if f.CheckID == "releasedPV" && (!strings.Contains(f.Message, tc.text) || !strings.Contains(f.Message, "8Gi") || !strings.Contains(f.Message, "app/data-db-0")) { t.Errorf("missing evidence: %+v", f) } } if tr.counts["releasedPV"][""] != 1 { t.Fatal("readable PV not evaluated") } if tc.name == "Delete unreadable events" && (len(tr.missingInputs) != 0 || !strings.Contains(fs[len(fs)-1].Message, "Deletion warning events not fully visible")) { t.Fatalf("flagged volume's evidence gap reported as a skip, or not stated: %v %+v", tr.missingInputs, fs) } if tc.name == "Delete no event" && !strings.Contains(fs[len(fs)-1].Message, "Released for 3d") { t.Fatal("release duration not labelled") } }) } } func TestReleasedPVWithoutReleaseTime(t *testing.T) { now := time.Now() for _, transition := range []struct { name string at *metav1.Time }{ {"nil", nil}, {"zero", &metav1.Time{}}, } { for _, tc := range []struct { name string change func(*CheckInput) findings int text string }{ {"Delete no warning", func(*CheckInput) {}, 0, ""}, {"Delete stale warning", func(i *CheckInput) { i.Events = []*corev1.Event{deleteFailure(now.Add(-48*time.Hour), "stale failure")} }, 0, ""}, {"Delete unreadable events", func(i *CheckInput) { i.Events = nil }, 0, ""}, {"Delete partial events", func(i *CheckInput) { i.PVDeletionEventsComplete = false }, 0, ""}, {"Delete recent warning", func(i *CheckInput) { i.Events = []*corev1.Event{deleteFailure(now.Add(-time.Minute), "current failure")} }, 1, "deletion is failing: current failure."}, {"Delete recent warning with partial events", func(i *CheckInput) { i.PVDeletionEventsComplete = false i.Events = []*corev1.Event{deleteFailure(now.Add(-time.Minute), "current failure")} }, 1, "deletion is failing: current failure."}, {"Retain", func(i *CheckInput) { i.PersistentVolumes[0].Spec.PersistentVolumeReclaimPolicy = corev1.PersistentVolumeReclaimRetain }, 1, "release time unknown"}, {"Retain young PV", func(i *CheckInput) { i.PersistentVolumes[0].Spec.PersistentVolumeReclaimPolicy = corev1.PersistentVolumeReclaimRetain i.PersistentVolumes[0].CreationTimestamp = metav1.NewTime(now.Add(-time.Minute)) }, 1, "release time unknown"}, } { t.Run(transition.name+"/"+tc.name, func(t *testing.T) { i := storageInput() i.PersistentVolumeClaims = []*corev1.PersistentVolumeClaim{} i.PersistentVolumes[0].Status.Phase = corev1.VolumeReleased i.PersistentVolumes[0].Status.LastPhaseTransitionTime = transition.at i.PersistentVolumes[0].Spec.PersistentVolumeReclaimPolicy = corev1.PersistentVolumeReclaimDelete tc.change(i) tr := newEvalTracker() r := buildResults(checkStorage(tr, i, now), tr, nil) if count := r.CheckCounts["releasedPV"]; count.Evaluated != tc.findings || count.Passed != 0 || len(r.Findings) != tc.findings { t.Fatalf("unknown release counted as passing or age-based finding: %+v findings=%+v", count, r.Findings) } var wantMissing []string if tc.findings == 0 { wantMissing = []string{"pv-release-time"} } if !slices.Equal(r.MissingInputs, wantMissing) { t.Fatalf("missing inputs %v, want %v", r.MissingInputs, wantMissing) } for _, f := range r.Findings { if !strings.Contains(f.Message, tc.text) || !strings.Contains(f.Message, "PV age ") || strings.Contains(f.Message, "Released for") || strings.Contains(f.Message, "deletion has not completed") { t.Fatalf("unknown release duration misstated: %+v", f) } if strings.Contains(f.Message, "Deletion warning events not fully visible") == i.PVDeletionEventsComplete { t.Fatalf("event coverage caveat wrong: %+v", f) } } }) } } } func TestPVDeletionEventIdentityAndTimes(t *testing.T) { now := time.Now() for _, tc := range []struct { name string change func(*corev1.Event) want bool }{ {"current UID", func(*corev1.Event) {}, true}, {"old UID", func(e *corev1.Event) { e.InvolvedObject.UID = "previous" }, false}, {"wrong name", func(e *corev1.Event) { e.InvolvedObject.Name = "other" }, false}, {"namespaced ref", func(e *corev1.Event) { e.InvolvedObject.Namespace = "app" }, false}, {"CRD ref", func(e *corev1.Event) { e.InvolvedObject.APIVersion = "storage.example.com/v1" }, false}, {"wrong kind", func(e *corev1.Event) { e.InvolvedObject.Kind = "PersistentVolumeClaim" }, false}, {"Normal", func(e *corev1.Event) { e.Type = corev1.EventTypeNormal }, false}, {"wrong reason", func(e *corev1.Event) { e.Reason = "ProvisioningFailed" }, false}, {"future timestamp", func(e *corev1.Event) { e.LastTimestamp = metav1.NewTime(now.Add(time.Hour)) }, false}, {"event time", func(e *corev1.Event) { e.LastTimestamp = metav1.Time{}; e.EventTime = metav1.NewMicroTime(now) }, true}, {"creation time", func(e *corev1.Event) { e.LastTimestamp = metav1.Time{}; e.CreationTimestamp = metav1.NewTime(now) }, true}, } { t.Run(tc.name, func(t *testing.T) { i := storageInput() i.PersistentVolumes[0].Status.Phase = corev1.VolumeReleased i.PersistentVolumes[0].Spec.PersistentVolumeReclaimPolicy = corev1.PersistentVolumeReclaimDelete e := deleteFailure(now, "warning") tc.change(e) got := latestPVReclaimWarnings(i.PersistentVolumes, []*corev1.Event{e}, now) if (got["disk-uid"].event != nil) != tc.want { t.Fatalf("event identity/timestamp: %+v", got) } }) } } func deleteFailure(at time.Time, message string) *corev1.Event { return &corev1.Event{ObjectMeta: metav1.ObjectMeta{Namespace: "app"}, InvolvedObject: corev1.ObjectReference{APIVersion: "v1", Kind: "PersistentVolume", Name: "disk", UID: "disk-uid"}, Type: corev1.EventTypeWarning, Reason: "VolumeFailedDelete", Message: message, LastTimestamp: metav1.NewTime(at)} } func TestRecentReleasedPVEventCoverage(t *testing.T) { now := time.Now() for _, tc := range []struct { name string change func(*CheckInput) evaluated, passed, findings int missing bool }{ {"complete events", func(*CheckInput) {}, 1, 1, 0, false}, {"unreadable events", func(i *CheckInput) { i.Events = nil }, 0, 0, 0, true}, {"partial events", func(i *CheckInput) { i.PVDeletionEventsComplete = false }, 0, 0, 0, true}, {"partial events with warning", func(i *CheckInput) { i.PVDeletionEventsComplete = false i.Events = []*corev1.Event{deleteFailure(now, "visible failure")} }, 1, 0, 1, false}, {"unreadable events after grace", func(i *CheckInput) { i.Events = nil i.PersistentVolumes[0].Status.LastPhaseTransitionTime = ptr(metav1.NewTime(now.Add(-2 * time.Hour))) }, 1, 0, 1, false}, {"Retain does not need events", func(i *CheckInput) { i.Events = nil i.PersistentVolumes[0].Spec.PersistentVolumeReclaimPolicy = corev1.PersistentVolumeReclaimRetain }, 1, 1, 0, false}, {"Bound does not need events", func(i *CheckInput) { i.Events = nil i.PersistentVolumes[0].Status.Phase = corev1.VolumeBound }, 1, 1, 0, false}, } { t.Run(tc.name, func(t *testing.T) { i := storageInput() i.PersistentVolumeClaims = []*corev1.PersistentVolumeClaim{} i.PersistentVolumes[0].Spec.PersistentVolumeReclaimPolicy = corev1.PersistentVolumeReclaimDelete i.PersistentVolumes[0].Status.Phase = corev1.VolumeReleased i.PersistentVolumes[0].Status.LastPhaseTransitionTime = ptr(metav1.NewTime(now.Add(-time.Minute))) tc.change(i) tr := newEvalTracker() r := buildResults(checkStorage(tr, i, now), tr, nil) count := r.CheckCounts["releasedPV"] if count.Evaluated != tc.evaluated || count.Passed != tc.passed || len(r.Findings) != tc.findings || slices.Contains(r.MissingInputs, "pv-deletion-events") != tc.missing { t.Fatalf("evaluated=%d passed=%d findings=%+v missing=%v", count.Evaluated, count.Passed, r.Findings, r.MissingInputs) } complete := i.Events != nil && i.PVDeletionEventsComplete for _, f := range r.Findings { if strings.Contains(f.Message, "Deletion warning events not fully visible") == complete { t.Fatalf("event coverage caveat wrong for complete=%v: %+v", complete, f) } } }) } } func TestPVDeletionWarningsCurrentRelease(t *testing.T) { now := time.Now() for _, tc := range []struct { name string change func(*CheckInput) message string }{ {"previous release warning", func(*CheckInput) {}, ""}, {"previous release warning after grace", func(i *CheckInput) { i.PersistentVolumes[0].Status.LastPhaseTransitionTime = ptr(metav1.NewTime(now.Add(-2 * time.Hour))) }, "deletion has not completed"}, {"warning at transition", func(i *CheckInput) { i.Events[0].LastTimestamp = *i.PersistentVolumes[0].Status.LastPhaseTransitionTime }, "deletion is failing"}, {"series continued in current release", func(i *CheckInput) { i.Events[0].Series = &corev1.EventSeries{LastObservedTime: metav1.NewMicroTime(now)} }, "deletion is failing"}, {"transition unavailable", func(i *CheckInput) { i.PersistentVolumes[0].Status.LastPhaseTransitionTime = nil }, "deletion is failing"}, {"transition zero", func(i *CheckInput) { i.PersistentVolumes[0].Status.LastPhaseTransitionTime = &metav1.Time{} }, "deletion is failing"}, } { t.Run(tc.name, func(t *testing.T) { i := storageInput() i.PersistentVolumeClaims = []*corev1.PersistentVolumeClaim{} i.PersistentVolumes[0].Spec.PersistentVolumeReclaimPolicy = corev1.PersistentVolumeReclaimDelete i.PersistentVolumes[0].Status.Phase = corev1.VolumeReleased i.PersistentVolumes[0].Status.LastPhaseTransitionTime = ptr(metav1.NewTime(now.Add(-time.Minute))) i.Events = []*corev1.Event{deleteFailure(now.Add(-3*time.Hour), "previous release failure")} tc.change(i) fs := checkStorage(newEvalTracker(), i, now) if tc.message == "" { if len(fs) != 0 { t.Fatalf("previous release warning reused: %+v", fs) } } else if len(fs) != 1 || !strings.Contains(fs[0].Message, tc.message) { t.Fatalf("current release outcome: %+v", fs) } if tc.name == "previous release warning after grace" && strings.Contains(fs[0].Message, "previous release failure") { t.Fatalf("stale warning included as evidence: %+v", fs[0]) } }) } } func TestLongPendingPVCStorageClassEvidence(t *testing.T) { now := time.Now() for _, tc := range []struct { name string change func(*CheckInput) message string hidden bool }{ {"unreadable classes", func(i *CheckInput) { i.StorageClasses = nil }, "binding mode not visible", true}, {"unreadable default class", func(i *CheckInput) { i.StorageClasses = nil i.PersistentVolumeClaims[0].Spec.StorageClassName = nil }, "binding mode not visible", true}, {"missing named class", func(i *CheckInput) { i.PersistentVolumeClaims[0].Spec.StorageClassName = ptr("missing") }, `StorageClass "missing" not found`, false}, {"empty class inventory", func(i *CheckInput) { i.StorageClasses = []*storagev1.StorageClass{} }, `StorageClass "local" not found`, false}, {"no default class", func(i *CheckInput) { i.PersistentVolumeClaims[0].Spec.StorageClassName = nil }, "No default StorageClass applies", false}, {"no classes or default", func(i *CheckInput) { i.PersistentVolumeClaims[0].Spec.StorageClassName = nil i.StorageClasses = []*storagev1.StorageClass{} }, "No default StorageClass applies", false}, {"default WFFC class", func(i *CheckInput) { i.PersistentVolumeClaims[0].Spec.StorageClassName = nil i.StorageClasses[0].Annotations = map[string]string{"storageclass.kubernetes.io/is-default-class": "true"} }, "Default StorageClass \"local\" uses WaitForFirstConsumer", false}, {"newest default class", func(i *CheckInput) { i.PersistentVolumeClaims[0].Spec.StorageClassName = nil i.StorageClasses[0].Annotations = map[string]string{"storageclass.kubernetes.io/is-default-class": "true"} i.StorageClasses[0].CreationTimestamp = metav1.NewTime(now.Add(-time.Hour)) i.StorageClasses = append([]*storagev1.StorageClass{{ObjectMeta: metav1.ObjectMeta{Name: "new", CreationTimestamp: metav1.NewTime(now), Annotations: map[string]string{"storageclass.kubernetes.io/is-default-class": "true"}}}}, i.StorageClasses...) }, "Default StorageClass \"new\" applies", false}, {"default timestamp tie", func(i *CheckInput) { i.PersistentVolumeClaims[0].Spec.StorageClassName = nil i.StorageClasses[0].Annotations = map[string]string{"storageclass.kubernetes.io/is-default-class": "true"} i.StorageClasses = append(i.StorageClasses, &storagev1.StorageClass{ObjectMeta: metav1.ObjectMeta{Name: "alphabetical", Annotations: map[string]string{"storageclass.kubernetes.io/is-default-class": "true"}}}) }, "Default StorageClass \"alphabetical\" applies", false}, {"explicit no class", func(i *CheckInput) { i.PersistentVolumeClaims[0].Spec.StorageClassName = ptr("") }, "storage class \"\"", false}, {"explicit no class with unreadable classes", func(i *CheckInput) { i.PersistentVolumeClaims[0].Spec.StorageClassName = ptr("") i.StorageClasses = nil }, "storage class \"\"", false}, } { t.Run(tc.name, func(t *testing.T) { i := storageInput() i.PersistentVolumeClaims[0].Status.Phase = corev1.ClaimPending tc.change(i) tr := newEvalTracker() fs := checkStorage(tr, i, now) if len(fs) != 1 || !strings.Contains(fs[0].Message, tc.message) { t.Fatalf("class evidence: %+v", fs) } // The claim is evaluated either way, so class visibility is a // caveat in the finding, never a missing input. if len(tr.missingInputs) != 0 { t.Fatalf("evaluated claim reported as skipped: %+v", tr.missingInputs) } if !tc.hidden && strings.Contains(fs[0].Message, "binding mode not visible") { t.Fatalf("readable class evidence described as invisible: %+v", fs[0]) } }) } }