package audit import ( "fmt" "strconv" "strings" "time" "github.com/skyhook-io/radar/pkg/resourceid" "github.com/skyhook-io/radar/pkg/timeutil" corev1 "k8s.io/api/core/v1" storagev1 "k8s.io/api/storage/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/types" ) const ( pvcReviewAge = 24 * time.Hour pvDeleteGrace = time.Hour pvRetainGrace = 24 * time.Hour pvDeleteEventWindow = 24 * time.Hour ) func checkStorage(tr *evalTracker, input *CheckInput, now time.Time) []Finding { if input.PersistentVolumeClaims == nil { tr.skip("persistentvolumeclaims") } consumerNamespaces := map[string]bool{} for _, ns := range input.PVCConsumerNamespaces { consumerNamespaces[ns] = true } pvs := map[string]*corev1.PersistentVolume{} for _, pv := range input.PersistentVolumes { pvs[pv.Name] = pv } classes := map[string]*storagev1.StorageClass{} var defaultClass *storagev1.StorageClass for _, sc := range input.StorageClasses { classes[sc.Name] = sc if sc.Annotations["storageclass.kubernetes.io/is-default-class"] == "true" && (defaultClass == nil || sc.CreationTimestamp.After(defaultClass.CreationTimestamp.Time) || sc.CreationTimestamp.Equal(&defaultClass.CreationTimestamp) && sc.Name < defaultClass.Name) { defaultClass = sc } } consumers := collectPVCConsumers(input) var findings []Finding for _, pvc := range input.PersistentVolumeClaims { if pvc.DeletionTimestamp != nil || hasControllerOwnerReference(pvc.OwnerReferences) { continue } if pvc.Status.Phase != corev1.ClaimBound && pvc.Status.Phase != corev1.ClaimPending { continue } consumed := consumers[pvc.Namespace+"/"+pvc.Name] reviewSince := pvc.CreationTimestamp var unusedSince metav1.Time if pvc.Status.Phase == corev1.ClaimBound { for _, condition := range pvc.Status.Conditions { if condition.Type != corev1.PersistentVolumeClaimUnused { continue } switch condition.Status { case corev1.ConditionFalse: consumed = true case corev1.ConditionTrue: unusedSince = condition.LastTransitionTime if !unusedSince.IsZero() { reviewSince = unusedSince } } break } } if reviewSince.IsZero() || now.Sub(reviewSince.Time) <= pvcReviewAge { continue } complete := consumerNamespaces[pvc.Namespace] && input.Pods != nil && input.Deployments != nil && input.ReplicaSets != nil && input.StatefulSets != nil && input.DaemonSets != nil && input.Jobs != nil && input.CronJobs != nil if !consumed && !complete { tr.skip("pvc-consumers") continue } checkID, message := "", "" switch pvc.Status.Phase { case corev1.ClaimBound: tr.record("pvcNoConsumer", pvc.Namespace) if consumed { continue } checkID = "pvcNoConsumer" if !unusedSince.IsZero() { message = fmt.Sprintf("Kubernetes reports it unused by any Pod since %s (%s ago).", unusedSince.UTC().Format(time.RFC3339), timeutil.FormatAgeShort(now.Sub(unusedSince.Time))) } case corev1.ClaimPending: tr.record("pvcLongPending", pvc.Namespace) if consumed { continue } checkID = "pvcLongPending" message = "Pending since creation, " + storageAge(pvc.CreationTimestamp, now) + " ago." if pvc.Spec.StorageClassName == nil || *pvc.Spec.StorageClassName != "" { sc := defaultClass if pvc.Spec.StorageClassName != nil { sc = classes[*pvc.Spec.StorageClassName] } if input.StorageClasses == nil { message += " StorageClass binding mode not visible." } else if sc == nil { if pvc.Spec.StorageClassName == nil { message += " No default StorageClass applies." } else { message += fmt.Sprintf(" StorageClass %q not found.", *pvc.Spec.StorageClassName) } } else { classDescription := "StorageClass" if pvc.Spec.StorageClassName == nil { classDescription = fmt.Sprintf("Default StorageClass %q", sc.Name) } if sc.VolumeBindingMode != nil && *sc.VolumeBindingMode == storagev1.VolumeBindingWaitForFirstConsumer { message += " " + classDescription + " uses WaitForFirstConsumer." } else if pvc.Spec.StorageClassName == nil { message += " " + classDescription + " applies." } } } } if message != "" { message += " " } message += pvcStorageDetails(pvc, pvs[pvc.Spec.VolumeName], now) findings = append(findings, Finding{Kind: "PersistentVolumeClaim", Namespace: pvc.Namespace, Name: pvc.Name, CheckID: checkID, Category: CategoryEfficiency, Severity: SeverityWarning, Message: message}) } if input.PersistentVolumes == nil { tr.skip("persistentvolumes") } reclaimWarnings := latestPVReclaimWarnings(input.PersistentVolumes, input.Events, now) deletionEventsComplete := input.Events != nil && input.PVDeletionEventsComplete for _, pv := range input.PersistentVolumes { if pv.DeletionTimestamp != nil { // A deleting volume is a stuckTerminating subject, as a deleting // claim is. continue } warning, hasWarning := reclaimWarnings[pv.UID] switch pv.Status.Phase { case corev1.VolumeBound: tr.record("releasedPV", "") continue case corev1.VolumeFailed: // The PV controller sets Failed only when reclaiming the volume // failed, so the phase alone establishes the finding; events add // detail and never decide whether the volume is evaluated. tr.record("releasedPV", "") message := fmt.Sprintf("Failed PV: reclaiming the volume failed after its claim was deleted (reclaim policy %s).", pv.Spec.PersistentVolumeReclaimPolicy) if hasWarning { message += " Latest reclaim warning (" + timeutil.FormatAgeShort(now.Sub(warning.at)) + " ago): " + asSentence(warning.event.Message) } else if pv.Status.Message != "" { message += " Status message: " + asSentence(pv.Status.Message) } if !deletionEventsComplete { message += " Deletion warning events not fully visible." } findings = append(findings, releasedPVFinding(pv, message, now)) continue case corev1.VolumeReleased: default: // An Available or Pending volume has no claim to have outlived; // this check has no verdict on it, so it is not a passing subject. continue } var releasedAt metav1.Time if pv.Status.LastPhaseTransitionTime != nil && !pv.Status.LastPhaseTransitionTime.IsZero() { releasedAt = *pv.Status.LastPhaseTransitionTime } recentFailure := hasWarning && now.Sub(warning.at) <= pvDeleteEventWindow if pv.Spec.PersistentVolumeReclaimPolicy == corev1.PersistentVolumeReclaimDelete && releasedAt.IsZero() && !recentFailure { tr.skip("pv-release-time") continue } inDeleteGrace := !releasedAt.IsZero() && now.Sub(releasedAt.Time) <= pvDeleteGrace if pv.Spec.PersistentVolumeReclaimPolicy == corev1.PersistentVolumeReclaimDelete && inDeleteGrace && !recentFailure && !deletionEventsComplete { // A warning Radar can't see may already show this deletion failing. tr.skip("pv-deletion-events") continue } tr.record("releasedPV", "") message := "" switch pv.Spec.PersistentVolumeReclaimPolicy { case corev1.PersistentVolumeReclaimRetain: if !releasedAt.IsZero() && now.Sub(releasedAt.Time) <= pvRetainGrace { continue } message = "Released PV is kept after its claim was deleted (reclaim policy Retain)." if releasedAt.IsZero() { message = "Released PV is kept after its claim was deleted (reclaim policy Retain); release time unknown." } case corev1.PersistentVolumeReclaimDelete: if recentFailure { message = "Released PV deletion is failing: " + asSentence(warning.event.Message) } else { if inDeleteGrace { continue } message = "Released PV deletion has not completed (reclaim policy Delete)." if hasWarning { message += " Latest deletion warning (" + timeutil.FormatAgeShort(now.Sub(warning.at)) + " ago): " + asSentence(warning.event.Message) } } if !deletionEventsComplete { message += " Deletion warning events not fully visible." } default: continue } findings = append(findings, releasedPVFinding(pv, message, now)) } return findings } func releasedPVFinding(pv *corev1.PersistentVolume, message string, now time.Time) Finding { age := "PV age " + storageAge(pv.CreationTimestamp, now) if pv.Status.LastPhaseTransitionTime != nil && !pv.Status.LastPhaseTransitionTime.IsZero() { age = string(pv.Status.Phase) + " for " + storageAge(*pv.Status.LastPhaseTransitionTime, now) } formerClaim := "unknown" if pv.Spec.ClaimRef != nil { formerClaim = pv.Spec.ClaimRef.Namespace + "/" + pv.Spec.ClaimRef.Name } message += fmt.Sprintf(" Capacity %s; storage class %q; former claim %s; %s.", storageQuantity(pv.Spec.Capacity), pv.Spec.StorageClassName, formerClaim, age) return Finding{Kind: "PersistentVolume", Name: pv.Name, CheckID: "releasedPV", Category: CategoryEfficiency, Severity: SeverityWarning, Message: message} } func collectPVCConsumers(input *CheckInput) map[string]bool { refs := map[string]bool{} add := func(ns string, spec corev1.PodSpec) { for _, volume := range spec.Volumes { if volume.PersistentVolumeClaim != nil { refs[ns+"/"+volume.PersistentVolumeClaim.ClaimName] = true } } } for _, p := range input.Pods { add(p.Namespace, p.Spec) for _, v := range p.Spec.Volumes { if v.Ephemeral != nil { refs[p.Namespace+"/"+p.Name+"-"+v.Name] = true } } } for _, d := range input.Deployments { add(d.Namespace, d.Spec.Template.Spec) } for _, rs := range input.ReplicaSets { add(rs.Namespace, rs.Spec.Template.Spec) } for _, ds := range input.DaemonSets { add(ds.Namespace, ds.Spec.Template.Spec) } for _, ss := range input.StatefulSets { add(ss.Namespace, ss.Spec.Template.Spec) } for _, j := range input.Jobs { add(j.Namespace, j.Spec.Template.Spec) } for _, cj := range input.CronJobs { add(cj.Namespace, cj.Spec.JobTemplate.Spec.Template.Spec) } for _, pvc := range input.PersistentVolumeClaims { if pvc.Status.Phase != corev1.ClaimPending { continue } if source := pvc.Spec.DataSource; source != nil && source.Kind == "PersistentVolumeClaim" && (source.APIGroup == nil || *source.APIGroup == "") { refs[pvc.Namespace+"/"+source.Name] = true } if source := pvc.Spec.DataSourceRef; source != nil && source.Kind == "PersistentVolumeClaim" && (source.APIGroup == nil || *source.APIGroup == "") { namespace := pvc.Namespace if source.Namespace != nil && *source.Namespace != "" { namespace = *source.Namespace } refs[namespace+"/"+source.Name] = true } } // Match retained ordinals too: scaling to zero does not relinquish data. generated := map[string]bool{} for _, ss := range input.StatefulSets { for _, template := range ss.Spec.VolumeClaimTemplates { generated[ss.Namespace+"/"+template.Name+"-"+ss.Name] = true } } for _, pvc := range input.PersistentVolumeClaims { cut := strings.LastIndexByte(pvc.Name, '-') if cut < 0 || !generated[pvc.Namespace+"/"+pvc.Name[:cut]] { continue } ordinal := pvc.Name[cut+1:] if n, err := strconv.ParseUint(ordinal, 10, 32); err == nil && strconv.FormatUint(n, 10) == ordinal { refs[pvc.Namespace+"/"+pvc.Name] = true } } return refs } func pvcStorageDetails(pvc *corev1.PersistentVolumeClaim, pv *corev1.PersistentVolume, now time.Time) string { class, volume, reclaim := "not assigned", "not bound", "not visible" if pvc.Spec.StorageClassName != nil { class = fmt.Sprintf("%q", *pvc.Spec.StorageClassName) } if pvc.Spec.VolumeName != "" { volume = pvc.Spec.VolumeName } if pv != nil { reclaim = string(pv.Spec.PersistentVolumeReclaimPolicy) } details := fmt.Sprintf("Requested %s; storage class %s; PVC age %s.", storageQuantity(pvc.Spec.Resources.Requests), class, storageAge(pvc.CreationTimestamp, now)) if pvc.Status.Phase == corev1.ClaimBound { details += fmt.Sprintf(" Bound PV %s; reclaim policy %s.", volume, reclaim) } return details } func storageQuantity(resources corev1.ResourceList) string { if size, ok := resources[corev1.ResourceStorage]; ok { return size.String() } return "unknown" } func storageAge(created metav1.Time, now time.Time) string { if created.IsZero() { return "unknown" } return timeutil.FormatAgeShort(now.Sub(created.Time)) } // PVNeedsReclaimEvents reports whether a releasedPV finding for pv can draw // on reclaim warning events: a Released volume awaiting deletion, or a volume // whose reclaim already failed. func PVNeedsReclaimEvents(pv *corev1.PersistentVolume) bool { if pv.DeletionTimestamp != nil { return false } return pv.Status.Phase == corev1.VolumeFailed || pv.Status.Phase == corev1.VolumeReleased && pv.Spec.PersistentVolumeReclaimPolicy == corev1.PersistentVolumeReclaimDelete } // isReclaimWarning matches the warnings the PV controllers record when a // reclaim fails. Recycling failures only apply once the volume is Failed: a // Released volume is judged on deletion alone. func isReclaimWarning(pv *corev1.PersistentVolume, reason string) bool { return reason == "VolumeFailedDelete" || reason == "VolumeFailedRecycle" && pv.Status.Phase == corev1.VolumeFailed } type pvReclaimWarning struct { event *corev1.Event at time.Time } func latestPVReclaimWarnings(pvs []*corev1.PersistentVolume, events []*corev1.Event, now time.Time) map[types.UID]pvReclaimWarning { volumes := map[types.UID]*corev1.PersistentVolume{} for _, pv := range pvs { if pv.UID != "" && PVNeedsReclaimEvents(pv) { volumes[pv.UID] = pv } } latest := map[types.UID]pvReclaimWarning{} if len(volumes) == 0 { return latest } for _, e := range events { ref := e.InvolvedObject pv := volumes[ref.UID] if pv == nil || e.Type != corev1.EventTypeWarning || !isReclaimWarning(pv, e.Reason) || ref.Kind != "PersistentVolume" || ref.Name != pv.Name || ref.Namespace != "" || resourceid.GroupFromAPIVersion(ref.APIVersion) != "" { continue } at := e.LastTimestamp.Time if e.Series != nil && e.Series.LastObservedTime.Time.After(at) { at = e.Series.LastObservedTime.Time } if at.IsZero() { at = e.EventTime.Time } if at.IsZero() { at = e.CreationTimestamp.Time } if at.IsZero() || at.After(now) || !at.After(latest[ref.UID].at) { continue } if pv.Status.LastPhaseTransitionTime != nil && !pv.Status.LastPhaseTransitionTime.IsZero() && at.Before(pv.Status.LastPhaseTransitionTime.Time) { continue } latest[ref.UID] = pvReclaimWarning{event: e, at: at} } return latest } func asSentence(message string) string { message = strings.TrimSpace(message) if message == "" || strings.HasSuffix(message, ".") || strings.HasSuffix(message, "!") || strings.HasSuffix(message, "?") { return message } return message + "." }