package helmhistory import ( "fmt" "regexp" "sort" "strings" "time" "github.com/skyhook-io/radar/pkg/timeutil" ) const ( KindReleaseFailed OperationKind = "release_failed" KindUpgradeFailed OperationKind = "upgrade_failed" KindUpgradeRolledBack OperationKind = "upgrade_rolled_back" KindRollback OperationKind = "rollback" KindPending OperationKind = "pending" StatusFailed OperationStatus = "failed" StatusRolledBack OperationStatus = "rolled_back" StatusCompleted OperationStatus = "completed" StatusStuck OperationStatus = "stuck_pending" ConfidenceHigh Confidence = "high" ConfidenceMedium Confidence = "medium" ConfidenceLow Confidence = "low" SourceStatus Source = "helm_status" SourceHistory Source = "helm_history" ) const defaultPendingStuckAfter = 10 * time.Minute var ( rollbackToPattern = regexp.MustCompile(`(?i)^rollback to ([0-9]+)(?:\b|$)`) upgradeFailurePattern = regexp.MustCompile(`(?i)^upgrade(?:\s+"[^"]+")?\s+failed:`) ) type OperationKind string type OperationStatus string type Confidence string type Source string type Revision struct { Revision int Status string Chart string AppVersion string Description string Updated time.Time Deleted time.Time } type Operation struct { Kind OperationKind `json:"kind"` Status OperationStatus `json:"status"` Source Source `json:"source"` Confidence Confidence `json:"confidence"` Message string `json:"message"` RawMessage string `json:"rawMessage,omitempty"` Evidence string `json:"evidence,omitempty"` FailureDescription string `json:"failureDescription,omitempty"` Revision int `json:"revision,omitempty"` FailedRevision int `json:"failedRevision,omitempty"` RollbackRevision int `json:"rollbackRevision,omitempty"` TargetRevision int `json:"targetRevision,omitempty"` PendingStatus string `json:"pendingStatus,omitempty"` Updated time.Time `json:"updated,omitempty"` } type Options struct { PendingStuckAfter time.Duration MaxOperations int Now time.Time } type Analysis struct { LastOperation *Operation `json:"lastOperation,omitempty"` Operations []Operation `json:"operations,omitempty"` } func Analyze(releaseName string, currentRevision int, revisions []Revision, opts Options) Analysis { if len(revisions) == 0 { return Analysis{} } pendingStuckAfter := opts.PendingStuckAfter if pendingStuckAfter <= 0 { pendingStuckAfter = defaultPendingStuckAfter } now := opts.Now if now.IsZero() { now = time.Now() } ordered := append([]Revision(nil), revisions...) sort.Slice(ordered, func(i, j int) bool { return ordered[i].Revision < ordered[j].Revision }) var ops []Operation for i := 0; i < len(ordered); i++ { rev := ordered[i] status := normalizeStatus(rev.Status) if status == "failed" { if i+1 < len(ordered) { next := ordered[i+1] if next.Revision == rev.Revision+1 && isCompletedRevisionStatus(next.Status) { if target, ok := rollbackTarget(next.Description); ok { op := rolledBackOperation(rev, next, target) ops = append(ops, op) i++ continue } } } ops = append(ops, failedOperation(releaseName, rev, SourceHistory)) continue } if isCompletedRevisionStatus(status) { if target, ok := rollbackTarget(rev.Description); ok { ops = append(ops, rollbackOperation(rev, target)) } } } current, hasCurrent := findCurrent(ordered, currentRevision) resolvedCurrentRevision := currentRevision if hasCurrent { resolvedCurrentRevision = current.Revision } var live *Operation if hasCurrent { status := normalizeStatus(current.Status) started := current.Updated if status == "uninstalling" { started = current.Deleted } switch { case status == "failed": op := failedOperation(releaseName, current, SourceStatus) live = &op case isPending(status) && !started.IsZero() && now.Sub(started) >= pendingStuckAfter: op := pendingOperation(current, now.Sub(started)) live = &op } } if live != nil && live.Status == StatusFailed { ops = withoutDuplicateLiveFailure(ops, *live) } sort.Slice(ops, func(i, j int) bool { return operationRevision(ops[i]) > operationRevision(ops[j]) }) if opts.MaxOperations > 0 && len(ops) > opts.MaxOperations { ops = ops[:opts.MaxOperations] } var last *Operation if live != nil { last = live } else { for i := range ops { if operationRevision(ops[i]) == resolvedCurrentRevision && (ops[i].Kind == KindUpgradeRolledBack || ops[i].Kind == KindRollback) { op := ops[i] last = &op break } } } return Analysis{LastOperation: last, Operations: ops} } func rolledBackOperation(failed, rollback Revision, target int) Operation { return Operation{ Kind: KindUpgradeRolledBack, Status: StatusRolledBack, Source: SourceHistory, Confidence: ConfidenceMedium, Message: fmt.Sprintf("Upgrade failed at rev %d; Helm rolled back to rev %d as rev %d.", failed.Revision, target, rollback.Revision), Evidence: fmt.Sprintf("failed revision %d followed by rollback revision %d", failed.Revision, rollback.Revision), FailureDescription: strings.TrimSpace(failed.Description), FailedRevision: failed.Revision, RollbackRevision: rollback.Revision, TargetRevision: target, Updated: rollback.Updated, } } func failedOperation(releaseName string, rev Revision, source Source) Operation { kind := KindReleaseFailed message := fmt.Sprintf("Release failed at rev %d.", rev.Revision) if isUpgradeFailureDescription(rev.Description) { kind = KindUpgradeFailed message = fmt.Sprintf("Upgrade failed at rev %d.", rev.Revision) } if rev.Description != "" { message = message + " " + rev.Description } rawMessage := "" if IsReadinessTimeoutMessage(message) { rawMessage = strings.TrimSpace(rev.Description) message = readinessTimeoutMessage(releaseName) } evidence := "Helm history revision status is failed" if source == SourceStatus { evidence = "latest Helm revision status is failed" } return Operation{ Kind: kind, Status: StatusFailed, Source: source, Confidence: ConfidenceHigh, Message: message, RawMessage: rawMessage, Evidence: evidence, Revision: rev.Revision, Updated: rev.Updated, } } func IsReadinessTimeoutMessage(msg string) bool { lower := strings.ToLower(msg) if !strings.Contains(lower, "context deadline exceeded") || !strings.Contains(lower, "status: inprogress") { return false } return strings.Contains(lower, "not ready") || strings.Contains(lower, "available: 0/") } func readinessTimeoutMessage(releaseName string) string { name := strings.TrimSpace(releaseName) if name == "" { return "Helm release did not become ready before Helm timed out." } return fmt.Sprintf("Helm release %q did not become ready before Helm timed out.", name) } func rollbackOperation(rev Revision, target int) Operation { return Operation{ Kind: KindRollback, Status: StatusCompleted, Source: SourceHistory, Confidence: ConfidenceMedium, Message: fmt.Sprintf("Helm rolled back to rev %d as rev %d.", target, rev.Revision), Evidence: fmt.Sprintf("revision %d description indicates rollback", rev.Revision), Revision: rev.Revision, TargetRevision: target, Updated: rev.Updated, } } func pendingOperation(rev Revision, age time.Duration) Operation { evidence := "latest Helm revision is still pending" updated := rev.Updated if normalizeStatus(rev.Status) == "uninstalling" { evidence = "latest Helm revision is still uninstalling, timed from Helm Info.Deleted" updated = rev.Deleted } return Operation{ Kind: KindPending, Status: StatusStuck, Source: SourceStatus, Confidence: ConfidenceHigh, Message: fmt.Sprintf("Release has been %s for %s.", rev.Status, timeutil.FormatAgeShort(age)), Evidence: evidence, Revision: rev.Revision, PendingStatus: normalizeStatus(rev.Status), Updated: updated, } } func rollbackTarget(description string) (int, bool) { matches := rollbackToPattern.FindStringSubmatch(strings.TrimSpace(description)) if len(matches) != 2 { return 0, false } var target int if _, err := fmt.Sscanf(matches[1], "%d", &target); err != nil || target <= 0 { return 0, false } return target, true } func isUpgradeFailureDescription(description string) bool { description = strings.TrimSpace(description) if description == "" { return false } return upgradeFailurePattern.MatchString(description) } func findCurrent(ordered []Revision, currentRevision int) (Revision, bool) { for i := len(ordered) - 1; i >= 0; i-- { if currentRevision <= 0 || ordered[i].Revision == currentRevision { return ordered[i], true } } return Revision{}, false } func isPending(status string) bool { status = normalizeStatus(status) return status == "pending-install" || status == "pending-upgrade" || status == "pending-rollback" || status == "uninstalling" } func isCompletedRevisionStatus(status string) bool { status = normalizeStatus(status) return status == "deployed" || status == "superseded" } func normalizeStatus(status string) string { return strings.ToLower(strings.TrimSpace(status)) } func withoutDuplicateLiveFailure(ops []Operation, live Operation) []Operation { out := ops[:0] for _, op := range ops { if op.Status == StatusFailed && op.Revision == live.Revision { continue } out = append(out, op) } return out } func operationRevision(op Operation) int { if op.RollbackRevision > 0 { return op.RollbackRevision } if op.Revision > 0 { return op.Revision } return op.FailedRevision }