/* Copyright 2026 The Jumpstarter Authors Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. */ // Package exporterset implements the ExporterSet controller — the scaling // engine for virtual exporter pools (JEP-0014). // // Scaling rules, evaluated in priority order each reconcile: // 1. Floor: replicas < minReplicas → scale up // 2. Warm buffer: available < minAvailableReplicas → scale up // 3. Demand: pending Leases match our labels, no capacity → scale up // 4. Excess: available > min for longer than cooldown → disable one // // Creates at most 1 exporter per reconcile to avoid cache-race duplicates. // Graceful scale-down: disable → drain leases → delete on next cycle. // // Why not HPA/KEDA: scaling depends on Exporter CRD fields (leaseRef, // enabled, online) and Lease state (pending condition) — not expressible // as standard metrics. The disable-drain-delete protocol has no HPA analog. package exporterset import ( "context" "encoding/json" "errors" "fmt" "maps" "sync" "time" corev1 "k8s.io/api/core/v1" networkingv1 "k8s.io/api/networking/v1" apiextensionsv1 "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1" apierrors "k8s.io/apimachinery/pkg/api/errors" "k8s.io/apimachinery/pkg/api/meta" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/labels" "k8s.io/apimachinery/pkg/runtime" "k8s.io/apimachinery/pkg/types" "k8s.io/apimachinery/pkg/util/validation" "k8s.io/client-go/tools/record" ctrl "sigs.k8s.io/controller-runtime" "sigs.k8s.io/controller-runtime/pkg/client" "sigs.k8s.io/controller-runtime/pkg/controller/controllerutil" "sigs.k8s.io/controller-runtime/pkg/handler" "sigs.k8s.io/controller-runtime/pkg/log" "sigs.k8s.io/controller-runtime/pkg/reconcile" jumpstarterdevv1alpha1 "github.com/jumpstarter-dev/jumpstarter/controller/api/v1alpha1" virtualtargetv1alpha1 "github.com/jumpstarter-dev/jumpstarter/controller/api/virtualtarget/v1alpha1" ) const ( annotationSurplusSince = "exporterset.jumpstarter.dev/surplus-since" // Bridges the grandparent lookup (ExporterSet -> Exporter -> Pod), and // tells clients which pool an exporter came from: membership otherwise // lives only in ownerReferences, which the client API does not expose. labelExporterSetName = "exporterset.jumpstarter.dev/name" // The VirtualTargetClass backing the pool, and the provisioner named by // that class. Together they let clients tell how an exporter is // provisioned without cluster access: the provisioner is a property of // the class, which a client cannot read, so it is carried on the Exporter. // // All three identity labels are client-visible by default. The operator // lists them in hiddenLabels.keys so they stay out of `jmp get exporters` // unless --show-hidden-labels is passed; they remain usable in selectors. labelVirtualTargetClass = "exporterset.jumpstarter.dev/class" labelProvisioner = "exporterset.jumpstarter.dev/provisioner" defaultScaleDownCooldown = 5 * time.Minute // exporterOfflineThreshold mirrors the Exporter controller // (internal/controller/exporter_controller.go): an exporter is considered // offline only once its last heartbeat is older than this. If either copy // changes, both must change. exporterOfflineThreshold = time.Minute // exporterStartupDeadline bounds how long a deployed off-cluster Exporter // that has never registered may hold a pool slot. It is measured from // deployment completion (Deployer.DeployedAt), so a slow deploy is not // recycled the moment it succeeds. Past the deadline the deployment is // wedged — containers running, no registration (e.g. the exporter cannot // reach the controller) — and ExitAndReplace recycles it instead of // leaking the slot for the life of the pool. A deployment that fails // outright never sets the deployed marker: it is retried by the next // reconcile and keeps its slot until it succeeds. exporterStartupDeadline = 10 * time.Minute // kindExporter is the Kind string used in OwnerReference lookups. kindExporter = "Exporter" finalizerRemoteCleanupPrefix = "exporterset.jumpstarter.dev/remote-cleanup-" ) // ExporterSetReconciler reconciles an ExporterSet object. // It watches ExporterSets, Exporters, and Leases to maintain a warm pool // of virtual exporter instances with configurable autoscaling. // Provisioner-specific logic (Pod rendering, cleanup) is delegated to the // Provisioner interface implementation. type ExporterSetReconciler struct { client.Client Scheme *runtime.Scheme Recorder record.EventRecorder Provisioner Provisioner lastScaleDownMu sync.Mutex // Guards against disabling multiple exporters within the same cooldown // when the informer cache is stale. LastScaleDownAction map[types.NamespacedName]time.Time } // +kubebuilder:rbac:groups=networking.k8s.io,resources=networkpolicies,verbs=get;list;watch;create;update;patch // +kubebuilder:rbac:groups=virtualtarget.jumpstarter.dev,resources=exportersets,verbs=get;list;watch;create;update;patch;delete // +kubebuilder:rbac:groups=virtualtarget.jumpstarter.dev,resources=exportersets/status,verbs=get;update;patch // +kubebuilder:rbac:groups=virtualtarget.jumpstarter.dev,resources=exportersets/finalizers,verbs=update // +kubebuilder:rbac:groups=virtualtarget.jumpstarter.dev,resources=exportersets/scale,verbs=get;update;patch // +kubebuilder:rbac:groups=virtualtarget.jumpstarter.dev,resources=virtualtargetclasses,verbs=get;list;watch // +kubebuilder:rbac:groups=jumpstarter.dev,resources=exporters,verbs=get;list;watch;create;update;patch;delete // +kubebuilder:rbac:groups=jumpstarter.dev,resources=leases,verbs=get;list;watch // +kubebuilder:rbac:groups=core,resources=pods,verbs=get;list;watch;create;update;patch;delete // +kubebuilder:rbac:groups="",resources=events,verbs=create;patch // +kubebuilder:rbac:groups="",resources=secrets,verbs=get;list;watch;create;update;patch // +kubebuilder:rbac:groups="",resources=configmaps,verbs=get;list;watch // Reconcile is the main reconciliation loop for ExporterSet resources. // It resolves the referenced VirtualTargetClass, counts owned instances, // and scales the pool to maintain the desired number of available replicas. func (r *ExporterSetReconciler) Reconcile(ctx context.Context, req ctrl.Request) (ctrl.Result, error) { logger := log.FromContext(ctx) var exporterSet virtualtargetv1alpha1.ExporterSet if err := r.Get(ctx, req.NamespacedName, &exporterSet); err != nil { if client.IgnoreNotFound(err) == nil { // ExporterSet deleted — clean up in-memory state. r.lastScaleDownMu.Lock() if r.LastScaleDownAction != nil { delete(r.LastScaleDownAction, req.NamespacedName) } r.lastScaleDownMu.Unlock() return ctrl.Result{}, nil } return ctrl.Result{}, fmt.Errorf("Reconcile: unable to get ExporterSet: %w", err) } // Deletion must be handled before resolving the VirtualTargetClass. If the // class has already been removed, provisioner cleanup must fail explicitly // and retain the finalizer rather than leaving the ExporterSet silently stuck. if !exporterSet.DeletionTimestamp.IsZero() { _, err := r.reconcileFinalizer(ctx, &exporterSet) return ctrl.Result{}, err } vtc, err := r.resolveVirtualTargetClass(ctx, &exporterSet) if err != nil { return ctrl.Result{}, err } if vtc == nil { return ctrl.Result{}, nil } // Only reconcile ExporterSets whose class matches our provisioner if vtc.Spec.Provisioner != r.Provisioner.Name() { return ctrl.Result{}, nil } done, err := r.reconcileFinalizer(ctx, &exporterSet) if err != nil { return ctrl.Result{}, err } if done { return ctrl.Result{}, nil } if err := r.syncNetworkPolicy(ctx, &exporterSet); err != nil { return ctrl.Result{}, err } logger.Info("reconciling ExporterSet", "name", exporterSet.Name, "provisioner", vtc.Spec.Provisioner, "minReplicas", exporterSet.Spec.MinReplicas, "maxReplicas", exporterSet.Spec.MaxReplicas, "minAvailableReplicas", exporterSet.Spec.MinAvailableReplicas, ) prev := snapshotConditions(&exporterSet) // Exporters are found through spec.selector. If the labels this set stamps // on its exporters don't satisfy it (e.g. the selector pins an // exporterset.jumpstarter.dev/* value the identity labels override), new // exporters would be invisible to scaling and created without bound, and // relabelling existing ones would orphan them. Stop until the spec is fixed. if mismatch, err := r.selectorMismatch(ctx, &exporterSet); err != nil { return ctrl.Result{}, err } else if mismatch != "" { logger.Info("ExporterSet selector does not match its exporter labels; not scaling", "detail", mismatch) return r.reportUnavailable(ctx, &exporterSet, "SelectorMismatch", mismatch) } ownedExporters, err := r.listOwnedExporters(ctx, &exporterSet) if err != nil { return ctrl.Result{}, err } // Keep identity labels current on exporters created before them, or after // the set's class changed. if err := r.reconcileExporterLabels(ctx, &exporterSet, ownedExporters); err != nil { return ctrl.Result{}, err } // Single Pod List shared by terminal cleanup and ensureExporterPods. podsByExporter, err := r.listPodsGroupedByExporter(ctx, &exporterSet) if err != nil { return ctrl.Result{}, err } // Clean up drained (disabled+unleased) exporters and ExitAndReplace // terminal pods (Succeeded/Failed) before computing pool state. cleanupResult, cleaned, err := r.cleanupBeforeScale( ctx, &exporterSet, ownedExporters, podsByExporter, prev.available, prev.progressing, prev.degraded, prev.scalingLimited, ) if err != nil { return cleanupResult, err } if cleaned { return cleanupResult, nil } state := computePoolState(ownedExporters) mergedParameters, err := deepMergeParameters(vtc.Spec.Parameters, exporterSet.Spec.Parameters) if err != nil { return ctrl.Result{}, fmt.Errorf("unable to merge parameters: %w", err) } var result ctrl.Result // Scale-up logic. Uses potentiallyAvailable (replicas - leased) to avoid // cache-race over-creation. One exporter per reconcile; Owns watch // triggers subsequent cycles. scaled, err := r.reconcileScaleUp(ctx, &exporterSet, state) if err != nil { return ctrl.Result{}, err } if !scaled { result, err = r.reconcileScaleDown(ctx, &exporterSet, ownedExporters, state) if err != nil { return ctrl.Result{}, err } } // Phase 2: create config Secrets + Pods for Exporters that now have credentials. // Returns true when some Exporters are still waiting; we requeue to retry // rather than relying solely on the Owns(&Exporter{}) watch event. waiting, ensureErr := r.ensureExporterPods(ctx, &exporterSet, vtc, mergedParameters, ownedExporters, podsByExporter) if ensureErr != nil { return ctrl.Result{}, ensureErr } if waiting && result.RequeueAfter == 0 { result.RequeueAfter = 5 * time.Second } // Update status counts, conditions, and emit events. if err := r.reconcileStatusCounts(ctx, &exporterSet); err != nil { return ctrl.Result{}, err } r.reconcileConditions(&exporterSet) if err := r.Status().Update(ctx, &exporterSet); err != nil { return requeueConflict(logger, err) } r.emitConditionEvents(&exporterSet, prev.available, prev.progressing, prev.degraded, prev.scalingLimited) // Carry the startup-deadline requeue from cleanupBeforeScale into the // final result when it fires sooner than any other pending requeue. if cleanupResult.RequeueAfter > 0 && (result.RequeueAfter == 0 || cleanupResult.RequeueAfter < result.RequeueAfter) { result.RequeueAfter = cleanupResult.RequeueAfter } return result, nil } func (r *ExporterSetReconciler) cleanupBeforeScale( ctx context.Context, es *virtualtargetv1alpha1.ExporterSet, exporters []jumpstarterdevv1alpha1.Exporter, podsByExporter map[string][]corev1.Pod, prevAvailable, prevProgressing, prevDegraded, prevScalingLimited bool, ) (ctrl.Result, bool, error) { deleted, err := r.cleanupDisabledExporters(ctx, es, exporters) if err != nil { return ctrl.Result{}, false, err } var requeueAfter time.Duration if !deleted { var terminalErr error deleted, requeueAfter, terminalErr = r.cleanupTerminalExporters(ctx, es, exporters, podsByExporter) if terminalErr != nil { return ctrl.Result{}, false, terminalErr } } if !deleted { if requeueAfter > 0 { return ctrl.Result{RequeueAfter: requeueAfter}, false, nil } return ctrl.Result{}, false, nil } if err := r.reconcileStatusCounts(ctx, es); err != nil { return ctrl.Result{}, false, err } r.reconcileConditions(es) if err := r.Status().Update(ctx, es); err != nil { result, err := requeueConflict(log.FromContext(ctx), err) return result, true, err } r.emitConditionEvents(es, prevAvailable, prevProgressing, prevDegraded, prevScalingLimited) return ctrl.Result{}, true, nil } // NetworkPolicyProvisioner isolates backend listeners before any workload Pod is created. type NetworkPolicyProvisioner interface { RenderNetworkPolicy(*virtualtargetv1alpha1.ExporterSet) *networkingv1.NetworkPolicy } func (r *ExporterSetReconciler) syncNetworkPolicy(ctx context.Context, es *virtualtargetv1alpha1.ExporterSet) error { provisioner, ok := r.Provisioner.(NetworkPolicyProvisioner) if !ok { return nil } desired := provisioner.RenderNetworkPolicy(es) policy := &networkingv1.NetworkPolicy{ObjectMeta: metav1.ObjectMeta{Name: desired.Name, Namespace: desired.Namespace}} _, err := controllerutil.CreateOrUpdate(ctx, r.Client, policy, func() error { if !policy.CreationTimestamp.IsZero() && !metav1.IsControlledBy(policy, es) { return fmt.Errorf("network policy %s is not owned by ExporterSet", policy.Name) } if err := ctrl.SetControllerReference(es, policy, r.Scheme); err != nil { return err } policy.Spec = *desired.Spec.DeepCopy() return nil }) if err != nil { return fmt.Errorf("ensure runtime network isolation: %w", err) } return nil } // reconcileFinalizer handles the finalizer lifecycle: on deletion it runs // remote cleanup and removes the finalizer; otherwise it ensures the // finalizer is present. Returns (true, nil) when the caller should return // immediately (object is being deleted or finalizer was just added). func (r *ExporterSetReconciler) reconcileFinalizer( ctx context.Context, exporterSet *virtualtargetv1alpha1.ExporterSet, ) (bool, error) { logger := log.FromContext(ctx) finalizer := finalizerRemoteCleanupPrefix + r.Provisioner.Name() if !exporterSet.DeletionTimestamp.IsZero() { if controllerutil.ContainsFinalizer(exporterSet, finalizer) { ownedExporters, err := r.listOwnedExporters(ctx, exporterSet) if err != nil { return false, err } for i := range ownedExporters { if err := r.Provisioner.Cleanup(ctx, exporterSet, &ownedExporters[i]); err != nil { logger.Error(err, "failed to clean up remote resources for exporter", "exporter", ownedExporters[i].Name) return false, fmt.Errorf("clean up remote resources for exporter %s: %w", ownedExporters[i].Name, err) } } controllerutil.RemoveFinalizer(exporterSet, finalizer) if err := r.Update(ctx, exporterSet); err != nil { return false, err } } return true, nil } if !controllerutil.ContainsFinalizer(exporterSet, finalizer) { controllerutil.AddFinalizer(exporterSet, finalizer) if err := r.Update(ctx, exporterSet); err != nil { return false, err } } return false, nil } // resolveVirtualTargetClass fetches the VirtualTargetClass for an ExporterSet. // Returns (nil, nil) when the VTC is not found (status is updated in-place). func (r *ExporterSetReconciler) resolveVirtualTargetClass( ctx context.Context, exporterSet *virtualtargetv1alpha1.ExporterSet, ) (*virtualtargetv1alpha1.VirtualTargetClass, error) { logger := log.FromContext(ctx) var vtc virtualtargetv1alpha1.VirtualTargetClass vtcKey := client.ObjectKey{ Namespace: exporterSet.Namespace, Name: exporterSet.Spec.VirtualTargetClassName, } if err := r.Get(ctx, vtcKey, &vtc); err != nil { if apierrors.IsNotFound(err) { logger.Info("VirtualTargetClass not found", "virtualTargetClassName", exporterSet.Spec.VirtualTargetClassName) prev := snapshotConditions(exporterSet) if countErr := r.reconcileStatusCounts(ctx, exporterSet); countErr != nil { return nil, countErr } r.reconcileConditions(exporterSet) meta.SetStatusCondition(&exporterSet.Status.Conditions, metav1.Condition{ Type: string(virtualtargetv1alpha1.ExporterSetConditionAvailable), Status: metav1.ConditionFalse, ObservedGeneration: exporterSet.Generation, Reason: "VirtualTargetClassNotFound", Message: fmt.Sprintf("VirtualTargetClass %q not found", exporterSet.Spec.VirtualTargetClassName), }) if updateErr := r.Status().Update(ctx, exporterSet); updateErr != nil { if apierrors.IsConflict(updateErr) { logger.Info("conflict on status update, will retry") } return nil, updateErr } r.emitConditionEvents(exporterSet, prev.available, prev.progressing, prev.degraded, prev.scalingLimited) return nil, nil } return nil, fmt.Errorf("unable to get VirtualTargetClass %q: %w", exporterSet.Spec.VirtualTargetClassName, err) } return &vtc, nil } type conditionSnapshot struct { available, progressing, degraded, scalingLimited bool } func snapshotConditions(es *virtualtargetv1alpha1.ExporterSet) conditionSnapshot { return conditionSnapshot{ available: meta.IsStatusConditionTrue( es.Status.Conditions, string(virtualtargetv1alpha1.ExporterSetConditionAvailable), ), progressing: meta.IsStatusConditionTrue( es.Status.Conditions, string(virtualtargetv1alpha1.ExporterSetConditionProgressing), ), degraded: meta.IsStatusConditionTrue( es.Status.Conditions, string(virtualtargetv1alpha1.ExporterSetConditionDegraded), ), scalingLimited: meta.IsStatusConditionTrue( es.Status.Conditions, string(virtualtargetv1alpha1.ExporterSetConditionScalingLimited), ), } } type poolState struct { replicas int32 ready int32 available int32 leased int32 } func computePoolState(exporters []jumpstarterdevv1alpha1.Exporter) poolState { var s poolState for i := range exporters { exp := &exporters[i] s.replicas++ isOnline := meta.IsStatusConditionTrue( exp.Status.Conditions, string(jumpstarterdevv1alpha1.ExporterConditionTypeOnline), ) isEnabled := exp.IsEnabled() isLeased := exp.Status.LeaseRef != nil if isOnline { s.ready++ } if isLeased { s.leased++ } if isOnline && isEnabled && !isLeased { s.available++ } } return s } // scaleUp creates Exporter CRs only (phase 1). Pod creation happens in // ensureExporterPods once credentials and endpoint are ready. func (r *ExporterSetReconciler) scaleUp( ctx context.Context, es *virtualtargetv1alpha1.ExporterSet, count int32, ) error { logger := log.FromContext(ctx) for range count { exporter := &jumpstarterdevv1alpha1.Exporter{ ObjectMeta: metav1.ObjectMeta{ GenerateName: es.Name + "-", Namespace: es.Namespace, Labels: r.exporterLabels(ctx, es), Annotations: maps.Clone(es.Spec.Template.Metadata.Annotations), }, Spec: jumpstarterdevv1alpha1.ExporterSpec{ Enabled: new(true), }, } if err := ctrl.SetControllerReference(es, exporter, r.Scheme); err != nil { return fmt.Errorf("unable to set owner reference on Exporter: %w", err) } if err := r.Create(ctx, exporter); err != nil { return fmt.Errorf("unable to create Exporter: %w", err) } logger.Info("created Exporter (awaiting credentials)", "exporter", exporter.Name) if r.Recorder != nil { r.Recorder.Eventf(es, corev1.EventTypeNormal, "ScaleUp", "Created Exporter %s (awaiting credentials)", exporter.Name) } } return nil } // ensureExporterPods creates a config Secret and Pod for each Exporter that // has credentials but no Pod yet. Config Secrets are always synced so token // rotation takes effect without a Pod restart (Kubernetes refreshes Secret-backed // volume mounts automatically). // // For off-cluster provisioners (those implementing Deployer), this method // calls Deploy instead of creating a Pod — see ensureExporterDeployments. // // Returns true if any Exporter is still waiting for its credential Secret. func (r *ExporterSetReconciler) ensureExporterPods( ctx context.Context, es *virtualtargetv1alpha1.ExporterSet, vtc *virtualtargetv1alpha1.VirtualTargetClass, mergedParameters map[string]any, ownedExporters []jumpstarterdevv1alpha1.Exporter, podsByExporter map[string][]corev1.Pod, ) (waiting bool, err error) { // Off-cluster provisioners manage instances via SSH/API instead of Pods. if deployer, ok := r.Provisioner.(Deployer); ok { return r.ensureExporterDeployments(ctx, es, vtc, mergedParameters, ownedExporters, deployer) } return r.ensureExporterPodsInCluster(ctx, es, vtc, mergedParameters, ownedExporters, podsByExporter) } // ensureExporterPodsInCluster is the in-cluster Pod path (unchanged from // original ensureExporterPods logic). func (r *ExporterSetReconciler) ensureExporterPodsInCluster( ctx context.Context, es *virtualtargetv1alpha1.ExporterSet, vtc *virtualtargetv1alpha1.VirtualTargetClass, mergedParameters map[string]any, ownedExporters []jumpstarterdevv1alpha1.Exporter, podsByExporter map[string][]corev1.Pod, ) (waiting bool, err error) { logger := log.FromContext(ctx) // Existence set derived from the shared Reconcile Pod list (no second List). exportersWithPod := make(map[string]struct{}, len(podsByExporter)) for name := range podsByExporter { exportersWithPod[name] = struct{}{} } // CA bundle is read lazily — only when at least one exporter needs it. var caBundle string var caRead bool for i := range ownedExporters { exp := &ownedExporters[i] if !exp.IsEnabled() { continue } if exp.Status.Credential == nil || exp.Status.Endpoint == "" { logger.V(1).Info("waiting for credential", "exporter", exp.Name) waiting = true continue } if !caRead { caBundle, err = r.readCABundle(ctx, vtc) if err != nil { return false, err } caRead = true } if _, hasPod := exportersWithPod[exp.Name]; !hasPod { // Render the Pod first so API-backed provisioners can create remote // resources needed while enriching the exporter configuration. if err := r.createExporterPod(ctx, es, vtc, mergedParameters, mergeImages(vtc.Spec.Images, es.Spec.Images), exp); err != nil { return waiting, err } } // Sync config Secret on every reconcile so token rotation takes effect. if err := r.syncConfigSecret(ctx, es, vtc, exp, caBundle, mergedParameters); err != nil { return waiting, err } } return waiting, nil } // ensureExporterDeployments is the off-cluster path: calls Deployer.Deploy // for exporters that have credentials but haven't been deployed yet. func (r *ExporterSetReconciler) ensureExporterDeployments( ctx context.Context, es *virtualtargetv1alpha1.ExporterSet, vtc *virtualtargetv1alpha1.VirtualTargetClass, mergedParameters map[string]any, ownedExporters []jumpstarterdevv1alpha1.Exporter, deployer Deployer, ) (waiting bool, err error) { logger := log.FromContext(ctx) var caBundle string var caRead bool images := mergeImages(vtc.Spec.Images, es.Spec.Images) for i := range ownedExporters { exp := &ownedExporters[i] if !exp.IsEnabled() { continue } if exp.Status.Credential == nil || exp.Status.Endpoint == "" { logger.V(1).Info("waiting for credential", "exporter", exp.Name) waiting = true continue } if !caRead { caBundle, err = r.readCABundle(ctx, vtc) if err != nil { return false, err } caRead = true } _, deployed := deployer.DeployedAt(exp) if deployed { continue } if err := deployer.Deploy(ctx, es, vtc, mergedParameters, images, exp, caBundle); err != nil { return waiting, fmt.Errorf("deploy %s: %w", exp.Name, err) } logger.Info("deployed exporter on remote host", "exporter", exp.Name) if r.Recorder != nil { r.Recorder.Eventf(es, corev1.EventTypeNormal, "Deployed", "Deployed Exporter %s on remote host", exp.Name) } } return waiting, nil } // syncConfigSecret creates or updates the per-exporter config Secret so the // exporter sidecar always has a fresh token and correct configuration. func (r *ExporterSetReconciler) syncConfigSecret( ctx context.Context, es *virtualtargetv1alpha1.ExporterSet, vtc *virtualtargetv1alpha1.VirtualTargetClass, exp *jumpstarterdevv1alpha1.Exporter, caBundle string, mergedParameters map[string]any, ) error { configSecret, err := r.buildExporterConfigSecret(ctx, es, vtc, exp, caBundle, mergedParameters) if err != nil { if r.Recorder != nil { r.Recorder.Eventf(es, corev1.EventTypeWarning, "ExporterConfigFailed", "Cannot build config for Exporter %s: %v", exp.Name, err) } return fmt.Errorf("build config for %s: %w", exp.Name, err) } existing := &corev1.Secret{ ObjectMeta: metav1.ObjectMeta{ Name: configSecret.Name, Namespace: configSecret.Namespace, }, } _, err = controllerutil.CreateOrUpdate(ctx, r.Client, existing, func() error { if err := ctrl.SetControllerReference(exp, existing, r.Scheme); err != nil { return fmt.Errorf("set owner on config Secret for %s: %w", exp.Name, err) } existing.Labels = configSecret.Labels existing.Data = configSecret.Data return nil }) return err } // createExporterPod issues a Pod for a single Exporter that has credentials // but no Pod yet. Kubernetes retries mounting the config Secret until the // subsequent syncConfigSecret call creates it. func (r *ExporterSetReconciler) createExporterPod( ctx context.Context, es *virtualtargetv1alpha1.ExporterSet, vtc *virtualtargetv1alpha1.VirtualTargetClass, mergedParameters map[string]any, images *virtualtargetv1alpha1.ImageOverrides, exp *jumpstarterdevv1alpha1.Exporter, ) error { logger := log.FromContext(ctx) pod, err := r.Provisioner.RenderPod(ctx, es, vtc, mergedParameters, images, exp) if err != nil { if r.Recorder != nil { r.Recorder.Eventf(es, corev1.EventTypeWarning, "PodRenderFailed", "Cannot render Pod for Exporter %s: %v", exp.Name, err) } return fmt.Errorf("render Pod for %s: %w", exp.Name, err) } if !injectConfigVolume(pod, exp.Name) { logger.Info("exporter container not found in pod spec; config volume not mounted", "exporter", exp.Name, "expectedContainer", exporterContainerName) } injectCredentialsSecretRef(pod, vtc) if pod.Labels == nil { pod.Labels = make(map[string]string) } pod.Labels[labelExporterSetName] = es.Name if err := ctrl.SetControllerReference(exp, pod, r.Scheme); err != nil { return fmt.Errorf("set owner on Pod for %s: %w", exp.Name, err) } if err := r.Create(ctx, pod); err != nil { if apierrors.IsAlreadyExists(err) { logger.V(1).Info("Pod already exists", "exporter", exp.Name, "pod", pod.Name) return nil } return fmt.Errorf("create Pod for %s: %w", exp.Name, err) } logger.Info("created Pod for Exporter", "exporter", exp.Name, "pod", pod.Name) if r.Recorder != nil { r.Recorder.Eventf(es, corev1.EventTypeNormal, "PodCreated", "Created Pod %s for Exporter %s", pod.Name, exp.Name) } return nil } // readCABundle fetches PEM CA data. If VTC specifies a CABundleConfigMapRef, // that is used. Otherwise falls back to the CertManager-generated ConfigMap. func (r *ExporterSetReconciler) readCABundle( ctx context.Context, vtc *virtualtargetv1alpha1.VirtualTargetClass, ) (string, error) { cmName := caConfigMapName cmKey := caConfigMapKey if vtc.Spec.CABundleConfigMapRef != nil { cmName = vtc.Spec.CABundleConfigMapRef.Name if vtc.Spec.CABundleConfigMapRef.Key != "" { cmKey = vtc.Spec.CABundleConfigMapRef.Key } else { cmKey = "ca-bundle.crt" } } var cm corev1.ConfigMap if err := r.Get(ctx, client.ObjectKey{ Name: cmName, Namespace: vtc.Namespace, }, &cm); err != nil { return "", fmt.Errorf("get CA bundle ConfigMap %q: %w", cmName, err) } ca, ok := cm.Data[cmKey] if !ok { return "", fmt.Errorf("CA bundle ConfigMap %q missing key %q", cmName, cmKey) } return ca, nil } // readCredentialToken reads the JWT from the Exporter's credential Secret. func (r *ExporterSetReconciler) readCredentialToken( ctx context.Context, exp *jumpstarterdevv1alpha1.Exporter, ) (string, error) { var secret corev1.Secret if err := r.Get(ctx, client.ObjectKey{ Name: exp.Status.Credential.Name, Namespace: exp.Namespace, }, &secret); err != nil { return "", fmt.Errorf("get credential Secret %q: %w", exp.Status.Credential.Name, err) } token, ok := secret.Data["token"] if !ok { return "", fmt.Errorf("credential Secret %q missing 'token' key", exp.Status.Credential.Name) } return string(token), nil } // listPodsGroupedByExporter maps Exporter names to their owned Pods. func (r *ExporterSetReconciler) listPodsGroupedByExporter( ctx context.Context, es *virtualtargetv1alpha1.ExporterSet, ) (map[string][]corev1.Pod, error) { var podList corev1.PodList if err := r.List(ctx, &podList, client.InNamespace(es.Namespace), client.MatchingLabels{labelExporterSetName: es.Name}, ); err != nil { return nil, fmt.Errorf("list Pods for ExporterSet %s: %w", es.Name, err) } byExporter := make(map[string][]corev1.Pod) for i := range podList.Items { pod := podList.Items[i] for _, ref := range pod.OwnerReferences { if ref.Kind == kindExporter && ref.Controller != nil && *ref.Controller { byExporter[ref.Name] = append(byExporter[ref.Name], pod) break } } } return byExporter, nil } // injectConfigVolume adds the config Secret as a volume and mounts it in the // "exporter" init-container. Returns false if that container isn't found. func injectConfigVolume(pod *corev1.Pod, exporterName string) bool { pod.Spec.Volumes = append(pod.Spec.Volumes, corev1.Volume{ Name: configVolumeName, VolumeSource: corev1.VolumeSource{ Secret: &corev1.SecretVolumeSource{ SecretName: configSecretPrefix + exporterName, }, }, }) mount := corev1.VolumeMount{ Name: configVolumeName, MountPath: configMountPath, ReadOnly: true, } // Exporter is the main container (default logs / ExitAndReplace lifecycle). for i := range pod.Spec.Containers { if pod.Spec.Containers[i].Name == exporterContainerName { pod.Spec.Containers[i].VolumeMounts = append(pod.Spec.Containers[i].VolumeMounts, mount) return true } } // Backward-compatible: older Pod templates ran exporter as a native sidecar. for i := range pod.Spec.InitContainers { if pod.Spec.InitContainers[i].Name == exporterContainerName { pod.Spec.InitContainers[i].VolumeMounts = append(pod.Spec.InitContainers[i].VolumeMounts, mount) return true } } return false } // injectCredentialsSecretRef mounts VTC.credentialsSecretRef as env vars in // the exporter container only. Credentials are scoped to containers that need // them to avoid widening the blast radius via auxiliary containers (keepalive). func injectCredentialsSecretRef(pod *corev1.Pod, vtc *virtualtargetv1alpha1.VirtualTargetClass) { if vtc.Spec.CredentialsSecretRef == nil { return } envFrom := corev1.EnvFromSource{ SecretRef: &corev1.SecretEnvSource{ LocalObjectReference: corev1.LocalObjectReference{ Name: vtc.Spec.CredentialsSecretRef.Name, }, }, } for i := range pod.Spec.InitContainers { if pod.Spec.InitContainers[i].Name == exporterContainerName { pod.Spec.InitContainers[i].EnvFrom = append( pod.Spec.InitContainers[i].EnvFrom, envFrom, ) } } for i := range pod.Spec.Containers { if pod.Spec.Containers[i].Name == exporterContainerName { pod.Spec.Containers[i].EnvFrom = append( pod.Spec.Containers[i].EnvFrom, envFrom, ) } } } // cleanupDisabledExporters deletes disabled, unleased exporters. Returns true if any were deleted. func (r *ExporterSetReconciler) cleanupDisabledExporters( ctx context.Context, es *virtualtargetv1alpha1.ExporterSet, exporters []jumpstarterdevv1alpha1.Exporter, ) (bool, error) { logger := log.FromContext(ctx) deleted := false for i := range exporters { exp := &exporters[i] if exp.IsEnabled() || exp.Status.LeaseRef != nil { continue } if err := r.Provisioner.Cleanup(ctx, es, exp); err != nil { return deleted, fmt.Errorf("unable to cleanup Exporter %s: %w", exp.Name, err) } if err := r.Delete(ctx, exp); err != nil && !apierrors.IsNotFound(err) { return deleted, fmt.Errorf("unable to delete Exporter %s: %w", exp.Name, err) } logger.Info("deleted disabled Exporter", "exporter", exp.Name) if r.Recorder != nil { r.Recorder.Eventf(es, corev1.EventTypeNormal, "ScaleDown", "Deleted Exporter %s", exp.Name) } deleted = true } return deleted, nil } // cleanupTerminalExporters deletes unleased exporters whose lifecycle has // ended. This is the ExitAndReplace recycle path: the exporter completes its // work, then the controller deletes the Exporter CR so scale-up can refill // minAvailableReplicas. Without this, Offline/Succeeded instances inflate // replicas, block warm-buffer refill at maxReplicas, and leave stale // resources behind. // // For in-cluster provisioners the terminal signal is a Pod in // Succeeded/Failed phase. For off-cluster provisioners (Deployer), it is // an exporter that was deployed but has gone offline — meaning the remote // container exited after the lease ended. // // InPlaceReuse skips this path entirely. // podsByExporter comes from a single List in Reconcile. func (r *ExporterSetReconciler) cleanupTerminalExporters( ctx context.Context, es *virtualtargetv1alpha1.ExporterSet, exporters []jumpstarterdevv1alpha1.Exporter, podsByExporter map[string][]corev1.Pod, ) (deleted bool, requeueAfter time.Duration, err error) { if es.Spec.RecycleStrategy == virtualtargetv1alpha1.RecycleStrategyInPlaceReuse { return false, 0, nil } logger := log.FromContext(ctx) deployer, isOffCluster := r.Provisioner.(Deployer) for i := range exporters { exp := &exporters[i] if exp.Status.LeaseRef != nil { continue } terminal := false if isOffCluster { // Off-cluster: terminal means "deployed and offline" — either // seen and since gone offline, or never registered and past the // startup deadline (a wedged deploy). "Never seen" on its own, // while still inside the deadline, is the normal // pre-registration state and must not recycle. ds, deployed := deployer.DeployedAt(exp) if deployed { if isExporterOffline(exp, ds) { terminal = true } else if exp.Status.LastSeen.IsZero() { if remaining := timeToStartupDeadline(exp, ds); remaining > 0 { if requeueAfter == 0 || remaining < requeueAfter { requeueAfter = remaining } } } } } else { // In-cluster: terminal means all Pods reached Succeeded/Failed. pods := podsByExporter[exp.Name] terminal = allPodsTerminal(pods) } if !terminal { continue } if err := r.Provisioner.Cleanup(ctx, es, exp); err != nil { return deleted, 0, fmt.Errorf("unable to cleanup Exporter %s: %w", exp.Name, err) } if err := r.Delete(ctx, exp); err != nil && !apierrors.IsNotFound(err) { return deleted, 0, fmt.Errorf("unable to delete Exporter %s: %w", exp.Name, err) } logger.Info("deleted terminal Exporter (ExitAndReplace)", "exporter", exp.Name, "offCluster", isOffCluster) if r.Recorder != nil { r.Recorder.Eventf(es, corev1.EventTypeNormal, "Recycle", "Deleted Exporter %s after terminal lifecycle", exp.Name) } deleted = true } return deleted, requeueAfter, nil } // isExporterOffline reports whether a deployed exporter is done and can be // recycled. Normally that means it was seen and has gone offline. An exporter // that has never registered is still starting and is not terminal — unless it // has outlived exporterStartupDeadline since deployment completed, in which // case the deployment is wedged and the slot would otherwise leak. Note that // the Exporter reconciler sets Online=False with "Never seen" as soon as the CR // exists (LastSeen still zero); that alone must not count as terminal or // ExitAndReplace would delete off-cluster exporters before they can register. // // deployedSince is when the provisioner finished deploying the instance, or // zero if it does not record it; see Deployer.DeployedAt. func isExporterOffline(exp *jumpstarterdevv1alpha1.Exporter, deployedSince time.Time) bool { // Never seen: the exporter has not registered yet, so it is still starting // until the startup deadline expires. if exp.Status.LastSeen.IsZero() { return isExporterPastStartupDeadline(exp, deployedSince) } cond := meta.FindStatusCondition( exp.Status.Conditions, string(jumpstarterdevv1alpha1.ExporterConditionTypeOnline), ) if cond == nil || cond.Status != metav1.ConditionFalse { return false } // An exporter that reports Offline does so as it shuts down // ("Exporter shutting down" right before Unregister). ReportStatus writes // LastSeen, ExporterStatusValue and the Online condition in the same status // patch, so the heartbeat is fresh even though the process is already gone. // Recycling it is unambiguous: do not wait for the heartbeat to go stale, or // a capped pool keeps a dead exporter's slot for up to a minute while leases // wait for its replacement. // // The Status stream clears a stale Offline value to "" when it (re)connects, // in the same patch as the LastSeen update, so a reconnecting exporter never // shows fresh LastSeen together with Offline. if exp.Status.ExporterStatusValue == jumpstarterdevv1alpha1.ExporterStatusOffline { return true } // Otherwise Online=False is only meaningful once the heartbeat is stale: a // recently (re)connected exporter can still carry the old "Never seen"/False // condition until the Exporter controller catches up, and must not be // recycled. return time.Since(exp.Status.LastSeen.Time) > exporterOfflineThreshold } // isExporterPastStartupDeadline reports whether an exporter that has never // registered has outlived exporterStartupDeadline. The deadline runs from // deployment completion when the provisioner records it, so a deploy that takes // longer than the deadline to succeed is not recycled the instant it finishes. // It falls back to the creation timestamp for exporters deployed before that // time was recorded. A zero timestamp (object not observed by the API server // yet) never expires. func isExporterPastStartupDeadline(exp *jumpstarterdevv1alpha1.Exporter, deployedSince time.Time) bool { start := deployedSince if start.IsZero() { start = exp.CreationTimestamp.Time } if start.IsZero() { return false } return time.Since(start) > exporterStartupDeadline } // timeToStartupDeadline returns how long until an exporter's startup deadline // expires. A non-positive value means it already expired (or could not be // computed). The reconciler uses this to schedule a RequeueAfter so it wakes // up when the deadline fires, even if no other event triggers reconciliation. func timeToStartupDeadline(exp *jumpstarterdevv1alpha1.Exporter, deployedSince time.Time) time.Duration { start := deployedSince if start.IsZero() { start = exp.CreationTimestamp.Time } if start.IsZero() { return 0 } return time.Until(start.Add(exporterStartupDeadline)) } func allPodsTerminal(pods []corev1.Pod) bool { if len(pods) == 0 { return false } for i := range pods { if !isTerminalPodPhase(pods[i].Status.Phase) { return false } } return true } func isTerminalPodPhase(phase corev1.PodPhase) bool { return phase == corev1.PodSucceeded || phase == corev1.PodFailed } // reconcileScaleUp evaluates the three scale-up rules in priority order and // creates one Exporter if any rule fires. Returns true if a scale-up was // attempted (caller should skip scale-down for this cycle). func (r *ExporterSetReconciler) reconcileScaleUp( ctx context.Context, es *virtualtargetv1alpha1.ExporterSet, state poolState, ) (scaled bool, err error) { logger := log.FromContext(ctx) potentiallyAvailable := state.replicas - state.leased // 1. Floor: never drop below minReplicas. if state.replicas < es.Spec.MinReplicas && potentiallyAvailable < es.Spec.MinReplicas { if maxScaleUp(es, state.replicas) > 0 { return true, r.scaleUp(ctx, es, 1) } return false, nil } // 2. Warm buffer: keep minAvailableReplicas ready-and-unleased instances. if state.available < es.Spec.MinAvailableReplicas && potentiallyAvailable < es.Spec.MinAvailableReplicas && !atMaxReplicas(es, state.replicas) { if maxScaleUp(es, state.replicas) > 0 { return true, r.scaleUp(ctx, es, 1) } return false, nil } // 3. Demand: more pending Leases than instances that could serve them. // Instances still booting count as capacity here, otherwise one waiting // lease would add another Exporter on every reconcile until maxReplicas. if !atMaxReplicas(es, state.replicas) { pending, err := r.countPendingLeases(ctx, es) if err != nil { logger.Error(err, "failed to count pending leases") return false, nil } if pending > potentiallyAvailable && maxScaleUp(es, state.replicas) > 0 { return true, r.scaleUp(ctx, es, 1) } } return false, nil } // reconcileScaleDown disables one idle exporter per cooldown period. func (r *ExporterSetReconciler) reconcileScaleDown( ctx context.Context, es *virtualtargetv1alpha1.ExporterSet, ownedExporters []jumpstarterdevv1alpha1.Exporter, state poolState, ) (ctrl.Result, error) { logger := log.FromContext(ctx) // Scale down when available > minAvailableReplicas AND replicas > minReplicas. surplus := state.available - es.Spec.MinAvailableReplicas excessOverMin := state.replicas - es.Spec.MinReplicas if excessOverMin <= 0 || surplus <= 0 { r.clearSurplusAnnotation(ctx, es) return ctrl.Result{}, nil } cooldown := defaultScaleDownCooldown if es.Spec.ScaleDownCooldown != nil { cooldown = es.Spec.ScaleDownCooldown.Duration } // In-memory guard against stale-cache double-disables. key := types.NamespacedName{Name: es.Name, Namespace: es.Namespace} r.lastScaleDownMu.Lock() if r.LastScaleDownAction == nil { r.LastScaleDownAction = make(map[types.NamespacedName]time.Time) } lastAction := r.LastScaleDownAction[key] r.lastScaleDownMu.Unlock() if !lastAction.IsZero() { if remaining := cooldown - time.Since(lastAction); remaining > 0 { return ctrl.Result{RequeueAfter: remaining}, nil } } surplusSince, hasAnnotation := es.Annotations[annotationSurplusSince] if !hasAnnotation { r.setSurplusAnnotation(ctx, es) return ctrl.Result{RequeueAfter: cooldown}, nil } surplusTime, err := time.Parse(time.RFC3339, surplusSince) if err != nil { r.setSurplusAnnotation(ctx, es) return ctrl.Result{RequeueAfter: cooldown}, nil } elapsed := time.Since(surplusTime) if elapsed < cooldown { return ctrl.Result{RequeueAfter: cooldown - elapsed}, nil } // Cooldown elapsed — pick one exporter to disable (prefer online first). for _, wantOnline := range []bool{true, false} { for i := range ownedExporters { exp := &ownedExporters[i] if !exp.IsEnabled() || exp.Status.LeaseRef != nil { continue } isOnline := meta.IsStatusConditionTrue( exp.Status.Conditions, string(jumpstarterdevv1alpha1.ExporterConditionTypeOnline), ) if isOnline != wantOnline { continue } exp.Spec.Enabled = new(false) if err := r.Update(ctx, exp); err != nil { return ctrl.Result{}, fmt.Errorf("unable to disable Exporter %s: %w", exp.Name, err) } logger.Info("disabled Exporter for scale-down", "exporter", exp.Name) if r.Recorder != nil { r.Recorder.Eventf(es, corev1.EventTypeNormal, "ScaleDown", "Disabled Exporter %s for removal", exp.Name) } r.lastScaleDownMu.Lock() if r.LastScaleDownAction == nil { r.LastScaleDownAction = make(map[types.NamespacedName]time.Time) } r.LastScaleDownAction[key] = time.Now() r.lastScaleDownMu.Unlock() r.setSurplusAnnotation(ctx, es) return ctrl.Result{RequeueAfter: cooldown}, nil } } return ctrl.Result{}, nil } func (r *ExporterSetReconciler) setSurplusAnnotation(ctx context.Context, es *virtualtargetv1alpha1.ExporterSet) { if es.Annotations == nil { es.Annotations = make(map[string]string) } es.Annotations[annotationSurplusSince] = time.Now().UTC().Format(time.RFC3339) if err := r.Update(ctx, es); err != nil && !apierrors.IsConflict(err) { log.FromContext(ctx).Error(err, "failed to set surplus-since annotation") } } func (r *ExporterSetReconciler) clearSurplusAnnotation(ctx context.Context, es *virtualtargetv1alpha1.ExporterSet) { if _, ok := es.Annotations[annotationSurplusSince]; !ok { return } delete(es.Annotations, annotationSurplusSince) if err := r.Update(ctx, es); err != nil && !apierrors.IsConflict(err) { log.FromContext(ctx).Error(err, "failed to clear surplus-since annotation") } } // identityLabels mark which pool an exporter belongs to and how it is // provisioned. Reconcile has already established that the referenced class // names this reconciler's provisioner, so it is the provisioner in effect. // // It returns the labels to set and the identity keys to remove. A value that // is not a valid label value (object names may be up to 253 characters, label // values only 63) can't be set: patching it would fail with a 422 on every // reconcile. Its key is removed instead, so an exporter never keeps a stale // value (say, the class it had before the set moved to a long-named class). func (r *ExporterSetReconciler) identityLabels( ctx context.Context, es *virtualtargetv1alpha1.ExporterSet, ) (set map[string]string, remove []string) { candidates := map[string]string{ labelExporterSetName: es.Name, labelVirtualTargetClass: es.Spec.VirtualTargetClassName, } if r.Provisioner != nil { candidates[labelProvisioner] = r.Provisioner.Name() } set = make(map[string]string, len(candidates)) for key, value := range candidates { if value == "" { remove = append(remove, key) continue } if errs := validation.IsValidLabelValue(value); len(errs) > 0 { log.FromContext(ctx).V(1).Info("omitting exporter set label with invalid value", "label", key, "value", value, "reason", errs) remove = append(remove, key) continue } set[key] = value } return set, remove } // exporterLabels are the labels an Exporter of this set carries: the set's // template labels plus the identity labels above. The // exporterset.jumpstarter.dev/ identity keys are reserved for this controller: // they override template values and are dropped from the template when the // controller omits them, so a template can never misreport pool membership. func (r *ExporterSetReconciler) exporterLabels( ctx context.Context, es *virtualtargetv1alpha1.ExporterSet, ) map[string]string { labels := maps.Clone(es.Spec.Template.Metadata.Labels) if labels == nil { labels = map[string]string{} } set, remove := r.identityLabels(ctx, es) for _, key := range remove { delete(labels, key) } for key, value := range set { if prev, ok := labels[key]; ok && prev != value { log.FromContext(ctx).V(1).Info("template label overridden by exporter set identity label", "label", key, "template", prev, "identity", value) } labels[key] = value } return labels } // selectorMismatch reports, as a human-readable message, why the labels this // set gives its exporters don't satisfy its own selector. It returns "" when // they match. func (r *ExporterSetReconciler) selectorMismatch( ctx context.Context, es *virtualtargetv1alpha1.ExporterSet, ) (string, error) { selector, err := metav1.LabelSelectorAsSelector(&es.Spec.Selector) if err != nil { return "", fmt.Errorf("invalid label selector: %w", err) } exporterLabels := labels.Set(r.exporterLabels(ctx, es)) if selector.Matches(exporterLabels) { return "", nil } return fmt.Sprintf("selector %q does not match exporter labels %q; "+ "exporterset.jumpstarter.dev/ label values are set by the controller", selector.String(), exporterLabels.String()), nil } // reportUnavailable records a spec problem that stops reconciliation: it // refreshes status counts, marks the set not Available with the given reason, // and returns without requeuing (a spec change retriggers reconcile). func (r *ExporterSetReconciler) reportUnavailable( ctx context.Context, es *virtualtargetv1alpha1.ExporterSet, reason, message string, ) (ctrl.Result, error) { logger := log.FromContext(ctx) prevAvailable := meta.IsStatusConditionTrue(es.Status.Conditions, string(virtualtargetv1alpha1.ExporterSetConditionAvailable)) prevDegraded := meta.IsStatusConditionTrue(es.Status.Conditions, string(virtualtargetv1alpha1.ExporterSetConditionDegraded)) prevProgressing := meta.IsStatusConditionTrue(es.Status.Conditions, string(virtualtargetv1alpha1.ExporterSetConditionProgressing)) prevScalingLimited := meta.IsStatusConditionTrue(es.Status.Conditions, string(virtualtargetv1alpha1.ExporterSetConditionScalingLimited)) if err := r.reconcileStatusCounts(ctx, es); err != nil { return ctrl.Result{}, err } r.reconcileConditions(es) meta.SetStatusCondition(&es.Status.Conditions, metav1.Condition{ Type: string(virtualtargetv1alpha1.ExporterSetConditionAvailable), Status: metav1.ConditionFalse, ObservedGeneration: es.Generation, Reason: reason, Message: message, }) if err := r.Status().Update(ctx, es); err != nil { return requeueConflict(logger, err) } r.emitConditionEvents(es, prevAvailable, prevProgressing, prevDegraded, prevScalingLimited) return ctrl.Result{}, nil } // reconcileExporterLabels keeps the identity labels on owned exporters current: // it stamps them on exporters that predate them, updates them after the set's // class changes, and removes identity keys the controller now omits. func (r *ExporterSetReconciler) reconcileExporterLabels( ctx context.Context, es *virtualtargetv1alpha1.ExporterSet, owned []jumpstarterdevv1alpha1.Exporter, ) error { logger := log.FromContext(ctx) set, remove := r.identityLabels(ctx, es) // Label every exporter we can this cycle: one failed patch should not // leave the rest of the pool unlabeled until the next reconcile. var errs []error for i := range owned { exporter := &owned[i] changed := map[string]string{} for key, value := range set { if exporter.Labels[key] != value { changed[key] = value } } var removed []string for _, key := range remove { if _, ok := exporter.Labels[key]; ok { removed = append(removed, key) } } if len(changed) == 0 && len(removed) == 0 { continue } patch := client.MergeFrom(exporter.DeepCopy()) if exporter.Labels == nil { exporter.Labels = map[string]string{} } maps.Copy(exporter.Labels, changed) for _, key := range removed { delete(exporter.Labels, key) } if err := r.Patch(ctx, exporter, patch); err != nil { logger.Error(err, "unable to label exporter", "exporter", exporter.Name) errs = append(errs, fmt.Errorf("unable to label Exporter %s: %w", exporter.Name, err)) continue } logger.Info("updated exporter set labels", "exporter", exporter.Name, "set", changed, "removed", removed) } return errors.Join(errs...) } func (r *ExporterSetReconciler) listOwnedExporters( ctx context.Context, es *virtualtargetv1alpha1.ExporterSet, ) ([]jumpstarterdevv1alpha1.Exporter, error) { selector, err := metav1.LabelSelectorAsSelector(&es.Spec.Selector) if err != nil { return nil, fmt.Errorf("invalid label selector: %w", err) } var exporterList jumpstarterdevv1alpha1.ExporterList if err := r.List(ctx, &exporterList, client.InNamespace(es.Namespace), client.MatchingLabelsSelector{Selector: selector}, ); err != nil { return nil, fmt.Errorf("unable to list Exporters: %w", err) } owned := make([]jumpstarterdevv1alpha1.Exporter, 0, len(exporterList.Items)) for i := range exporterList.Items { if isOwnedBy(&exporterList.Items[i], es.UID) { owned = append(owned, exporterList.Items[i]) } } return owned, nil } func (r *ExporterSetReconciler) reconcileStatusCounts( ctx context.Context, es *virtualtargetv1alpha1.ExporterSet, ) error { selector, err := metav1.LabelSelectorAsSelector(&es.Spec.Selector) if err != nil { return fmt.Errorf("invalid label selector: %w", err) } es.Status.Selector = selector.String() var exporterList jumpstarterdevv1alpha1.ExporterList if err := r.List(ctx, &exporterList, client.InNamespace(es.Namespace), client.MatchingLabelsSelector{Selector: selector}, ); err != nil { return fmt.Errorf("unable to list Exporters: %w", err) } ownedExporters := filterOwnedExporters(exporterList.Items, es.UID) var replicas, ready, available, leased int32 var active, idle, disabled, offline int32 for i := range ownedExporters { exp := &ownedExporters[i] replicas++ isOnline := meta.IsStatusConditionTrue( exp.Status.Conditions, string(jumpstarterdevv1alpha1.ExporterConditionTypeOnline), ) isEnabled := exp.IsEnabled() isLeased := exp.Status.LeaseRef != nil if isOnline { ready++ } if isLeased { leased++ } if isOnline && isEnabled && !isLeased { available++ } if !isEnabled { disabled++ } else if isLeased && isOnline { active++ } else if isOnline { idle++ } else { offline++ } } es.Status.Replicas = replicas es.Status.ReadyReplicas = ready es.Status.AvailableReplicas = available es.Status.UnavailableReplicas = offline es.Status.LeasedReplicas = leased es.Status.ExportersActive = active es.Status.ExportersIdle = idle es.Status.ExportersDisabled = disabled es.Status.ExportersOffline = offline var podList corev1.PodList if err := r.List(ctx, &podList, client.InNamespace(es.Namespace), client.MatchingLabelsSelector{Selector: selector}, ); err != nil { return fmt.Errorf("unable to list Pods: %w", err) } var pending, running, failed, unknown int32 for i := range podList.Items { // Pods are owned by Exporters (not ExporterSets directly); skip any // that lack a controller owner of Kind=Exporter to avoid counting // stray pods that happen to share the selector labels. if !isOwnedByKind(&podList.Items[i], kindExporter) { continue } switch podList.Items[i].Status.Phase { case corev1.PodPending: pending++ case corev1.PodRunning: running++ case corev1.PodFailed: failed++ default: unknown++ } } es.Status.PodsPending = pending es.Status.PodsRunning = running es.Status.PodsFailed = failed es.Status.PodsUnknown = unknown return nil } func (r *ExporterSetReconciler) reconcileConditions(es *virtualtargetv1alpha1.ExporterSet) { r.reconcileConditionAvailable(es) r.reconcileConditionProgressing(es) r.reconcileConditionDegraded(es) r.reconcileConditionScalingLimited(es) } func (r *ExporterSetReconciler) reconcileConditionAvailable(es *virtualtargetv1alpha1.ExporterSet) { condType := string(virtualtargetv1alpha1.ExporterSetConditionAvailable) if es.Spec.MinReplicas == 0 && es.Status.Replicas == 0 { meta.SetStatusCondition(&es.Status.Conditions, metav1.Condition{ Type: condType, Status: metav1.ConditionTrue, ObservedGeneration: es.Generation, Reason: "NoReplicasRequired", Message: "No minimum replicas configured", }) return } if es.Status.ReadyReplicas >= es.Spec.MinReplicas { meta.SetStatusCondition(&es.Status.Conditions, metav1.Condition{ Type: condType, Status: metav1.ConditionTrue, ObservedGeneration: es.Generation, Reason: "MinimumReplicasAvailable", Message: fmt.Sprintf("%d/%d replicas ready", es.Status.ReadyReplicas, es.Spec.MinReplicas), }) return } meta.SetStatusCondition(&es.Status.Conditions, metav1.Condition{ Type: condType, Status: metav1.ConditionFalse, ObservedGeneration: es.Generation, Reason: "MinimumReplicasUnavailable", Message: fmt.Sprintf("%d/%d replicas ready, need %d", es.Status.ReadyReplicas, es.Status.Replicas, es.Spec.MinReplicas), }) } func (r *ExporterSetReconciler) reconcileConditionProgressing(es *virtualtargetv1alpha1.ExporterSet) { condType := string(virtualtargetv1alpha1.ExporterSetConditionProgressing) if es.Status.PodsPending > 0 { meta.SetStatusCondition(&es.Status.Conditions, metav1.Condition{ Type: condType, Status: metav1.ConditionTrue, ObservedGeneration: es.Generation, Reason: "PodsStarting", Message: fmt.Sprintf("%d pod(s) pending", es.Status.PodsPending), }) return } if es.Status.UnavailableReplicas > 0 { meta.SetStatusCondition(&es.Status.Conditions, metav1.Condition{ Type: condType, Status: metav1.ConditionTrue, ObservedGeneration: es.Generation, Reason: "ReplicasNotReady", Message: fmt.Sprintf("%d replica(s) not yet ready", es.Status.UnavailableReplicas), }) return } meta.SetStatusCondition(&es.Status.Conditions, metav1.Condition{ Type: condType, Status: metav1.ConditionFalse, ObservedGeneration: es.Generation, Reason: "AllReplicasReady", Message: "All replicas are ready", }) } func (r *ExporterSetReconciler) reconcileConditionDegraded(es *virtualtargetv1alpha1.ExporterSet) { condType := string(virtualtargetv1alpha1.ExporterSetConditionDegraded) if es.Status.PodsFailed > 0 { meta.SetStatusCondition(&es.Status.Conditions, metav1.Condition{ Type: condType, Status: metav1.ConditionTrue, ObservedGeneration: es.Generation, Reason: "PodsFailing", Message: fmt.Sprintf("%d pod(s) in Failed state", es.Status.PodsFailed), }) return } if es.Status.UnavailableReplicas > 0 && es.Status.PodsPending == 0 { meta.SetStatusCondition(&es.Status.Conditions, metav1.Condition{ Type: condType, Status: metav1.ConditionTrue, ObservedGeneration: es.Generation, Reason: "ExportersOffline", Message: fmt.Sprintf("%d exporter(s) not ready with no pending pods", es.Status.UnavailableReplicas), }) return } meta.SetStatusCondition(&es.Status.Conditions, metav1.Condition{ Type: condType, Status: metav1.ConditionFalse, ObservedGeneration: es.Generation, Reason: "AllReplicasHealthy", Message: "No degraded replicas", }) } func (r *ExporterSetReconciler) reconcileConditionScalingLimited(es *virtualtargetv1alpha1.ExporterSet) { condType := string(virtualtargetv1alpha1.ExporterSetConditionScalingLimited) atMax := es.Spec.MaxReplicas > 0 && es.Status.Replicas >= es.Spec.MaxReplicas needMore := es.Status.AvailableReplicas < es.Spec.MinAvailableReplicas if atMax && needMore { meta.SetStatusCondition(&es.Status.Conditions, metav1.Condition{ Type: condType, Status: metav1.ConditionTrue, ObservedGeneration: es.Generation, Reason: "AtMaxReplicas", Message: fmt.Sprintf("Cannot scale beyond %d replicas; %d available, need %d", es.Spec.MaxReplicas, es.Status.AvailableReplicas, es.Spec.MinAvailableReplicas), }) return } meta.SetStatusCondition(&es.Status.Conditions, metav1.Condition{ Type: condType, Status: metav1.ConditionFalse, ObservedGeneration: es.Generation, Reason: "WithinLimits", Message: "Scaling is not constrained", }) } func (r *ExporterSetReconciler) emitConditionEvents( es *virtualtargetv1alpha1.ExporterSet, prevAvailable, prevProgressing, prevDegraded, prevScalingLimited bool, ) { if r.Recorder == nil { return } nowAvailable := meta.IsStatusConditionTrue( es.Status.Conditions, string(virtualtargetv1alpha1.ExporterSetConditionAvailable), ) nowProgressing := meta.IsStatusConditionTrue( es.Status.Conditions, string(virtualtargetv1alpha1.ExporterSetConditionProgressing), ) nowDegraded := meta.IsStatusConditionTrue( es.Status.Conditions, string(virtualtargetv1alpha1.ExporterSetConditionDegraded), ) nowScalingLimited := meta.IsStatusConditionTrue( es.Status.Conditions, string(virtualtargetv1alpha1.ExporterSetConditionScalingLimited), ) if !prevAvailable && nowAvailable { r.Recorder.Eventf(es, corev1.EventTypeNormal, "ExporterSetAvailable", "ExporterSet %s is available: %d ready replicas", es.Name, es.Status.ReadyReplicas) } else if prevAvailable && !nowAvailable { r.Recorder.Eventf(es, corev1.EventTypeWarning, "ExporterSetUnavailable", "ExporterSet %s is unavailable: %d/%d replicas ready", es.Name, es.Status.ReadyReplicas, es.Spec.MinReplicas) } if !prevProgressing && nowProgressing { r.Recorder.Eventf(es, corev1.EventTypeNormal, "ExporterSetProgressing", "ExporterSet %s is progressing: %d pending pods, %d unavailable replicas", es.Name, es.Status.PodsPending, es.Status.UnavailableReplicas) } else if prevProgressing && !nowProgressing { r.Recorder.Eventf(es, corev1.EventTypeNormal, "ExporterSetSettled", "ExporterSet %s finished progressing: all replicas ready", es.Name) } if !prevDegraded && nowDegraded { r.Recorder.Eventf(es, corev1.EventTypeWarning, "ExporterSetDegraded", "ExporterSet %s is degraded: %d failed pods, %d unavailable replicas", es.Name, es.Status.PodsFailed, es.Status.UnavailableReplicas) } else if prevDegraded && !nowDegraded { r.Recorder.Eventf(es, corev1.EventTypeNormal, "ExporterSetRecovered", "ExporterSet %s recovered: all replicas healthy", es.Name) } if !prevScalingLimited && nowScalingLimited { r.Recorder.Eventf(es, corev1.EventTypeWarning, "ScalingLimited", "ExporterSet %s scaling limited at %d replicas", es.Name, es.Spec.MaxReplicas) } else if prevScalingLimited && !nowScalingLimited { r.Recorder.Eventf(es, corev1.EventTypeNormal, "ScalingUnlimited", "ExporterSet %s scaling no longer limited", es.Name) } } func filterOwnedExporters( exporters []jumpstarterdevv1alpha1.Exporter, ownerUID types.UID, ) []jumpstarterdevv1alpha1.Exporter { owned := make([]jumpstarterdevv1alpha1.Exporter, 0, len(exporters)) for i := range exporters { if isOwnedBy(&exporters[i], ownerUID) { owned = append(owned, exporters[i]) } } return owned } // SetupWithManager sets up the controller with the Manager. func (r *ExporterSetReconciler) SetupWithManager(mgr ctrl.Manager) error { builder := ctrl.NewControllerManagedBy(mgr). For(&virtualtargetv1alpha1.ExporterSet{}). Owns(&jumpstarterdevv1alpha1.Exporter{}). Watches( &corev1.Pod{}, handler.EnqueueRequestsFromMapFunc(r.findExporterSetForPod), ). Watches( &virtualtargetv1alpha1.VirtualTargetClass{}, handler.EnqueueRequestsFromMapFunc(r.findExporterSetsForVTC), ). Watches( &jumpstarterdevv1alpha1.Lease{}, handler.EnqueueRequestsFromMapFunc(r.findExporterSetsForLease), ). Named("exporterset") if _, ok := r.Provisioner.(NetworkPolicyProvisioner); ok { builder = builder.Owns(&networkingv1.NetworkPolicy{}) } return builder.Complete(r) } // findExporterSetForPod bridges the ExporterSet→Exporter→Pod grandchild gap via label. func (r *ExporterSetReconciler) findExporterSetForPod( ctx context.Context, obj client.Object, ) []reconcile.Request { pod, ok := obj.(*corev1.Pod) if !ok { return nil } esName, ok := pod.Labels[labelExporterSetName] if !ok { return nil } return []reconcile.Request{{ NamespacedName: types.NamespacedName{ Name: esName, Namespace: pod.Namespace, }, }} } func (r *ExporterSetReconciler) findExporterSetsForVTC( ctx context.Context, obj client.Object, ) []reconcile.Request { vtc, ok := obj.(*virtualtargetv1alpha1.VirtualTargetClass) if !ok { return nil } var exporterSetList virtualtargetv1alpha1.ExporterSetList if err := r.List(ctx, &exporterSetList, client.InNamespace(vtc.Namespace)); err != nil { log.FromContext(ctx).Error(err, "unable to list ExporterSets for VirtualTargetClass", "namespace", vtc.Namespace, "name", vtc.Name) return nil } requests := make([]reconcile.Request, 0, len(exporterSetList.Items)) for _, exporterSet := range exporterSetList.Items { if exporterSet.Spec.VirtualTargetClassName != vtc.Name { continue } requests = append(requests, reconcile.Request{ NamespacedName: types.NamespacedName{ Name: exporterSet.Name, Namespace: exporterSet.Namespace, }, }) } return requests } // findExporterSetsForLease enqueues ExporterSets whose template labels match the Lease's selector. func (r *ExporterSetReconciler) findExporterSetsForLease( ctx context.Context, obj client.Object, ) []reconcile.Request { lease, ok := obj.(*jumpstarterdevv1alpha1.Lease) if !ok { return nil } if lease.Status.Ended || !leaseWantsPoolMember(lease) { return nil } leaseSelector, err := metav1.LabelSelectorAsSelector(&lease.Spec.Selector) if err != nil { return nil } var exporterSetList virtualtargetv1alpha1.ExporterSetList if err := r.List(ctx, &exporterSetList, client.InNamespace(lease.Namespace)); err != nil { log.FromContext(ctx).Error(err, "unable to list ExporterSets for Lease", "namespace", lease.Namespace, "lease", lease.Name) return nil } var requests []reconcile.Request for _, es := range exporterSetList.Items { templateLabels := es.Spec.Template.Metadata.Labels if leaseSelector.Matches(labels.Set(templateLabels)) { requests = append(requests, reconcile.Request{ NamespacedName: types.NamespacedName{ Name: es.Name, Namespace: es.Namespace, }, }) } } return requests } // countPendingLeases counts unassigned pending Leases that match this pool's template labels. func (r *ExporterSetReconciler) countPendingLeases( ctx context.Context, es *virtualtargetv1alpha1.ExporterSet, ) (int32, error) { var leaseList jumpstarterdevv1alpha1.LeaseList if err := r.List(ctx, &leaseList, client.InNamespace(es.Namespace)); err != nil { return 0, fmt.Errorf("unable to list Leases: %w", err) } templateLabels := labels.Set(es.Spec.Template.Metadata.Labels) var count int32 for i := range leaseList.Items { lease := &leaseList.Items[i] if lease.Status.Ended || lease.Status.ExporterRef != nil || !leaseWantsPoolMember(lease) { continue } if !meta.IsStatusConditionTrue(lease.Status.Conditions, string(jumpstarterdevv1alpha1.LeaseConditionTypePending)) { continue } sel, err := metav1.LabelSelectorAsSelector(&lease.Spec.Selector) if err != nil { continue } if sel.Matches(templateLabels) { count++ } } return count, nil } // leaseWantsPoolMember reports whether a Lease can be served by a pool instance // at all. A lease pinned to a named exporter, or one without a selector (an // empty selector converts to labels.Everything and would match every pool), // is no demand for any ExporterSet. func leaseWantsPoolMember(lease *jumpstarterdevv1alpha1.Lease) bool { if lease.Spec.ExporterRef != nil { return false } return len(lease.Spec.Selector.MatchLabels) > 0 || len(lease.Spec.Selector.MatchExpressions) > 0 } func isOwnedBy(obj client.Object, ownerUID types.UID) bool { for _, ref := range obj.GetOwnerReferences() { if ref.UID == ownerUID && ref.Controller != nil && *ref.Controller { return true } } return false } // isOwnedByKind reports whether the object's controller owner has the given Kind. func isOwnedByKind(obj client.Object, kind string) bool { for _, ref := range obj.GetOwnerReferences() { if ref.Kind == kind && ref.Controller != nil && *ref.Controller { return true } } return false } func atMaxReplicas(es *virtualtargetv1alpha1.ExporterSet, current int32) bool { return es.Spec.MaxReplicas > 0 && current >= es.Spec.MaxReplicas } func maxScaleUp(es *virtualtargetv1alpha1.ExporterSet, current int32) int32 { if es.Spec.MaxReplicas <= 0 { return int32(1<<31 - 1) } room := es.Spec.MaxReplicas - current if room < 0 { return 0 } return room } func requeueConflict(logger interface{ Info(string, ...any) }, err error) (ctrl.Result, error) { if apierrors.IsConflict(err) { logger.Info("conflict on status update, will retry") } return ctrl.Result{}, err } // deepMergeParameters merges VTC + ExporterSet params (maps recursive, scalars replace). func deepMergeParameters( classParams *apiextensionsv1.JSON, setParams *apiextensionsv1.JSON, ) (map[string]any, error) { base := make(map[string]any) override := make(map[string]any) if classParams != nil && classParams.Raw != nil { if err := json.Unmarshal(classParams.Raw, &base); err != nil { return nil, fmt.Errorf("unable to unmarshal class parameters: %w", err) } } if setParams != nil && setParams.Raw != nil { if err := json.Unmarshal(setParams.Raw, &override); err != nil { return nil, fmt.Errorf("unable to unmarshal set parameters: %w", err) } } return DeepMerge(base, override), nil } // DeepMerge recursively overlays override on base: nested maps are merged, // scalars (and non-map values) replace. Exported so the qemu-ssh provisioner // can use the same logic in Cleanup (merging class + set parameters). func DeepMerge(base, override map[string]any) map[string]any { result := make(map[string]any, len(base)+len(override)) maps.Copy(result, base) for k, v := range override { if baseMap, ok := result[k].(map[string]any); ok { if overrideMap, ok := v.(map[string]any); ok { result[k] = DeepMerge(baseMap, overrideMap) continue } } result[k] = v } return result } // mergeImages merges VTC-level and ExporterSet-level image overrides. // ExporterSet values take precedence over VTC values at the ImageSpec level. func mergeImages(vtcImages, esImages *virtualtargetv1alpha1.ImageOverrides) *virtualtargetv1alpha1.ImageOverrides { if vtcImages == nil && esImages == nil { return nil } if vtcImages == nil { return esImages.DeepCopy() } if esImages == nil { return vtcImages.DeepCopy() } merged := vtcImages.DeepCopy() if esImages.Exporter != nil { merged.Exporter = esImages.Exporter.DeepCopy() } if esImages.Runtime != nil { merged.Runtime = esImages.Runtime.DeepCopy() } if esImages.Turn != nil { merged.Turn = esImages.Turn.DeepCopy() } return merged }