/* Copyright 2025. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. */ package v1alpha1 import ( metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/util/intstr" ) // AnnotationsClearedOnRecycle lists all annotation keys that are removed from a // sandbox when it is successfully recycled and returned to the pool. When adding // a new annotation that should be cleared during recycle, append it here to avoid // missing the recycle in resetSandboxForPool. // // Note: AnnotationUpdatedMetadataInClaim is handled separately because it is // consumed before deletion to determine user-specified metadata keys. // Annotations from other packages (e.g. identity.AgentKeyTokenRefreshStatus) // are handled individually in resetSandboxForPool. var AnnotationsClearedOnRecycle = []string{ AnnotationClaimTime, AnnotationLock, AnnotationOwner, AnnotationInitRuntimeRequest, AnnotationRuntimeAccessToken, AnnotationCleanup, AnnotationCleanupRetainOnFailure, AnnotationCSIVolumeConfig, SandboxAnnotationPriority, AnnotationEnvdAccessToken, AnnotationEnvdURL, AnnotationRuntimeURL, // AnnotationSecurityRules carries the previous tenant's L7 egress rules // (header injections, blocks); leaking it across a claim would apply one // tenant's rules — potentially carrying credentials — to another. AnnotationSecurityRules, } // InternalKeysPreservedOnCreation lists internal keys (with the InternalPrefix) // that are preserved when creating a new sandbox from a SandboxSet template. // All other internal keys are cleared to ensure a clean slate. When adding a // new internal key that should survive sandbox creation, add it here to avoid // accidental deletion in clearAndInitInnerKeys. var InternalKeysPreservedOnCreation = map[string]struct{}{ AnnotationCleanupEnabled: {}, AnnotationCleanupRetainOnFailure: {}, } // UpdatedMetadataInClaim records the keys of labels/annotations added or modified during claim. // Used by the recycle flow to determine which metadata to reset. type UpdatedMetadataInClaim struct { Labels []string `json:"labels,omitempty"` Annotations []string `json:"annotations,omitempty"` } const ( SandboxStateCreating = "creating" SandboxStateAvailable = "available" SandboxStateRunning = "running" SandboxStatePaused = "paused" SandboxStateDead = "dead" ) // Reason values reported alongside SandboxState* by utils.GetSandboxState. // Exported so downstream controllers can match on them without repeating the // hard-coded string literal. const ( SandboxStateReasonResourcePending = "ResourcePending" ) var SandboxSetControllerKind = GroupVersion.WithKind("SandboxSet") // SandboxSetSpec defines the desired state of SandboxSet type SandboxSetSpec struct { // Replicas is the number of unused sandboxes, including available and creating ones. Replicas int32 `json:"replicas"` // PauseStrategy configures how sandboxes created from this SandboxSet are // paused when their spec.paused is true. It is copied verbatim onto each // created Sandbox and is part of the update revision hash, so changing it // triggers a rolling update of already-created pool sandboxes. // +optional PauseStrategy *PauseStrategy `json:"pauseStrategy,omitempty"` // PersistentContents indicates resume pod with persistent content, Enum: ip, memory, filesystem // +listType=atomic PersistentContents []string `json:"persistentContents,omitempty"` // Runtimes - Runtime configuration for sandbox object // +optional // +listType=atomic Runtimes []RuntimeConfig `json:"runtimes,omitempty"` // Probes defines the named probes that sandboxes created from this SandboxSet // run while they are Running. It is copied verbatim onto each created Sandbox // and is part of the update revision hash, so changing it triggers a rolling // update of already-created pool sandboxes. // +optional // +listType=map // +listMapKey=name // +kubebuilder:validation:MaxItems=16 Probes []Probe `json:"probes,omitempty"` // AutoPausePolicy defines the pause/resume decision rules for sandboxes // created from this SandboxSet. Probe-driven rules reference probes by // name (so they are normally set together with spec.probes); the // OnIngressTraffic resume rule is event-driven and does not require a // probe. The policy is copied verbatim onto each created Sandbox and is // part of the update revision hash, so changing it triggers a rolling // update of already-created pool sandboxes. // +optional AutoPausePolicy *AutoPausePolicy `json:"autoPausePolicy,omitempty"` // Lifecycle defines pool-level pause/resume hooks copied to every Sandbox. // Upgrade hooks remain instance policy and are not allowed here. // +kubebuilder:validation:XValidation:rule="!has(self.preUpgrade) && !has(self.postUpgrade)",message="preUpgrade and postUpgrade are not allowed on SandboxSet" // +optional Lifecycle *SandboxLifecycle `json:"lifecycle,omitempty"` EmbeddedSandboxTemplate `json:",inline"` // ScaleStrategy indicates the ScaleStrategy that will be employed to // create and delete Sandboxes in the SandboxSet. ScaleStrategy SandboxSetScaleStrategy `json:"scaleStrategy,omitempty"` // UpdateStrategy indicates the strategy that will be employed to // update Sandboxes in the SandboxSet when the template changes. // +optional UpdateStrategy SandboxSetUpdateStrategy `json:"updateStrategy,omitempty"` } // SandboxSetScaleStrategy defines strategies for sandboxes scale. type SandboxSetScaleStrategy struct { // MaxUnavailable caps concurrent physical scale-up operations and serves as // the startup budget for the ScalingLimited condition. It can be an absolute // number (ex: 5) or a percentage of desired pods (ex: 10%); percentages are // rounded up against spec.replicas. If unset or invalid, the controller uses // the base (equivalent to 100%, i.e. no cap). Scale-down is unaffected. // // The physical scale-up budget is charged by startup blockers: sandboxes // whose Ready condition is False with reason PodCreateFailed, // StartContainerFailed, or Unschedulable, sandboxes stuck in // Creating/ResourcePending past the configured --max-pending-timeout, and // sandbox creations that have been issued but are not yet observed by the controller (they release their slot // once observed as healthy Creating sandboxes). Healthy observed Creating // sandboxes do NOT count against the budget. // // The ScalingLimited condition becomes True with reason // StartupBudgetExhausted when failed plus pending-timeout sandboxes exhaust // the resolved startup budget. // MaxUnavailable works only for scale-up. MaxUnavailable *intstr.IntOrString `json:"maxUnavailable,omitempty"` } // RollingUpdateStrategyType is the type of rolling update strategy. type RollingUpdateStrategyType string const ( // RecreateUpdateStrategyType means the controller will delete old pods and create new ones. RecreateUpdateStrategyType RollingUpdateStrategyType = "Recreate" ) // SandboxSetUpdateStrategy defines strategies for rolling update. type SandboxSetUpdateStrategy struct { // MaxUnavailable is the maximum number or percentage of pods that can be unavailable during the update. // MaxUnavailable can be an absolute number (ex: 5) or a percentage of desired pods (ex: 10%). // Absolute number is calculated from percentage by rounding down. // Default is 20%. // +optional MaxUnavailable *intstr.IntOrString `json:"maxUnavailable,omitempty"` } // SandboxSetConditionType is a valid value for SandboxSet conditions. type SandboxSetConditionType string const ( // SandboxSetConditionScalingLimited indicates whether startup blockers exhaust the scale-up budget. SandboxSetConditionScalingLimited SandboxSetConditionType = "ScalingLimited" ) // SandboxSetStatus defines the observed state of SandboxSet. type SandboxSetStatus struct { // observedGeneration is the most recent generation observed for this SandboxSet. It corresponds to the // SandboxSet's generation, which is updated on mutation by the API Server. ObservedGeneration int64 `json:"observedGeneration,omitempty"` // Replicas is the total number of creating, available, running and paused sandboxes. Replicas int32 `json:"replicas"` // AvailableReplicas is the number of available sandboxes, which are ready to be claimed. AvailableReplicas int32 `json:"availableReplicas"` // UpdateRevision is the FNV-32 hash computed from spec.template, // spec.persistentContents, spec.runtimes, spec.pauseStrategy, // spec.probes, and spec.autoPausePolicy. // It represents the latest desired template version. UpdateRevision string `json:"updateRevision,omitempty"` // CurrentRevision is the SandboxTemplate name corresponding to the currently // materialised revision, or spec.templateRef.Name when templateRef is used. // +optional CurrentRevision string `json:"currentRevision,omitempty"` // UpdatedReplicas is the number of sandboxes that have been updated to the UpdateRevision. // +optional UpdatedReplicas int32 `json:"updatedReplicas,omitempty"` // UpdatedAvailableReplicas is the number of updated sandboxes that are available. // +optional UpdatedAvailableReplicas int32 `json:"updatedAvailableReplicas,omitempty"` // conditions represent the current state of the SandboxSet resource. // Each condition has a unique type and reflects the status of a specific aspect of the resource. // The status of each condition is one of True, False, or Unknown. // +listType=map // +listMapKey=type // +optional Conditions []metav1.Condition `json:"conditions,omitempty"` // Selector is a label query over pods that should match the replica count. // This is same as the label selector but in the string format to avoid // duplication for CRDs that do not support structural schemas. // +optional Selector string `json:"selector,omitempty"` } // +genclient // +kubebuilder:object:root=true // +kubebuilder:subresource:status // +kubebuilder:subresource:scale:specpath=.spec.replicas,statuspath=.status.replicas,selectorpath=.status.selector // +kubebuilder:resource:path=sandboxsets,shortName={sbs},singular=sandboxset // +kubebuilder:storageversion // +kubebuilder:printcolumn:name="Replicas",type="integer",JSONPath=".status.replicas" // +kubebuilder:printcolumn:name="Available",type="integer",JSONPath=".status.availableReplicas" // +kubebuilder:printcolumn:name="UpdatedReplicas",type="integer",JSONPath=".status.updatedReplicas" // +kubebuilder:printcolumn:name="UpdatedAvailableReplicas",type="integer",JSONPath=".status.updatedAvailableReplicas" // +kubebuilder:printcolumn:name="UpdateRevision",type="string",JSONPath=".status.updateRevision" // +kubebuilder:printcolumn:name="Age",type="date",JSONPath=".metadata.creationTimestamp" // SandboxSet is the Schema for the sandboxsets API, which is an advanced workload for managing sandboxes. type SandboxSet struct { metav1.TypeMeta `json:",inline"` // metadata is a standard object metadata // +optional metav1.ObjectMeta `json:"metadata,omitempty,omitzero"` // spec defines the desired state of SandboxSet // +required Spec SandboxSetSpec `json:"spec"` // status defines the observed state of SandboxSet // +optional Status SandboxSetStatus `json:"status,omitempty,omitzero"` } // +kubebuilder:object:root=true // SandboxSetList contains a list of SandboxSet type SandboxSetList struct { metav1.TypeMeta `json:",inline"` metav1.ListMeta `json:"metadata,omitempty"` Items []SandboxSet `json:"items"` } func init() { SchemeBuilder.Register(&SandboxSet{}, &SandboxSetList{}) }