# The template creates a dualstack cluster with IPv4 as primary apiVersion: cluster.x-k8s.io/v1beta1 kind: Cluster metadata: labels: ccm: external csi: external name: "${CLUSTER_NAME}" spec: clusterNetwork: pods: cidrBlocks: - 192.168.0.0/16 - fd01::/48 services: cidrBlocks: - 172.30.0.0/16 - fd02::/112 controlPlaneRef: apiVersion: controlplane.cluster.x-k8s.io/v1beta1 kind: KubeadmControlPlane name: "${CLUSTER_NAME}-control-plane" infrastructureRef: apiVersion: infrastructure.cluster.x-k8s.io/v1beta2 kind: AWSCluster name: "${CLUSTER_NAME}" --- apiVersion: infrastructure.cluster.x-k8s.io/v1beta2 kind: AWSCluster metadata: name: "${CLUSTER_NAME}" spec: controlPlaneLoadBalancer: loadBalancerType: nlb healthCheckProtocol: HTTPS targetGroupIPType: ipv4 network: cni: cniIngressRules: # If using Calico as CNI provider, this rule is required. # Note: Calico currently supports IPv6 with VXLAN. - description: "VXLAN (calico)" protocol: udp fromPort: 4789 toPort: 4789 vpc: ipv6: {} region: "${AWS_REGION}" sshKeyName: "${AWS_SSH_KEY_NAME}" --- apiVersion: controlplane.cluster.x-k8s.io/v1beta1 kind: KubeadmControlPlane metadata: name: "${CLUSTER_NAME}-control-plane" spec: kubeadmConfigSpec: clusterConfiguration: controllerManager: extraArgs: cloud-provider: external initConfiguration: nodeRegistration: kubeletExtraArgs: cloud-provider: external name: "{{ ds.meta_data.local_hostname }}" joinConfiguration: nodeRegistration: kubeletExtraArgs: cloud-provider: external name: "{{ ds.meta_data.local_hostname }}" machineTemplate: infrastructureRef: apiVersion: infrastructure.cluster.x-k8s.io/v1beta2 kind: AWSMachineTemplate name: "${CLUSTER_NAME}-control-plane" replicas: ${CONTROL_PLANE_MACHINE_COUNT} version: "${KUBERNETES_VERSION}" --- apiVersion: infrastructure.cluster.x-k8s.io/v1beta2 kind: AWSMachineTemplate metadata: name: "${CLUSTER_NAME}-control-plane" spec: template: spec: iamInstanceProfile: "control-plane.cluster-api-provider-aws.sigs.k8s.io" instanceType: "${AWS_CONTROL_PLANE_MACHINE_TYPE}" sshKeyName: "${AWS_SSH_KEY_NAME}" # Since we create IPv4 target group, there is no need for a primary IPv6 address. assignPrimaryIPv6: disabled # Resource-based naming (RBN) allows AAAA record DNS query. # IP-based hostname only support A records. privateDnsName: enableResourceNameDnsAAAARecord: true enableResourceNameDnsARecord: true hostnameType: resource-name --- apiVersion: cluster.x-k8s.io/v1beta1 kind: MachineDeployment metadata: name: "${CLUSTER_NAME}-md-0" spec: clusterName: "${CLUSTER_NAME}" replicas: ${WORKER_MACHINE_COUNT} selector: matchLabels: null template: spec: bootstrap: configRef: apiVersion: bootstrap.cluster.x-k8s.io/v1beta1 kind: KubeadmConfigTemplate name: "${CLUSTER_NAME}-md-0" clusterName: "${CLUSTER_NAME}" infrastructureRef: apiVersion: infrastructure.cluster.x-k8s.io/v1beta2 kind: AWSMachineTemplate name: "${CLUSTER_NAME}-md-0" version: "${KUBERNETES_VERSION}" --- apiVersion: infrastructure.cluster.x-k8s.io/v1beta2 kind: AWSMachineTemplate metadata: name: "${CLUSTER_NAME}-md-0" spec: template: spec: iamInstanceProfile: nodes.cluster-api-provider-aws.sigs.k8s.io instanceType: "${AWS_NODE_MACHINE_TYPE}" sshKeyName: "${AWS_SSH_KEY_NAME}" # Since we create IPv4 target group, there is no need for a primary IPv6 address. assignPrimaryIPv6: disabled # Resource-based naming (RBN) allows AAAA record DNS query. # IP-based hostname only support A records. privateDnsName: enableResourceNameDnsAAAARecord: true enableResourceNameDnsARecord: true hostnameType: resource-name --- apiVersion: bootstrap.cluster.x-k8s.io/v1beta1 kind: KubeadmConfigTemplate metadata: name: "${CLUSTER_NAME}-md-0" spec: template: spec: joinConfiguration: nodeRegistration: kubeletExtraArgs: cloud-provider: external name: '{{ ds.meta_data.local_hostname }}' --- apiVersion: addons.cluster.x-k8s.io/v1beta1 kind: ClusterResourceSet metadata: name: crs-ccm spec: clusterSelector: matchLabels: ccm: external resources: - kind: ConfigMap name: cloud-controller-manager-addon strategy: ApplyOnce --- apiVersion: addons.cluster.x-k8s.io/v1beta1 kind: ClusterResourceSet metadata: name: crs-csi spec: clusterSelector: matchLabels: csi: external resources: - kind: ConfigMap name: aws-ebs-csi-driver-addon strategy: ApplyOnce --- # We need to provide cloud-config to the CCM via a ConfigMap to # set the NodeIPFamilies to IPv4 and IPv6. # This instructs the CCM to also consider IPv6 in the node's network interface. apiVersion: v1 data: aws-ccm-external.yaml: | --- apiVersion: apps/v1 kind: DaemonSet metadata: name: aws-cloud-controller-manager namespace: kube-system labels: k8s-app: aws-cloud-controller-manager spec: selector: matchLabels: k8s-app: aws-cloud-controller-manager updateStrategy: type: RollingUpdate template: metadata: labels: k8s-app: aws-cloud-controller-manager spec: nodeSelector: node-role.kubernetes.io/control-plane: "" tolerations: - key: node.cloudprovider.kubernetes.io/uninitialized value: "true" effect: NoSchedule - key: node-role.kubernetes.io/control-plane effect: NoSchedule affinity: nodeAffinity: requiredDuringSchedulingIgnoredDuringExecution: nodeSelectorTerms: - matchExpressions: - key: node-role.kubernetes.io/control-plane operator: Exists serviceAccountName: cloud-controller-manager containers: - name: aws-cloud-controller-manager image: registry.k8s.io/provider-aws/cloud-controller-manager:v1.28.3 args: - --v=2 - --cloud-provider=aws - --use-service-account-credentials=true - --configure-cloud-routes=false - --cloud-config=/etc/kubernetes/cloud-config.conf volumeMounts: - name: cloud-config mountPath: /etc/kubernetes/cloud-config.conf subPath: cloud-config.conf resources: requests: cpu: 200m hostNetwork: true volumes: - name: cloud-config configMap: name: cloud-config --- apiVersion: v1 kind: ConfigMap metadata: name: cloud-config namespace: kube-system data: cloud-config.conf: | [Global] NodeIPFamilies=ipv4 NodeIPFamilies=ipv6 --- apiVersion: v1 kind: ServiceAccount metadata: name: cloud-controller-manager namespace: kube-system --- apiVersion: rbac.authorization.k8s.io/v1 kind: RoleBinding metadata: name: cloud-controller-manager:apiserver-authentication-reader namespace: kube-system roleRef: apiGroup: rbac.authorization.k8s.io kind: Role name: extension-apiserver-authentication-reader subjects: - apiGroup: "" kind: ServiceAccount name: cloud-controller-manager namespace: kube-system --- apiVersion: rbac.authorization.k8s.io/v1 kind: ClusterRole metadata: name: system:cloud-controller-manager rules: - apiGroups: - "" resources: - events verbs: - create - patch - update - apiGroups: - "" resources: - nodes verbs: - '*' - apiGroups: - "" resources: - nodes/status verbs: - patch - apiGroups: - "" resources: - services verbs: - list - patch - update - watch - apiGroups: - "" resources: - services/status verbs: - list - patch - update - watch - apiGroups: - "" resources: - serviceaccounts verbs: - create - get - list - watch - apiGroups: - "" resources: - persistentvolumes verbs: - get - list - update - watch - apiGroups: - "" resources: - configmaps verbs: - list - watch - apiGroups: - "" resources: - endpoints verbs: - create - get - list - watch - update - apiGroups: - coordination.k8s.io resources: - leases verbs: - create - get - list - watch - update - apiGroups: - "" resources: - serviceaccounts/token verbs: - create --- kind: ClusterRoleBinding apiVersion: rbac.authorization.k8s.io/v1 metadata: name: system:cloud-controller-manager roleRef: apiGroup: rbac.authorization.k8s.io kind: ClusterRole name: system:cloud-controller-manager subjects: - apiGroup: "" kind: ServiceAccount name: cloud-controller-manager namespace: kube-system kind: ConfigMap metadata: annotations: note: generated labels: type: generated name: cloud-controller-manager-addon --- apiVersion: v1 data: aws-ebs-csi-external.yaml: |- apiVersion: v1 kind: Secret metadata: name: aws-secret namespace: kube-system stringData: key_id: "" access_key: "" --- apiVersion: v1 kind: ServiceAccount metadata: labels: app.kubernetes.io/name: aws-ebs-csi-driver name: ebs-csi-controller-sa namespace: kube-system --- apiVersion: v1 kind: ServiceAccount metadata: labels: app.kubernetes.io/name: aws-ebs-csi-driver name: ebs-csi-node-sa namespace: kube-system --- apiVersion: rbac.authorization.k8s.io/v1 kind: ClusterRole metadata: labels: app.kubernetes.io/name: aws-ebs-csi-driver name: ebs-external-attacher-role rules: - apiGroups: - "" resources: - persistentvolumes verbs: - get - list - watch - update - patch - apiGroups: - "" resources: - nodes verbs: - get - list - watch - apiGroups: - csi.storage.k8s.io resources: - csinodeinfos verbs: - get - list - watch - apiGroups: - storage.k8s.io resources: - volumeattachments verbs: - get - list - watch - update - patch - apiGroups: - storage.k8s.io resources: - volumeattachments/status verbs: - patch --- apiVersion: rbac.authorization.k8s.io/v1 kind: ClusterRole metadata: labels: app.kubernetes.io/name: aws-ebs-csi-driver name: ebs-csi-node rules: - apiGroups: - "" resources: - pods verbs: - get - patch - apiGroups: - "" resources: - nodes verbs: - get --- apiVersion: rbac.authorization.k8s.io/v1 kind: ClusterRole metadata: labels: app.kubernetes.io/name: aws-ebs-csi-driver name: ebs-external-provisioner-role rules: - apiGroups: - "" resources: - persistentvolumes verbs: - get - list - watch - create - delete - apiGroups: - "" resources: - persistentvolumeclaims verbs: - get - list - watch - update - apiGroups: - storage.k8s.io resources: - storageclasses verbs: - get - list - watch - apiGroups: - "" resources: - events verbs: - list - watch - create - update - patch - apiGroups: - snapshot.storage.k8s.io resources: - volumesnapshots verbs: - get - list - apiGroups: - snapshot.storage.k8s.io resources: - volumesnapshotcontents verbs: - get - list - apiGroups: - storage.k8s.io resources: - csinodes verbs: - get - list - watch - apiGroups: - "" resources: - nodes verbs: - get - list - watch - apiGroups: - coordination.k8s.io resources: - leases verbs: - get - watch - list - delete - update - create - apiGroups: - storage.k8s.io resources: - volumeattachments verbs: - get - list - watch --- apiVersion: rbac.authorization.k8s.io/v1 kind: ClusterRole metadata: labels: app.kubernetes.io/name: aws-ebs-csi-driver name: ebs-external-resizer-role rules: - apiGroups: - "" resources: - persistentvolumes verbs: - get - list - watch - update - patch - apiGroups: - "" resources: - persistentvolumeclaims verbs: - get - list - watch - apiGroups: - "" resources: - persistentvolumeclaims/status verbs: - update - patch - apiGroups: - storage.k8s.io resources: - storageclasses verbs: - get - list - watch - apiGroups: - "" resources: - events verbs: - list - watch - create - update - patch - apiGroups: - "" resources: - pods verbs: - get - list - watch --- apiVersion: rbac.authorization.k8s.io/v1 kind: ClusterRole metadata: labels: app.kubernetes.io/name: aws-ebs-csi-driver name: ebs-external-snapshotter-role rules: - apiGroups: - "" resources: - events verbs: - list - watch - create - update - patch - apiGroups: - "" resources: - secrets verbs: - get - list - apiGroups: - snapshot.storage.k8s.io resources: - volumesnapshotclasses verbs: - get - list - watch - apiGroups: - snapshot.storage.k8s.io resources: - volumesnapshotcontents verbs: - create - get - list - watch - update - delete - apiGroups: - snapshot.storage.k8s.io resources: - volumesnapshotcontents/status verbs: - update --- apiVersion: rbac.authorization.k8s.io/v1 kind: ClusterRoleBinding metadata: labels: app.kubernetes.io/name: aws-ebs-csi-driver name: ebs-csi-attacher-binding roleRef: apiGroup: rbac.authorization.k8s.io kind: ClusterRole name: ebs-external-attacher-role subjects: - kind: ServiceAccount name: ebs-csi-controller-sa namespace: kube-system --- apiVersion: rbac.authorization.k8s.io/v1 kind: ClusterRoleBinding metadata: labels: app.kubernetes.io/name: aws-ebs-csi-driver name: ebs-csi-provisioner-binding roleRef: apiGroup: rbac.authorization.k8s.io kind: ClusterRole name: ebs-external-provisioner-role subjects: - kind: ServiceAccount name: ebs-csi-controller-sa namespace: kube-system --- apiVersion: rbac.authorization.k8s.io/v1 kind: ClusterRoleBinding metadata: labels: app.kubernetes.io/name: aws-ebs-csi-driver name: ebs-csi-resizer-binding roleRef: apiGroup: rbac.authorization.k8s.io kind: ClusterRole name: ebs-external-resizer-role subjects: - kind: ServiceAccount name: ebs-csi-controller-sa namespace: kube-system --- apiVersion: rbac.authorization.k8s.io/v1 kind: ClusterRoleBinding metadata: labels: app.kubernetes.io/name: aws-ebs-csi-driver name: ebs-csi-snapshotter-binding roleRef: apiGroup: rbac.authorization.k8s.io kind: ClusterRole name: ebs-external-snapshotter-role subjects: - kind: ServiceAccount name: ebs-csi-controller-sa namespace: kube-system --- apiVersion: rbac.authorization.k8s.io/v1 kind: ClusterRoleBinding metadata: labels: app.kubernetes.io/name: aws-ebs-csi-driver name: ebs-csi-node-binding roleRef: apiGroup: rbac.authorization.k8s.io kind: ClusterRole name: ebs-csi-node subjects: - kind: ServiceAccount name: ebs-csi-node-sa namespace: kube-system --- apiVersion: apps/v1 kind: Deployment metadata: labels: app.kubernetes.io/name: aws-ebs-csi-driver name: ebs-csi-controller namespace: kube-system spec: replicas: 2 selector: matchLabels: app: ebs-csi-controller app.kubernetes.io/name: aws-ebs-csi-driver template: metadata: labels: app: ebs-csi-controller app.kubernetes.io/name: aws-ebs-csi-driver spec: containers: - args: - --endpoint=$(CSI_ENDPOINT) - --logtostderr - --v=2 env: - name: CSI_ENDPOINT value: unix:///var/lib/csi/sockets/pluginproxy/csi.sock - name: CSI_NODE_NAME valueFrom: fieldRef: fieldPath: spec.nodeName - name: AWS_ACCESS_KEY_ID valueFrom: secretKeyRef: key: key_id name: aws-secret optional: true - name: AWS_SECRET_ACCESS_KEY valueFrom: secretKeyRef: key: access_key name: aws-secret optional: true image: registry.k8s.io/provider-aws/aws-ebs-csi-driver:v1.25.0 imagePullPolicy: IfNotPresent livenessProbe: failureThreshold: 5 httpGet: path: /healthz port: healthz initialDelaySeconds: 10 periodSeconds: 10 timeoutSeconds: 3 name: ebs-plugin ports: - containerPort: 9808 name: healthz protocol: TCP readinessProbe: failureThreshold: 5 httpGet: path: /healthz port: healthz initialDelaySeconds: 10 periodSeconds: 10 timeoutSeconds: 3 volumeMounts: - mountPath: /var/lib/csi/sockets/pluginproxy/ name: socket-dir - args: - --csi-address=$(ADDRESS) - --v=2 - --feature-gates=Topology=true - --extra-create-metadata - --leader-election=true - --default-fstype=ext4 env: - name: ADDRESS value: /var/lib/csi/sockets/pluginproxy/csi.sock image: registry.k8s.io/sig-storage/csi-provisioner:v3.6.2 name: csi-provisioner volumeMounts: - mountPath: /var/lib/csi/sockets/pluginproxy/ name: socket-dir - args: - --csi-address=$(ADDRESS) - --v=2 - --leader-election=true env: - name: ADDRESS value: /var/lib/csi/sockets/pluginproxy/csi.sock image: registry.k8s.io/sig-storage/csi-attacher:v4.4.2 name: csi-attacher volumeMounts: - mountPath: /var/lib/csi/sockets/pluginproxy/ name: socket-dir - args: - --csi-address=$(ADDRESS) - --leader-election=true env: - name: ADDRESS value: /var/lib/csi/sockets/pluginproxy/csi.sock image: registry.k8s.io/sig-storage/csi-snapshotter:v6.3.2 name: csi-snapshotter volumeMounts: - mountPath: /var/lib/csi/sockets/pluginproxy/ name: socket-dir - args: - --csi-address=$(ADDRESS) - --v=2 env: - name: ADDRESS value: /var/lib/csi/sockets/pluginproxy/csi.sock image: registry.k8s.io/sig-storage/csi-resizer:v1.9.2 imagePullPolicy: Always name: csi-resizer volumeMounts: - mountPath: /var/lib/csi/sockets/pluginproxy/ name: socket-dir - args: - --csi-address=/csi/csi.sock image: registry.k8s.io/sig-storage/livenessprobe:v2.11.0 name: liveness-probe volumeMounts: - mountPath: /csi name: socket-dir nodeSelector: kubernetes.io/os: linux priorityClassName: system-cluster-critical serviceAccountName: ebs-csi-controller-sa tolerations: - key: CriticalAddonsOnly operator: Exists - effect: NoExecute operator: Exists tolerationSeconds: 300 - key: node-role.kubernetes.io/master effect: NoSchedule - effect: NoSchedule key: node-role.kubernetes.io/control-plane affinity: nodeAffinity: requiredDuringSchedulingIgnoredDuringExecution: nodeSelectorTerms: - matchExpressions: - key: node-role.kubernetes.io/control-plane operator: Exists - matchExpressions: - key: node-role.kubernetes.io/master operator: Exists volumes: - emptyDir: {} name: socket-dir --- apiVersion: policy/v1 kind: PodDisruptionBudget metadata: labels: app.kubernetes.io/name: aws-ebs-csi-driver name: ebs-csi-controller namespace: kube-system spec: maxUnavailable: 1 selector: matchLabels: app: ebs-csi-controller app.kubernetes.io/name: aws-ebs-csi-driver --- apiVersion: apps/v1 kind: DaemonSet metadata: labels: app.kubernetes.io/name: aws-ebs-csi-driver name: ebs-csi-node namespace: kube-system spec: selector: matchLabels: app: ebs-csi-node app.kubernetes.io/name: aws-ebs-csi-driver template: metadata: labels: app: ebs-csi-node app.kubernetes.io/name: aws-ebs-csi-driver spec: affinity: nodeAffinity: requiredDuringSchedulingIgnoredDuringExecution: nodeSelectorTerms: - matchExpressions: - key: eks.amazonaws.com/compute-type operator: NotIn values: - fargate containers: - args: - node - --endpoint=$(CSI_ENDPOINT) - --logtostderr - --v=2 env: - name: CSI_ENDPOINT value: unix:/csi/csi.sock - name: CSI_NODE_NAME valueFrom: fieldRef: fieldPath: spec.nodeName image: registry.k8s.io/provider-aws/aws-ebs-csi-driver:v1.25.0 livenessProbe: failureThreshold: 5 httpGet: path: /healthz port: healthz initialDelaySeconds: 10 periodSeconds: 10 timeoutSeconds: 3 name: ebs-plugin ports: - containerPort: 9808 name: healthz protocol: TCP securityContext: privileged: true volumeMounts: - mountPath: /var/lib/kubelet mountPropagation: Bidirectional name: kubelet-dir - mountPath: /csi name: plugin-dir - mountPath: /dev name: device-dir - args: - --csi-address=$(ADDRESS) - --kubelet-registration-path=$(DRIVER_REG_SOCK_PATH) - --v=2 env: - name: ADDRESS value: /csi/csi.sock - name: DRIVER_REG_SOCK_PATH value: /var/lib/kubelet/plugins/ebs.csi.aws.com/csi.sock image: registry.k8s.io/sig-storage/csi-node-driver-registrar:v2.9.2 name: node-driver-registrar volumeMounts: - mountPath: /csi name: plugin-dir - mountPath: /registration name: registration-dir - args: - --csi-address=/csi/csi.sock image: registry.k8s.io/sig-storage/livenessprobe:v2.11.0 name: liveness-probe volumeMounts: - mountPath: /csi name: plugin-dir nodeSelector: kubernetes.io/os: linux priorityClassName: system-node-critical serviceAccountName: ebs-csi-node-sa tolerations: - key: CriticalAddonsOnly operator: Exists - effect: NoExecute operator: Exists tolerationSeconds: 300 volumes: - hostPath: path: /var/lib/kubelet type: Directory name: kubelet-dir - hostPath: path: /var/lib/kubelet/plugins/ebs.csi.aws.com/ type: DirectoryOrCreate name: plugin-dir - hostPath: path: /var/lib/kubelet/plugins_registry/ type: Directory name: registration-dir - hostPath: path: /dev type: Directory name: device-dir updateStrategy: rollingUpdate: maxUnavailable: 10% type: RollingUpdate --- apiVersion: storage.k8s.io/v1 kind: CSIDriver metadata: labels: app.kubernetes.io/name: aws-ebs-csi-driver name: ebs.csi.aws.com spec: attachRequired: true podInfoOnMount: false kind: ConfigMap metadata: annotations: note: generated labels: type: generated name: aws-ebs-csi-driver-addon