apiVersion: cluster.x-k8s.io/v1alpha4 kind: Cluster metadata: labels: ccm: external cni: calico name: ${CLUSTER_NAME} namespace: default spec: clusterNetwork: pods: cidrBlocks: - 192.168.0.0/16 controlPlaneRef: apiVersion: controlplane.cluster.x-k8s.io/v1alpha4 kind: KubeadmControlPlane name: ${CLUSTER_NAME}-control-plane infrastructureRef: apiVersion: infrastructure.cluster.x-k8s.io/v1alpha4 kind: AzureCluster name: ${CLUSTER_NAME} --- apiVersion: infrastructure.cluster.x-k8s.io/v1alpha4 kind: AzureCluster metadata: name: ${CLUSTER_NAME} namespace: default spec: location: ${AZURE_LOCATION} networkSpec: vnet: name: ${AZURE_VNET_NAME:=${CLUSTER_NAME}-vnet} resourceGroup: ${AZURE_RESOURCE_GROUP:=${CLUSTER_NAME}} subscriptionID: ${AZURE_SUBSCRIPTION_ID} --- apiVersion: controlplane.cluster.x-k8s.io/v1alpha4 kind: KubeadmControlPlane metadata: name: ${CLUSTER_NAME}-control-plane namespace: default spec: infrastructureTemplate: apiVersion: infrastructure.cluster.x-k8s.io/v1alpha4 kind: AzureMachineTemplate name: ${CLUSTER_NAME}-control-plane kubeadmConfigSpec: clusterConfiguration: apiServer: extraArgs: cloud-config: /etc/kubernetes/azure.json cloud-provider: azure extraVolumes: - hostPath: /etc/kubernetes/azure.json mountPath: /etc/kubernetes/azure.json name: cloud-config readOnly: true timeoutForControlPlane: 20m controllerManager: extraArgs: allocate-node-cidrs: "false" cloud-config: /etc/kubernetes/azure.json cloud-provider: external cluster-name: ${CLUSTER_NAME} extraVolumes: - hostPath: /etc/kubernetes/azure.json mountPath: /etc/kubernetes/azure.json name: cloud-config readOnly: true etcd: local: dataDir: /var/lib/etcddisk/etcd diskSetup: filesystems: - device: /dev/disk/azure/scsi1/lun0 extraOpts: - -E - lazy_itable_init=1,lazy_journal_init=1 filesystem: ext4 label: etcd_disk - device: ephemeral0.1 filesystem: ext4 label: ephemeral0 replaceFS: ntfs partitions: - device: /dev/disk/azure/scsi1/lun0 layout: true overwrite: false tableType: gpt files: - contentFrom: secret: key: control-plane-azure.json name: ${CLUSTER_NAME}-control-plane-azure-json owner: root:root path: /etc/kubernetes/azure.json permissions: "0644" initConfiguration: nodeRegistration: kubeletExtraArgs: cloud-config: /etc/kubernetes/azure.json cloud-provider: external name: '{{ ds.meta_data["local_hostname"] }}' joinConfiguration: nodeRegistration: kubeletExtraArgs: cloud-config: /etc/kubernetes/azure.json cloud-provider: external name: '{{ ds.meta_data["local_hostname"] }}' mounts: - - LABEL=etcd_disk - /var/lib/etcddisk replicas: ${CONTROL_PLANE_MACHINE_COUNT} version: ${KUBERNETES_VERSION} --- apiVersion: infrastructure.cluster.x-k8s.io/v1alpha4 kind: AzureMachineTemplate metadata: name: ${CLUSTER_NAME}-control-plane namespace: default spec: template: spec: dataDisks: - diskSizeGB: 256 lun: 0 nameSuffix: etcddisk osDisk: diskSizeGB: 128 managedDisk: storageAccountType: Premium_LRS osType: Linux sshPublicKey: ${AZURE_SSH_PUBLIC_KEY_B64:=""} vmSize: ${AZURE_CONTROL_PLANE_MACHINE_TYPE} --- apiVersion: cluster.x-k8s.io/v1alpha4 kind: MachineDeployment metadata: name: ${CLUSTER_NAME}-md-0 namespace: default spec: clusterName: ${CLUSTER_NAME} replicas: ${WORKER_MACHINE_COUNT} selector: matchLabels: null template: spec: bootstrap: configRef: apiVersion: bootstrap.cluster.x-k8s.io/v1alpha4 kind: KubeadmConfigTemplate name: ${CLUSTER_NAME}-md-0 clusterName: ${CLUSTER_NAME} infrastructureRef: apiVersion: infrastructure.cluster.x-k8s.io/v1alpha4 kind: AzureMachineTemplate name: ${CLUSTER_NAME}-md-0 version: ${KUBERNETES_VERSION} --- apiVersion: infrastructure.cluster.x-k8s.io/v1alpha4 kind: AzureMachineTemplate metadata: name: ${CLUSTER_NAME}-md-0 namespace: default spec: template: spec: osDisk: diskSizeGB: 128 managedDisk: storageAccountType: Premium_LRS osType: Linux sshPublicKey: ${AZURE_SSH_PUBLIC_KEY_B64:=""} vmSize: ${AZURE_NODE_MACHINE_TYPE} --- apiVersion: bootstrap.cluster.x-k8s.io/v1alpha4 kind: KubeadmConfigTemplate metadata: name: ${CLUSTER_NAME}-md-0 namespace: default spec: template: spec: files: - contentFrom: secret: key: worker-node-azure.json name: ${CLUSTER_NAME}-md-0-azure-json owner: root:root path: /etc/kubernetes/azure.json permissions: "0644" joinConfiguration: nodeRegistration: kubeletExtraArgs: cloud-config: /etc/kubernetes/azure.json cloud-provider: external name: '{{ ds.meta_data["local_hostname"] }}' --- apiVersion: addons.cluster.x-k8s.io/v1alpha4 kind: ClusterResourceSet metadata: name: crs-ccm namespace: default spec: clusterSelector: matchLabels: ccm: external resources: - kind: ConfigMap name: cloud-controller-manager-addon strategy: ApplyOnce --- apiVersion: addons.cluster.x-k8s.io/v1alpha4 kind: ClusterResourceSet metadata: name: crs-node-manager namespace: default spec: clusterSelector: matchLabels: ccm: external resources: - kind: ConfigMap name: cloud-node-manager-addon strategy: ApplyOnce --- apiVersion: v1 data: cloud-controller-manager.yaml: | apiVersion: v1 kind: ServiceAccount metadata: name: cloud-controller-manager namespace: kube-system --- apiVersion: rbac.authorization.k8s.io/v1 kind: ClusterRole metadata: name: system:cloud-controller-manager annotations: rbac.authorization.kubernetes.io/autoupdate: "true" labels: k8s-app: cloud-controller-manager rules: - apiGroups: - "" resources: - events verbs: - create - patch - update - apiGroups: - "" resources: - nodes verbs: - "*" - apiGroups: - "" resources: - nodes/status verbs: - patch - apiGroups: - "" resources: - services verbs: - list - patch - update - watch - apiGroups: - "" resources: - services/status verbs: - list - patch - update - watch - apiGroups: - "" resources: - serviceaccounts verbs: - create - get - list - watch - update - apiGroups: - "" resources: - persistentvolumes verbs: - get - list - update - watch - apiGroups: - "" resources: - endpoints verbs: - create - get - list - watch - update - apiGroups: - "" resources: - secrets verbs: - get - list - watch - apiGroups: - coordination.k8s.io resources: - leases verbs: - get - create - update --- kind: ClusterRoleBinding apiVersion: rbac.authorization.k8s.io/v1 metadata: name: system:cloud-controller-manager roleRef: apiGroup: rbac.authorization.k8s.io kind: ClusterRole name: system:cloud-controller-manager subjects: - kind: ServiceAccount name: cloud-controller-manager namespace: kube-system - kind: User name: cloud-controller-manager --- apiVersion: rbac.authorization.k8s.io/v1 kind: RoleBinding metadata: name: system:cloud-controller-manager:extension-apiserver-authentication-reader namespace: kube-system roleRef: apiGroup: rbac.authorization.k8s.io kind: Role name: extension-apiserver-authentication-reader subjects: - kind: ServiceAccount name: cloud-controller-manager namespace: kube-system - apiGroup: "" kind: User name: cloud-controller-manager --- apiVersion: v1 kind: Pod metadata: name: cloud-controller-manager namespace: kube-system labels: tier: control-plane component: cloud-controller-manager spec: priorityClassName: system-node-critical hostNetwork: true nodeSelector: node-role.kubernetes.io/master: "" serviceAccountName: cloud-controller-manager tolerations: - key: node-role.kubernetes.io/master effect: NoSchedule containers: - name: cloud-controller-manager image: ${AZURE_CLOUD_CONTROLLER_MANAGER_IMG:=mcr.microsoft.com/oss/kubernetes/azure-cloud-controller-manager:v0.7.2} imagePullPolicy: IfNotPresent command: ["cloud-controller-manager"] args: - "--allocate-node-cidrs=true" - "--cloud-config=/etc/kubernetes/azure.json" - "--cloud-provider=azure" - "--cluster-cidr=10.244.0.0/16" - "--cluster-name=${CLUSTER_NAME}" - "--controllers=*,-cloud-node" # disable cloud-node controller - "--configure-cloud-routes=true" # "false" for Azure CNI and "true" for other network plugins - "--leader-elect=true" - "--route-reconciliation-period=10s" - "--v=2" - "--port=10267" resources: requests: cpu: 100m memory: 128Mi limits: cpu: "4" memory: 2Gi livenessProbe: httpGet: path: /healthz port: 10267 initialDelaySeconds: 20 periodSeconds: 10 timeoutSeconds: 5 volumeMounts: - name: etc-kubernetes mountPath: /etc/kubernetes - name: etc-ssl mountPath: /etc/ssl readOnly: true - name: msi mountPath: /var/lib/waagent/ManagedIdentity-Settings readOnly: true volumes: - name: etc-kubernetes hostPath: path: /etc/kubernetes - name: etc-ssl hostPath: path: /etc/ssl - name: msi hostPath: path: /var/lib/waagent/ManagedIdentity-Settings kind: ConfigMap metadata: annotations: note: generated labels: type: generated name: cloud-controller-manager-addon namespace: default --- apiVersion: v1 data: cloud-node-manager.yaml: | apiVersion: v1 kind: ServiceAccount metadata: labels: k8s-app: cloud-node-manager name: cloud-node-manager namespace: kube-system --- kind: ClusterRole apiVersion: rbac.authorization.k8s.io/v1 metadata: name: cloud-node-manager labels: k8s-app: cloud-node-manager rules: - apiGroups: [""] resources: ["nodes"] verbs: ["watch", "list", "get", "update", "patch"] --- apiVersion: rbac.authorization.k8s.io/v1 kind: ClusterRoleBinding metadata: name: cloud-node-manager labels: k8s-app: cloud-node-manager roleRef: apiGroup: rbac.authorization.k8s.io kind: ClusterRole name: cloud-node-manager subjects: - kind: ServiceAccount name: cloud-node-manager namespace: kube-system --- apiVersion: apps/v1 kind: DaemonSet metadata: name: cloud-node-manager namespace: kube-system labels: component: cloud-node-manager spec: selector: matchLabels: k8s-app: cloud-node-manager template: metadata: labels: k8s-app: cloud-node-manager annotations: cluster-autoscaler.kubernetes.io/daemonset-pod: "true" spec: priorityClassName: system-node-critical serviceAccountName: cloud-node-manager hostNetwork: true # required to fetch correct hostname nodeSelector: kubernetes.io/os: linux tolerations: - key: CriticalAddonsOnly operator: Exists - key: node-role.kubernetes.io/master effect: NoSchedule - operator: "Exists" effect: NoExecute - operator: "Exists" effect: NoSchedule containers: - name: cloud-node-manager image: ${AZURE_CLOUD_NODE_MANAGER_IMG:=mcr.microsoft.com/oss/kubernetes/azure-cloud-node-manager:v0.7.2} imagePullPolicy: IfNotPresent command: - cloud-node-manager - --node-name=$(NODE_NAME) env: - name: NODE_NAME valueFrom: fieldRef: fieldPath: spec.nodeName resources: requests: cpu: 50m memory: 50Mi limits: cpu: 2000m memory: 512Mi kind: ConfigMap metadata: annotations: note: generated labels: type: generated name: cloud-node-manager-addon namespace: default