apiVersion: cluster.x-k8s.io/v1alpha4
kind: Cluster
metadata:
  labels:
    ccm: external
    cni: calico
  name: ${CLUSTER_NAME}
  namespace: default
spec:
  clusterNetwork:
    pods:
      cidrBlocks:
      - 192.168.0.0/16
  controlPlaneRef:
    apiVersion: controlplane.cluster.x-k8s.io/v1alpha4
    kind: KubeadmControlPlane
    name: ${CLUSTER_NAME}-control-plane
  infrastructureRef:
    apiVersion: infrastructure.cluster.x-k8s.io/v1alpha4
    kind: AzureCluster
    name: ${CLUSTER_NAME}
---
apiVersion: infrastructure.cluster.x-k8s.io/v1alpha4
kind: AzureCluster
metadata:
  name: ${CLUSTER_NAME}
  namespace: default
spec:
  identityRef:
    apiVersion: infrastructure.cluster.x-k8s.io/v1alpha4
    kind: AzureClusterIdentity
    name: ${CLUSTER_IDENTITY_NAME}
  location: ${AZURE_LOCATION}
  networkSpec:
    vnet:
      name: ${AZURE_VNET_NAME:=${CLUSTER_NAME}-vnet}
  resourceGroup: ${AZURE_RESOURCE_GROUP:=${CLUSTER_NAME}}
  subscriptionID: ${AZURE_SUBSCRIPTION_ID}
---
apiVersion: controlplane.cluster.x-k8s.io/v1alpha4
kind: KubeadmControlPlane
metadata:
  name: ${CLUSTER_NAME}-control-plane
  namespace: default
spec:
  kubeadmConfigSpec:
    clusterConfiguration:
      apiServer:
        extraArgs:
          cloud-config: /etc/kubernetes/azure.json
          cloud-provider: azure
        extraVolumes:
        - hostPath: /etc/kubernetes/azure.json
          mountPath: /etc/kubernetes/azure.json
          name: cloud-config
          readOnly: true
        timeoutForControlPlane: 20m
      controllerManager:
        extraArgs:
          allocate-node-cidrs: "false"
          cloud-config: /etc/kubernetes/azure.json
          cloud-provider: external
          cluster-name: ${CLUSTER_NAME}
        extraVolumes:
        - hostPath: /etc/kubernetes/azure.json
          mountPath: /etc/kubernetes/azure.json
          name: cloud-config
          readOnly: true
      etcd:
        local:
          dataDir: /var/lib/etcddisk/etcd
    diskSetup:
      filesystems:
      - device: /dev/disk/azure/scsi1/lun0
        extraOpts:
        - -E
        - lazy_itable_init=1,lazy_journal_init=1
        filesystem: ext4
        label: etcd_disk
      - device: ephemeral0.1
        filesystem: ext4
        label: ephemeral0
        replaceFS: ntfs
      partitions:
      - device: /dev/disk/azure/scsi1/lun0
        layout: true
        overwrite: false
        tableType: gpt
    files:
    - contentFrom:
        secret:
          key: control-plane-azure.json
          name: ${CLUSTER_NAME}-control-plane-azure-json
      owner: root:root
      path: /etc/kubernetes/azure.json
      permissions: "0644"
    initConfiguration:
      nodeRegistration:
        kubeletExtraArgs:
          azure-container-registry-config: /etc/kubernetes/azure.json
          cloud-config: /etc/kubernetes/azure.json
          cloud-provider: external
        name: '{{ ds.meta_data["local_hostname"] }}'
    joinConfiguration:
      nodeRegistration:
        kubeletExtraArgs:
          azure-container-registry-config: /etc/kubernetes/azure.json
          cloud-config: /etc/kubernetes/azure.json
          cloud-provider: external
        name: '{{ ds.meta_data["local_hostname"] }}'
    mounts:
    - - LABEL=etcd_disk
      - /var/lib/etcddisk
  machineTemplate:
    infrastructureRef:
      apiVersion: infrastructure.cluster.x-k8s.io/v1alpha4
      kind: AzureMachineTemplate
      name: ${CLUSTER_NAME}-control-plane
  replicas: ${CONTROL_PLANE_MACHINE_COUNT}
  version: ${KUBERNETES_VERSION}
---
apiVersion: infrastructure.cluster.x-k8s.io/v1alpha4
kind: AzureMachineTemplate
metadata:
  name: ${CLUSTER_NAME}-control-plane
  namespace: default
spec:
  template:
    spec:
      dataDisks:
      - diskSizeGB: 256
        lun: 0
        nameSuffix: etcddisk
      osDisk:
        diskSizeGB: 128
        osType: Linux
      sshPublicKey: ${AZURE_SSH_PUBLIC_KEY_B64:=""}
      vmSize: ${AZURE_CONTROL_PLANE_MACHINE_TYPE}
---
apiVersion: cluster.x-k8s.io/v1alpha4
kind: MachineDeployment
metadata:
  name: ${CLUSTER_NAME}-md-0
  namespace: default
spec:
  clusterName: ${CLUSTER_NAME}
  replicas: ${WORKER_MACHINE_COUNT}
  selector:
    matchLabels: null
  template:
    spec:
      bootstrap:
        configRef:
          apiVersion: bootstrap.cluster.x-k8s.io/v1alpha4
          kind: KubeadmConfigTemplate
          name: ${CLUSTER_NAME}-md-0
      clusterName: ${CLUSTER_NAME}
      infrastructureRef:
        apiVersion: infrastructure.cluster.x-k8s.io/v1alpha4
        kind: AzureMachineTemplate
        name: ${CLUSTER_NAME}-md-0
      version: ${KUBERNETES_VERSION}
---
apiVersion: infrastructure.cluster.x-k8s.io/v1alpha4
kind: AzureMachineTemplate
metadata:
  name: ${CLUSTER_NAME}-md-0
  namespace: default
spec:
  template:
    spec:
      osDisk:
        diskSizeGB: 128
        osType: Linux
      sshPublicKey: ${AZURE_SSH_PUBLIC_KEY_B64:=""}
      vmSize: ${AZURE_NODE_MACHINE_TYPE}
---
apiVersion: bootstrap.cluster.x-k8s.io/v1alpha4
kind: KubeadmConfigTemplate
metadata:
  name: ${CLUSTER_NAME}-md-0
  namespace: default
spec:
  template:
    spec:
      files:
      - contentFrom:
          secret:
            key: worker-node-azure.json
            name: ${CLUSTER_NAME}-md-0-azure-json
        owner: root:root
        path: /etc/kubernetes/azure.json
        permissions: "0644"
      joinConfiguration:
        nodeRegistration:
          kubeletExtraArgs:
            azure-container-registry-config: /etc/kubernetes/azure.json
            cloud-config: /etc/kubernetes/azure.json
            cloud-provider: external
          name: '{{ ds.meta_data["local_hostname"] }}'
---
apiVersion: infrastructure.cluster.x-k8s.io/v1alpha4
kind: AzureClusterIdentity
metadata:
  labels:
    clusterctl.cluster.x-k8s.io/move-hierarchy: "true"
  name: ${CLUSTER_IDENTITY_NAME}
  namespace: default
spec:
  allowedNamespaces: {}
  clientID: ${AZURE_CLIENT_ID}
  clientSecret:
    name: ${AZURE_CLUSTER_IDENTITY_SECRET_NAME}
    namespace: ${AZURE_CLUSTER_IDENTITY_SECRET_NAMESPACE}
  tenantID: ${AZURE_TENANT_ID}
  type: ServicePrincipal
---
apiVersion: addons.cluster.x-k8s.io/v1alpha4
kind: ClusterResourceSet
metadata:
  name: crs-ccm
  namespace: default
spec:
  clusterSelector:
    matchLabels:
      ccm: external
  resources:
  - kind: ConfigMap
    name: cloud-controller-manager-addon
  strategy: ApplyOnce
---
apiVersion: addons.cluster.x-k8s.io/v1alpha4
kind: ClusterResourceSet
metadata:
  name: crs-node-manager
  namespace: default
spec:
  clusterSelector:
    matchLabels:
      ccm: external
  resources:
  - kind: ConfigMap
    name: cloud-node-manager-addon
  strategy: ApplyOnce
---
apiVersion: v1
data:
  cloud-controller-manager.yaml: |
    apiVersion: v1
    kind: ServiceAccount
    metadata:
      name: cloud-controller-manager
      namespace: kube-system
    ---
    apiVersion: rbac.authorization.k8s.io/v1
    kind: ClusterRole
    metadata:
      name: system:cloud-controller-manager
      annotations:
        rbac.authorization.kubernetes.io/autoupdate: "true"
      labels:
        k8s-app: cloud-controller-manager
    rules:
      - apiGroups:
          - ""
        resources:
          - events
        verbs:
          - create
          - patch
          - update
      - apiGroups:
          - ""
        resources:
          - nodes
        verbs:
          - "*"
      - apiGroups:
          - ""
        resources:
          - nodes/status
        verbs:
          - patch
      - apiGroups:
          - ""
        resources:
          - services
        verbs:
          - list
          - patch
          - update
          - watch
      - apiGroups:
          - ""
        resources:
          - services/status
        verbs:
          - list
          - patch
          - update
          - watch
      - apiGroups:
          - ""
        resources:
          - serviceaccounts
        verbs:
          - create
          - get
          - list
          - watch
          - update
      - apiGroups:
          - ""
        resources:
          - persistentvolumes
        verbs:
          - get
          - list
          - update
          - watch
      - apiGroups:
          - ""
        resources:
          - endpoints
        verbs:
          - create
          - get
          - list
          - watch
          - update
      - apiGroups:
          - ""
        resources:
          - secrets
        verbs:
          - get
          - list
          - watch
      - apiGroups:
          - coordination.k8s.io
        resources:
          - leases
        verbs:
          - get
          - create
          - update
    ---
    kind: ClusterRoleBinding
    apiVersion: rbac.authorization.k8s.io/v1
    metadata:
      name: system:cloud-controller-manager
    roleRef:
      apiGroup: rbac.authorization.k8s.io
      kind: ClusterRole
      name: system:cloud-controller-manager
    subjects:
      - kind: ServiceAccount
        name: cloud-controller-manager
        namespace: kube-system
      - kind: User
        name: cloud-controller-manager
    ---
    apiVersion: rbac.authorization.k8s.io/v1
    kind: RoleBinding
    metadata:
      name: system:cloud-controller-manager:extension-apiserver-authentication-reader
      namespace: kube-system
    roleRef:
      apiGroup: rbac.authorization.k8s.io
      kind: Role
      name: extension-apiserver-authentication-reader
    subjects:
      - kind: ServiceAccount
        name: cloud-controller-manager
        namespace: kube-system
      - apiGroup: ""
        kind: User
        name: cloud-controller-manager
    ---
    apiVersion: v1
    kind: Pod
    metadata:
      name: cloud-controller-manager
      namespace: kube-system
      labels:
        tier: control-plane
        component: cloud-controller-manager
    spec:
      priorityClassName: system-node-critical
      hostNetwork: true
      nodeSelector:
        node-role.kubernetes.io/master: ""
      serviceAccountName: cloud-controller-manager
      tolerations:
        - key: node-role.kubernetes.io/master
          effect: NoSchedule
      containers:
        - name: cloud-controller-manager
          image: ${AZURE_CLOUD_CONTROLLER_MANAGER_IMG:=mcr.microsoft.com/oss/kubernetes/azure-cloud-controller-manager:v1.0.1}
          imagePullPolicy: IfNotPresent
          command: ["cloud-controller-manager"]
          args:
            - "--allocate-node-cidrs=true"
            - "--cloud-config=/etc/kubernetes/azure.json"
            - "--cloud-provider=azure"
            - "--cluster-cidr=10.244.0.0/16"
            - "--cluster-name=${CLUSTER_NAME}"
            - "--controllers=*,-cloud-node" # disable cloud-node controller
            - "--configure-cloud-routes=true" # "false" for Azure CNI and "true" for other network plugins
            - "--leader-elect=true"
            - "--route-reconciliation-period=10s"
            - "--v=2"
            - "--port=10267"
          resources:
            requests:
              cpu: 100m
              memory: 128Mi
            limits:
              cpu: "4"
              memory: 2Gi
          livenessProbe:
            httpGet:
              path: /healthz
              port: 10267
            initialDelaySeconds: 20
            periodSeconds: 10
            timeoutSeconds: 5
          volumeMounts:
            - name: etc-kubernetes
              mountPath: /etc/kubernetes
            - name: etc-ssl
              mountPath: /etc/ssl
              readOnly: true
            - name: msi
              mountPath: /var/lib/waagent/ManagedIdentity-Settings
              readOnly: true
      volumes:
        - name: etc-kubernetes
          hostPath:
            path: /etc/kubernetes
        - name: etc-ssl
          hostPath:
            path: /etc/ssl
        - name: msi
          hostPath:
            path: /var/lib/waagent/ManagedIdentity-Settings
kind: ConfigMap
metadata:
  annotations:
    note: generated
  labels:
    type: generated
  name: cloud-controller-manager-addon
  namespace: default
---
apiVersion: v1
data:
  cloud-node-manager.yaml: |
    apiVersion: v1
    kind: ServiceAccount
    metadata:
      labels:
        k8s-app: cloud-node-manager
      name: cloud-node-manager
      namespace: kube-system
    ---
    kind: ClusterRole
    apiVersion: rbac.authorization.k8s.io/v1
    metadata:
      name: cloud-node-manager
      labels:
        k8s-app: cloud-node-manager
    rules:
      - apiGroups: [""]
        resources: ["nodes"]
        verbs: ["watch", "list", "get", "update", "patch"]
    ---
    apiVersion: rbac.authorization.k8s.io/v1
    kind: ClusterRoleBinding
    metadata:
      name: cloud-node-manager
      labels:
        k8s-app: cloud-node-manager
    roleRef:
      apiGroup: rbac.authorization.k8s.io
      kind: ClusterRole
      name: cloud-node-manager
    subjects:
      - kind: ServiceAccount
        name: cloud-node-manager
        namespace: kube-system
    ---
    apiVersion: apps/v1
    kind: DaemonSet
    metadata:
      name: cloud-node-manager
      namespace: kube-system
      labels:
        component: cloud-node-manager
    spec:
      selector:
        matchLabels:
          k8s-app: cloud-node-manager
      template:
        metadata:
          labels:
            k8s-app: cloud-node-manager
          annotations:
            cluster-autoscaler.kubernetes.io/daemonset-pod: "true"
        spec:
          priorityClassName: system-node-critical
          serviceAccountName: cloud-node-manager
          hostNetwork: true # required to fetch correct hostname
          nodeSelector:
            kubernetes.io/os: linux
          tolerations:
            - key: CriticalAddonsOnly
              operator: Exists
            - key: node-role.kubernetes.io/master
              effect: NoSchedule
            - operator: "Exists"
              effect: NoExecute
            - operator: "Exists"
              effect: NoSchedule
          containers:
            - name: cloud-node-manager
              image: ${AZURE_CLOUD_NODE_MANAGER_IMG:=mcr.microsoft.com/oss/kubernetes/azure-cloud-node-manager:v1.0.1}
              imagePullPolicy: IfNotPresent
              command:
                - cloud-node-manager
                - --node-name=$(NODE_NAME)
              env:
                - name: NODE_NAME
                  valueFrom:
                    fieldRef:
                      fieldPath: spec.nodeName
              resources:
                requests:
                  cpu: 50m
                  memory: 50Mi
                limits:
                  cpu: 2000m
                  memory: 512Mi
kind: ConfigMap
metadata:
  annotations:
    note: generated
  labels:
    type: generated
  name: cloud-node-manager-addon
  namespace: default