apiVersion: storage.k8s.io/v1
kind: CSIDriver
metadata:
  name: csi.vsphere.vmware.com
spec:
  attachRequired: true
  podInfoOnMount: false
---
kind: ServiceAccount
apiVersion: v1
metadata:
  name: vsphere-csi-controller
  namespace: vmware-system-csi
---
kind: ClusterRole
apiVersion: rbac.authorization.k8s.io/v1
metadata:
  name: vsphere-csi-controller-role
rules:
  - apiGroups: [""]
    resources: ["nodes", "pods"]
    verbs: ["get", "list", "watch"]
  - apiGroups: [""]
    resources: ["configmaps"]
    verbs: ["get", "list", "watch", "create"]
  - apiGroups: [""]
    resources: ["persistentvolumeclaims"]
    verbs: ["get", "list", "watch", "update"]
  - apiGroups: [""]
    resources: ["persistentvolumeclaims/status"]
    verbs: ["patch"]
  - apiGroups: [""]
    resources: ["persistentvolumes"]
    verbs: ["get", "list", "watch", "create", "update", "delete", "patch"]
  - apiGroups: [""]
    resources: ["events"]
    verbs: ["get", "list", "watch", "create", "update", "patch"]
  - apiGroups: ["coordination.k8s.io"]
    resources: ["leases"]
    verbs: ["get", "watch", "list", "delete", "update", "create"]
  - apiGroups: ["storage.k8s.io"]
    resources: ["storageclasses", "csinodes"]
    verbs: ["get", "list", "watch"]
  - apiGroups: ["storage.k8s.io"]
    resources: ["volumeattachments"]
    verbs: ["get", "list", "watch", "patch"]
  - apiGroups: ["cns.vmware.com"]
    resources: ["triggercsifullsyncs"]
    verbs: ["create", "get", "update", "watch", "list"]
  - apiGroups: ["cns.vmware.com"]
    resources: ["cnsvspherevolumemigrations"]
    verbs: ["create", "get", "list", "watch", "update", "delete"]
  - apiGroups: ["cns.vmware.com"]
    resources: ["cnsvolumeinfoes"]
    verbs: ["create", "get", "list", "watch", "delete"]
  - apiGroups: ["apiextensions.k8s.io"]
    resources: ["customresourcedefinitions"]
    verbs: ["get", "create", "update"]
  - apiGroups: ["storage.k8s.io"]
    resources: ["volumeattachments/status"]
    verbs: ["patch"]
  - apiGroups: ["cns.vmware.com"]
    resources: ["cnsvolumeoperationrequests"]
    verbs: ["create", "get", "list", "update", "delete"]
  - apiGroups: [ "snapshot.storage.k8s.io" ]
    resources: [ "volumesnapshots" ]
    verbs: [ "get", "list" ]
  - apiGroups: [ "snapshot.storage.k8s.io" ]
    resources: [ "volumesnapshotclasses" ]
    verbs: [ "watch", "get", "list" ]
  - apiGroups: [ "snapshot.storage.k8s.io" ]
    resources: [ "volumesnapshotcontents" ]
    verbs: [ "create", "get", "list", "watch", "update", "delete", "patch"]
  - apiGroups: [ "snapshot.storage.k8s.io" ]
    resources: [ "volumesnapshotcontents/status" ]
    verbs: [ "update", "patch" ]
  - apiGroups: [ "cns.vmware.com" ]
    resources: [ "csinodetopologies" ]
    verbs: ["get", "update", "watch", "list"]
---
kind: ClusterRoleBinding
apiVersion: rbac.authorization.k8s.io/v1
metadata:
  name: vsphere-csi-controller-binding
subjects:
  - kind: ServiceAccount
    name: vsphere-csi-controller
    namespace: vmware-system-csi
roleRef:
  kind: ClusterRole
  name: vsphere-csi-controller-role
  apiGroup: rbac.authorization.k8s.io
---
kind: ServiceAccount
apiVersion: v1
metadata:
  name: vsphere-csi-node
  namespace: vmware-system-csi
---
kind: ClusterRole
apiVersion: rbac.authorization.k8s.io/v1
metadata:
  name: vsphere-csi-node-cluster-role
rules:
  - apiGroups: ["cns.vmware.com"]
    resources: ["csinodetopologies"]
    verbs: ["create", "watch", "get", "patch"]
  - apiGroups: [""]
    resources: ["nodes"]
    verbs: ["get"]
---
kind: ClusterRoleBinding
apiVersion: rbac.authorization.k8s.io/v1
metadata:
  name: vsphere-csi-node-cluster-role-binding
subjects:
  - kind: ServiceAccount
    name: vsphere-csi-node
    namespace: vmware-system-csi
roleRef:
  kind: ClusterRole
  name: vsphere-csi-node-cluster-role
  apiGroup: rbac.authorization.k8s.io
---
kind: Role
apiVersion: rbac.authorization.k8s.io/v1
metadata:
  name: vsphere-csi-node-role
  namespace: vmware-system-csi
rules:
  - apiGroups: [""]
    resources: ["configmaps"]
    verbs: ["get", "list", "watch"]
---
kind: RoleBinding
apiVersion: rbac.authorization.k8s.io/v1
metadata:
  name: vsphere-csi-node-binding
  namespace: vmware-system-csi
subjects:
  - kind: ServiceAccount
    name: vsphere-csi-node
    namespace: vmware-system-csi
roleRef:
  kind: Role
  name: vsphere-csi-node-role
  apiGroup: rbac.authorization.k8s.io
---
apiVersion: v1
data:
  "trigger-csi-fullsync": "false"
  "pv-to-backingdiskobjectid-mapping": "false"
  "high-pv-node-density": "false" # When enabled, increases the MAX_VOLUMES_PER_NODE from 59 to 255 for guest cluster nodes
kind: ConfigMap
metadata:
  name: internal-feature-states.csi.vsphere.vmware.com
  namespace: vmware-system-csi
---
apiVersion: v1
kind: Service
metadata:
  name: vsphere-csi-controller
  namespace: vmware-system-csi
  labels:
    app: vsphere-csi-controller
spec:
  ports:
    - name: ctlr
      port: 2112
      targetPort: 2112
      protocol: TCP
    - name: syncer
      port: 2113
      targetPort: 2113
      protocol: TCP
  selector:
    app: vsphere-csi-controller
---
kind: Deployment
apiVersion: apps/v1
metadata:
  name: vsphere-csi-controller
  namespace: vmware-system-csi
spec:
  replicas: 3
  strategy:
    type: RollingUpdate
    rollingUpdate:
      maxUnavailable: 1
      maxSurge: 0
  selector:
    matchLabels:
      app: vsphere-csi-controller
  template:
    metadata:
      labels:
        app: vsphere-csi-controller
        role: vsphere-csi
    spec:
      priorityClassName: system-cluster-critical # Guarantees scheduling for critical system pods
      affinity:
        podAntiAffinity:
          requiredDuringSchedulingIgnoredDuringExecution:
            - labelSelector:
                matchExpressions:
                  - key: "app"
                    operator: In
                    values:
                      - vsphere-csi-controller
              topologyKey: "kubernetes.io/hostname"
        nodeAffinity:
          requiredDuringSchedulingIgnoredDuringExecution:
            nodeSelectorTerms:
            - matchExpressions:
              - key: node-role.kubernetes.io/control-plane
                operator: Exists
            - matchExpressions:
              - key: node-role.kubernetes.io/controlplane
                operator: Exists
            - matchExpressions:
              - key: node-role.kubernetes.io/master
                operator: Exists
      serviceAccountName: vsphere-csi-controller
      tolerations:
        - key: node-role.kubernetes.io/master
          operator: Exists
          effect: NoSchedule
        - key: node-role.kubernetes.io/control-plane
          operator: Exists
          effect: NoSchedule
        # uncomment below toleration if you need an aggressive pod eviction in case when
        # node becomes not-ready or unreachable. Default is 300 seconds if not specified.
        #- key: node.kubernetes.io/not-ready
        #  operator: Exists
        #  effect: NoExecute
        #  tolerationSeconds: 30
        #- key: node.kubernetes.io/unreachable
        #  operator: Exists
        #  effect: NoExecute
        #  tolerationSeconds: 30
      dnsPolicy: "Default"
      containers:
        - name: csi-attacher
          image: registry.k8s.io/sig-storage/csi-attacher:v4.9.0
          args:
            - "--v=4"
            - "--timeout=300s"
            - "--csi-address=$(ADDRESS)"
            - "--leader-election"
            - "--leader-election-lease-duration=120s"
            - "--leader-election-renew-deadline=60s"
            - "--leader-election-retry-period=30s"
            - "--kube-api-qps=100"
            - "--kube-api-burst=100"
            - "--worker-threads=100"
          env:
            - name: ADDRESS
              value: /csi/csi.sock
          volumeMounts:
            - mountPath: /csi
              name: socket-dir
        - name: csi-resizer
          image: registry.k8s.io/sig-storage/csi-resizer:v1.12.0
          args:
            - "--v=4"
            - "--timeout=300s"
            - "--handle-volume-inuse-error=false"
            - "--csi-address=$(ADDRESS)"
            - "--kube-api-qps=100"
            - "--kube-api-burst=100"
            - "--leader-election"
            - "--leader-election-lease-duration=120s"
            - "--leader-election-renew-deadline=60s"
            - "--leader-election-retry-period=30s"
          env:
            - name: ADDRESS
              value: /csi/csi.sock
          volumeMounts:
            - mountPath: /csi
              name: socket-dir
        - name: vsphere-csi-controller
          image: registry.k8s.io/csi-vsphere/driver:v3.7.1
          args:
            - "--fss-name=internal-feature-states.csi.vsphere.vmware.com"
            - "--fss-namespace=$(CSI_NAMESPACE)"
            - "--enable-profile-server=false"
          imagePullPolicy: "Always"
          env:
            - name: CSI_ENDPOINT
              value: unix:///csi/csi.sock
            - name: X_CSI_MODE
              value: "controller"
            - name: X_CSI_SPEC_DISABLE_LEN_CHECK
              value: "true"
            - name: X_CSI_SERIAL_VOL_ACCESS_TIMEOUT
              value: 3m
            - name: VSPHERE_CSI_CONFIG
              value: "/etc/cloud/csi-vsphere.conf"
            - name: LOGGER_LEVEL
              value: "PRODUCTION" # Options: DEVELOPMENT, PRODUCTION
            - name: INCLUSTER_CLIENT_QPS
              value: "100"
            - name: INCLUSTER_CLIENT_BURST
              value: "100"
            - name: CSI_NAMESPACE
              valueFrom:
                fieldRef:
                  fieldPath: metadata.namespace
          securityContext:
            runAsNonRoot: true
            runAsUser: 65532
            runAsGroup: 65532
          volumeMounts:
            - mountPath: /etc/cloud
              name: vsphere-config-volume
              readOnly: true
            - mountPath: /csi
              name: socket-dir
          ports:
            - name: healthz
              containerPort: 9808
              protocol: TCP
            - name: prometheus
              containerPort: 2112
              protocol: TCP
          livenessProbe:
            httpGet:
              path: /healthz
              port: healthz
            initialDelaySeconds: 30
            timeoutSeconds: 10
            periodSeconds: 180
            failureThreshold: 3
        - name: liveness-probe
          image: registry.k8s.io/sig-storage/livenessprobe:v2.15.0
          args:
            - "--v=4"
            - "--csi-address=/csi/csi.sock"
          volumeMounts:
            - name: socket-dir
              mountPath: /csi
        - name: vsphere-syncer
          image: registry.k8s.io/csi-vsphere/syncer:v3.7.1
          args:
            - "--leader-election"
            - "--leader-election-lease-duration=30s"
            - "--leader-election-renew-deadline=20s"
            - "--leader-election-retry-period=10s"
            - "--fss-name=internal-feature-states.csi.vsphere.vmware.com"
            - "--fss-namespace=$(CSI_NAMESPACE)"
            - "--enable-profile-server=false"
          imagePullPolicy: "Always"
          ports:
            - containerPort: 2113
              name: prometheus
              protocol: TCP
          env:
            - name: FULL_SYNC_INTERVAL_MINUTES
              value: "30"
            - name: VSPHERE_CSI_CONFIG
              value: "/etc/cloud/csi-vsphere.conf"
            - name: LOGGER_LEVEL
              value: "PRODUCTION" # Options: DEVELOPMENT, PRODUCTION
            - name: INCLUSTER_CLIENT_QPS
              value: "100"
            - name: INCLUSTER_CLIENT_BURST
              value: "100"
            - name: CSI_NAMESPACE
              valueFrom:
                fieldRef:
                  fieldPath: metadata.namespace
          securityContext:
            runAsNonRoot: true
            runAsUser: 65532
            runAsGroup: 65532
          volumeMounts:
            - mountPath: /etc/cloud
              name: vsphere-config-volume
              readOnly: true
        - name: csi-provisioner
          image: registry.k8s.io/sig-storage/csi-provisioner:v4.0.1
          args:
            - "--v=4"
            - "--timeout=300s"
            - "--csi-address=$(ADDRESS)"
            - "--kube-api-qps=100"
            - "--kube-api-burst=100"
            - "--leader-election"
            - "--leader-election-lease-duration=120s"
            - "--leader-election-renew-deadline=60s"
            - "--leader-election-retry-period=30s"
            - "--default-fstype=ext4"
            # needed only for topology aware setup
            #- "--feature-gates=Topology=true"
            #- "--strict-topology"
          env:
            - name: ADDRESS
              value: /csi/csi.sock
          volumeMounts:
            - mountPath: /csi
              name: socket-dir
        - name: csi-snapshotter
          image: registry.k8s.io/sig-storage/csi-snapshotter:v8.2.0
          args:
            - "--v=4"
            - "--kube-api-qps=100"
            - "--kube-api-burst=100"
            - "--timeout=300s"
            - "--csi-address=$(ADDRESS)"
            - "--leader-election"
            - "--leader-election-lease-duration=120s"
            - "--leader-election-renew-deadline=60s"
            - "--leader-election-retry-period=30s"
            - "--extra-create-metadata"
          env:
            - name: ADDRESS
              value: /csi/csi.sock
          volumeMounts:
            - mountPath: /csi
              name: socket-dir
      volumes:
        - name: vsphere-config-volume
          secret:
            secretName: vsphere-config-secret
        - name: socket-dir
          emptyDir: {}
---
kind: DaemonSet
apiVersion: apps/v1
metadata:
  name: vsphere-csi-node
  namespace: vmware-system-csi
spec:
  selector:
    matchLabels:
      app: vsphere-csi-node
  updateStrategy:
    type: "RollingUpdate"
    rollingUpdate:
      maxUnavailable: 1
  template:
    metadata:
      labels:
        app: vsphere-csi-node
        role: vsphere-csi
    spec:
      priorityClassName: system-node-critical
      nodeSelector:
        kubernetes.io/os: linux
      serviceAccountName: vsphere-csi-node
      hostNetwork: true
      dnsPolicy: "ClusterFirstWithHostNet"
      containers:
        - name: node-driver-registrar
          image: registry.k8s.io/sig-storage/csi-node-driver-registrar:v2.13.0
          args:
            - "--v=5"
            - "--csi-address=$(ADDRESS)"
            - "--kubelet-registration-path=$(DRIVER_REG_SOCK_PATH)"
          env:
            - name: ADDRESS
              value: /csi/csi.sock
            - name: DRIVER_REG_SOCK_PATH
              value: /var/lib/kubelet/plugins/csi.vsphere.vmware.com/csi.sock
          volumeMounts:
            - name: plugin-dir
              mountPath: /csi
            - name: registration-dir
              mountPath: /registration
          livenessProbe:
            exec:
              command:
              - /csi-node-driver-registrar
              - --kubelet-registration-path=/var/lib/kubelet/plugins/csi.vsphere.vmware.com/csi.sock
              - --mode=kubelet-registration-probe
            initialDelaySeconds: 3
        - name: vsphere-csi-node
          image: registry.k8s.io/csi-vsphere/driver:v3.7.1
          args:
            - "--fss-name=internal-feature-states.csi.vsphere.vmware.com"
            - "--fss-namespace=$(CSI_NAMESPACE)"
          imagePullPolicy: "Always"
          env:
            - name: NODE_NAME
              valueFrom:
                fieldRef:
                  fieldPath: spec.nodeName
            - name: CSI_ENDPOINT
              value: unix:///csi/csi.sock
            - name: X_CSI_MODE
              value: "node"
            - name: X_CSI_SPEC_REQ_VALIDATION
              value: "false"
            - name: X_CSI_SPEC_DISABLE_LEN_CHECK
              value: "true"
            - name: LOGGER_LEVEL
              value: "PRODUCTION" # Options: DEVELOPMENT, PRODUCTION
            - name: CSI_NAMESPACE
              valueFrom:
                fieldRef:
                  fieldPath: metadata.namespace
            - name: NODEGETINFO_WATCH_TIMEOUT_MINUTES
              value: "1"
          securityContext:
            privileged: true
            capabilities:
              add: ["SYS_ADMIN"]
            allowPrivilegeEscalation: true
          volumeMounts:
            - name: plugin-dir
              mountPath: /csi
            - name: pods-mount-dir
              mountPath: /var/lib/kubelet
              # needed so that any mounts setup inside this container are
              # propagated back to the host machine.
              mountPropagation: "Bidirectional"
            - name: device-dir
              mountPath: /dev
            - name: blocks-dir
              mountPath: /sys/block
            - name: sys-devices-dir
              mountPath: /sys/devices
          ports:
            - name: healthz
              containerPort: 9808
              protocol: TCP
          livenessProbe:
            httpGet:
              path: /healthz
              port: healthz
            initialDelaySeconds: 10
            timeoutSeconds: 5
            periodSeconds: 5
            failureThreshold: 3
        - name: liveness-probe
          image: registry.k8s.io/sig-storage/livenessprobe:v2.15.0
          args:
            - "--v=4"
            - "--csi-address=/csi/csi.sock"
          volumeMounts:
            - name: plugin-dir
              mountPath: /csi
      volumes:
        - name: registration-dir
          hostPath:
            path: /var/lib/kubelet/plugins_registry
            type: Directory
        - name: plugin-dir
          hostPath:
            path: /var/lib/kubelet/plugins/csi.vsphere.vmware.com
            type: DirectoryOrCreate
        - name: pods-mount-dir
          hostPath:
            path: /var/lib/kubelet
            type: Directory
        - name: device-dir
          hostPath:
            path: /dev
        - name: blocks-dir
          hostPath:
            path: /sys/block
            type: Directory
        - name: sys-devices-dir
          hostPath:
            path: /sys/devices
            type: Directory
      tolerations:
        - effect: NoExecute
          operator: Exists
        - effect: NoSchedule
          operator: Exists
---
kind: DaemonSet
apiVersion: apps/v1
metadata:
  name: vsphere-csi-node-windows
  namespace: vmware-system-csi
spec:
  selector:
    matchLabels:
      app: vsphere-csi-node-windows
  updateStrategy:
    type: RollingUpdate
    rollingUpdate:
      maxUnavailable: 1
  template:
    metadata:
      labels:
        app: vsphere-csi-node-windows
        role: vsphere-csi-windows
    spec:
      priorityClassName: system-node-critical
      nodeSelector:
        kubernetes.io/os: windows
      dnsPolicy: "ClusterFirstWithHostNet"
      serviceAccountName: vsphere-csi-node
      securityContext:
        windowsOptions:
          hostProcess: true
          runAsUserName: "NT AUTHORITY\\SYSTEM"
      hostNetwork: true
      containers:
        - name: node-driver-registrar
          image: registry.k8s.io/sig-storage/csi-node-driver-registrar:v2.13.0
          command:
            - "csi-node-driver-registrar.exe"
          args:
            - "--v=5"
            - "--csi-address=$(ADDRESS)"
            - "--kubelet-registration-path=$(DRIVER_REG_SOCK_PATH)"
            - "--plugin-registration-path=$(PLUGIN_REG_DIR)"
          env:
            - name: ADDRESS
              value: unix://C:\\var\\lib\\kubelet\\plugins\\csi.vsphere.vmware.com\\csi.sock
            - name: DRIVER_REG_SOCK_PATH
              value: 'C:\\var\\lib\\kubelet\\plugins\\csi.vsphere.vmware.com\\csi.sock'
            - name: PLUGIN_REG_DIR
              value: C:\\var\\lib\\kubelet\\plugins_registry\\
          volumeMounts:
            - name: plugin-dir
              mountPath: /csi
            - name: registration-dir
              mountPath: /registration
        - name: vsphere-csi-node
          image: registry.k8s.io/csi-vsphere/driver:v3.7.1
          command:
            - "csi.exe"
          args:
            - "--fss-name=internal-feature-states.csi.vsphere.vmware.com"
            - "--fss-namespace=$(CSI_NAMESPACE)"
          imagePullPolicy: "Always"
          env:
            - name: NODE_NAME
              valueFrom:
                fieldRef:
                  apiVersion: v1
                  fieldPath: spec.nodeName
            - name: CSI_ENDPOINT
              value: unix://C:\\var\\lib\\kubelet\\plugins\\csi.vsphere.vmware.com\\csi.sock
            - name: X_CSI_MODE
              value: node
            - name: X_CSI_SPEC_REQ_VALIDATION
              value: 'false'
            - name: X_CSI_SPEC_DISABLE_LEN_CHECK
              value: "true"
            - name: LOGGER_LEVEL
              value: "PRODUCTION" # Options: DEVELOPMENT, PRODUCTION
            - name: X_CSI_LOG_LEVEL
              value: DEBUG
            - name: CSI_NAMESPACE
              valueFrom:
                fieldRef:
                  fieldPath: metadata.namespace
            - name: NODEGETINFO_WATCH_TIMEOUT_MINUTES
              value: "1"
          volumeMounts:
            - name: plugin-dir
              mountPath: 'C:\csi'
            - name: pods-mount-dir
              mountPath: 'C:\var\lib\kubelet'
          ports:
            - name: healthz
              containerPort: 9808
              protocol: TCP
          livenessProbe:
            httpGet:
              path: /healthz
              port: healthz
            initialDelaySeconds: 10
            timeoutSeconds: 5
            periodSeconds: 5
            failureThreshold: 3
        - name: liveness-probe
          image: registry.k8s.io/sig-storage/livenessprobe:v2.15.0
          command:
            - "livenessprobe.exe"
          args:
            - "--v=4"
            - --csi-address=unix://C:\\var\\lib\\kubelet\\plugins\\csi.vsphere.vmware.com\\csi.sock
          volumeMounts:
            - name: plugin-dir
              mountPath: /csi
      volumes:
        - name: registration-dir
          hostPath:
            path: 'C:\var\lib\kubelet\plugins_registry\'
            type: Directory
        - name: plugin-dir
          hostPath:
            path: 'C:\var\lib\kubelet\plugins\csi.vsphere.vmware.com\'
            type: DirectoryOrCreate
        - name: pods-mount-dir
          hostPath:
            path: \var\lib\kubelet
            type: Directory
      tolerations:
        - effect: NoExecute
          operator: Exists
        - effect: NoSchedule
          operator: Exists