# We may need to label nodes with clickhouse=allow label for this example to run
# See ./label_nodes.sh for this purpose

apiVersion: "clickhouse.altinity.com/v1"
kind: "ClickHouseInstallation"
metadata:
  name: "clickhouse-installation-max"
  labels:
    label1: label1_value
    label2: label2_value
  annotations:
    annotation1: annotation1_value
    annotation2: annotation2_value

spec:
  # Allows to define custom taskID for CHI update and watch status of this update execution.
  # Displayed in all .status.taskID* fields.
  # By default (if not filled) every update of CHI manifest will generate random taskID
  taskID: "qweqwe"

  # Allows to stop all ClickHouse clusters defined in a CHI.
  # Works as the following:
  #  - When `stop` is `1` operator sets `Replicas: 0` in each StatefulSet. Thie leads to having all `Pods` and `Service` deleted. All PVCs are kept intact.
  #  - When `stop` is `0` operator sets `Replicas: 1` and `Pod`s and `Service`s will created again and all retained PVCs will be attached to `Pod`s.
  stop: "no"

  # In case 'RollingUpdate' specified, the operator will always restart ClickHouse pods during reconcile.
  # This options is used in rare cases when force restart is required and is typically removed after the use in order to avoid unneeded restarts.
  restart: "RollingUpdate"

  # Allows to troubleshoot Pods during CrashLoopBack state.
  # This may happen when wrong configuration applied, in this case `clickhouse-server` wouldn't start.
  # Command within ClickHouse container is modified with `sleep` in order to avoid quick restarts
  # and give time to troubleshoot via CLI.
  # Liveness and Readiness probes are disabled as well.
  troubleshoot: "no"

  # Custom domain pattern which will be used for DNS names of `Service` or `Pod`.
  # Typical use scenario - custom cluster domain in Kubernetes cluster
  namespaceDomainPattern:  "%s.svc.my.test"

  # Optional, applicable inside ClickHouseInstallationTemplate only.
  # Defines current ClickHouseInstallationTemplate application policy to target ClickHouseInstallation(s)."
  templating:
    # When defined as `auto` inside ClickhouseInstallationTemplate, this ClickhouseInstallationTemplate
    # will be auto-added into ClickHouseInstallation, selectable by `chiSelector`.
    # Default value is `manual`, meaning ClickHouseInstallation should request this ClickhouseInstallationTemplate explicitly.
    # Possible values:
    #  - "auto"
    #  - "manual"
    policy: "manual"
    # Optional, defines selector for ClickHouseInstallation(s) to be templated with ClickhouseInstallationTemplate
    chiSelector:
      name1: value1
      name2: value2
    # TODO introduce one-time actions
    # chiTaskID: "qweqwe"
    # autoPurge: "yes"

  # Optional, allows tuning reconciling cycle for ClickhouseInstallation from clickhouse-operator side
  reconcile:
    # DISCUSSED TO BE DEPRECATED
    # Syntax sugar
    # Overrides all three 'reconcile.host.wait.{exclude, queries, include}' values from the operator's config
    # Possible values:
    #  - wait - should wait to exclude host, complete queries and include host back into the cluster
    #  - nowait - should NOT wait to exclude host, complete queries and include host back into the cluster
    policy: "nowait"

    # Timeout in seconds for `clickhouse-operator` to wait for modified `ConfigMap` to propagate into the `Pod`
    # More details: https://kubernetes.io/docs/concepts/configuration/configmap/#mounted-configmaps-are-updated-automatically
    configMapPropagationTimeout: 90

    # Optional, defines behavior for cleanup Kubernetes resources during reconcile cycle
    cleanup:
      # Describes what clickhouse-operator should do with found Kubernetes resources which should be managed by clickhouse-operator,
      # but do not have `ownerReference` to any currently managed `ClickHouseInstallation` resource.
      # Default behavior is `Delete`"
      unknownObjects:
        # Behavior policy for unknown PVC, `Delete` by default
        statefulSet: Delete
        # Behavior policy for unknown PVC, `Delete` by default
        pvc: Delete
        # Behavior policy for unknown ConfigMap, `Delete` by default
        configMap: Delete
        # Behavior policy for unknown Service, `Delete` by default
        service: Delete
      # Describes what clickhouse-operator should do with Kubernetes resources which are failed during reconcile.
      # Default behavior is `Retain`"
      reconcileFailedObjects:
        # Behavior policy for failed StatefulSet, `Retain` by default
        statefulSet: Retain
        # Behavior policy for failed PVC, `Retain` by default
        pvc: Retain
        # Behavior policy for failed ConfigMap, `Retain` by default
        configMap: Retain
        # Behavior policy for failed Service, `Retain` by default
        service: Retain

    # Optional, overwrites reconcile.runtime from the operator's config
    runtime:
      # Max number of concurrent shard reconciles within one cluster in progress
      reconcileShardsThreadsNumber: 5
      # Max percentage of concurrent shard reconciles within one cluster in progress
      reconcileShardsMaxConcurrencyPercent: 50

    # Optional, overwrites reconcile.statefulSet from the operator's config
    # Reconcile StatefulSet scenario
    statefulSet:
      # Create StatefulSet scenario
      create:
        # What to do in case created StatefulSet is not in 'Ready' after `reconcile.statefulSet.update.timeout` seconds
        # Possible options:
        # 1. abort - abort the process, do nothing with the problematic StatefulSet, leave it as it is,
        #    do not try to fix or delete or update it, just abort reconcile cycle.
        #    Do not proceed to the next StatefulSet(s) and wait for an admin to assist.
        # 2. delete - delete newly created problematic StatefulSet and follow 'abort' path afterwards.
        # 3. ignore - ignore an error, pretend nothing happened, continue reconcile and move on to the next StatefulSet.
        onFailure: ignore

      # Update StatefulSet scenario
      update:
        # How many seconds to wait for created/updated StatefulSet to be 'Ready'
        timeout: 300
        # How many seconds to wait between checks/polls for created/updated StatefulSet status
        pollInterval: 5
        # What to do in case updated StatefulSet is not in 'Ready' after `reconcile.statefulSet.update.timeout` seconds
        # Possible options:
        # 1. abort - abort the process, do nothing with the problematic StatefulSet, leave it as it is,
        #    do not try to fix or delete or update it, just abort reconcile cycle.
        #    Do not proceed to the next StatefulSet(s) and wait for an admin to assist.
        # 2. rollback - delete Pod and rollback StatefulSet to previous Generation.
        #    Pod would be recreated by StatefulSet based on rollback-ed StatefulSet configuration.
        #    Follow 'abort' path afterwards.
        # 3. ignore - ignore an error, pretend nothing happened, continue reconcile and move on to the next StatefulSet.
        onFailure: abort

      # Recreate StatefulSet scenario
      recreate:
        # What to do in case operator is in need to recreate StatefulSet?
        # Possible options:
        # 1. abort - abort the process, do nothing with the problematic StatefulSet, leave it as it is,
        #    do not try to fix or delete or update it, just abort reconcile cycle.
        #    Do not proceed to the next StatefulSet(s) and wait for an admin to assist.
        # 2. recreate - proceed and recreate StatefulSet.

        # Triggered when PVC data loss or missing volumes are detected
        onDataLoss: recreate
        # Triggered when StatefulSet update fails or StatefulSet is not ready
        onUpdateFailure: recreate

    # Optional, overwrites reconcile.host from the operator's config
    # Reconcile Host scenario
    host:
      # Whether the operator during reconcile procedure should wait for a ClickHouse host:
      #   - to be excluded from a ClickHouse cluster
      #   - to complete all running queries
      #   - to be included into a ClickHouse cluster
      # respectfully before moving forward with host reconcile
      wait:
        exclude: true
        queries: true
        include: false
        # The operator during reconcile procedure should wait for replicas to catch-up
        # replication delay a.k.a replication lag for the following replicas
        replicas:
          # All replicas (new and known earlier) are explicitly requested to wait for replication to catch-up
          all: no
          # New replicas only are requested to wait for replication to catch-up
          new: yes
          # Replication catch-up is considered to be completed as soon as replication delay
          # a.k.a replication lag - calculated as "MAX(absolute_delay) FROM system.replicas"
          # is within this specified delay (in seconds)
          delay: 10
        probes:
          # Whether the operator during host launch procedure should wait for startup probe to succeed.
          # In case probe is unspecified wait is assumed to be completed successfully.
          # Default option value is to do not wait.
          startup: no
          # Whether the operator during host launch procedure should wait for readiness probe to succeed.
          # In case probe is unspecified wait is assumed to be completed successfully.
          # Default option value is to wait.
          readiness: yes
      # The operator during reconcile procedure should drop the following entities:
      drop:
        replicas:
          # Whether the operator during reconcile procedure should drop replicas when replica is deleted
          onDelete: yes
          # Whether the operator during reconcile procedure should drop replicas when replica volume is lost
          onLostVolume: yes
          # Whether the operator during reconcile procedure should drop active replicas when replica is deleted or recreated
          active: no
      # Optional, hooks to execute before/after each host reconcile.
      # Hooks defined at CHI level are inherited by all clusters.
      # Hooks are skipped on first CHI creation (no live hosts yet).
      hooks:
        pre:
          - sql:
              queries:
                - "SYSTEM STOP REPLICATION QUEUES"
            # `events:` is REQUIRED and non-empty. It lists the reconcile lifecycle events
            # at which this hook fires; the hook is skipped on any reconcile whose
            # classifier does not emit at least one of the listed events.
            #
            # Events are grouped into TWO SCOPES that match the YAML hook nesting:
            # host-scope events for host-level hooks (this block, under
            # spec.reconcile.host.hooks) and cluster-scope events for cluster-level
            # hooks (under spec.configuration.clusters[N].reconcile.hooks). The two
            # scopes are independent — a cluster hook listing a Host* event is
            # rejected at apply time. `Always` is the only value valid in both scopes.
            #
            # Host-scope events (THIS hook is host-level, so these apply):
            #
            #   Always            - fires on every hook-evaluation point, including
            #                       the pre-delete sweep on the dying host
            #   HostCreate        - first reconcile that creates a host (no ancestor);
            #                       best paired with POST hooks (PRE on first creation
            #                       is silently inert because the pod is not up yet)
            #   HostUpdate        - reconcile that has prior state for the host (i.e.
            #                       neither create nor delete); the catch-all for
            #                       ongoing reconciles
            #   HostStart         - host transitions from stopped to running
            #   HostStop          - host is being stopped (current spec marks it stopped)
            #   HostConfigRestart - operator decided this host needs an in-place
            #                       software restart for a config change to take effect
            #   HostRollout       - pod-template change forces a StatefulSet rollout
            #                       (new env var, new image, new volume, ...)
            #   HostShutdown      - aggregate: fires whenever the pod is going to be
            #                       brought down for any reason (Stop, ConfigRestart,
            #                       Rollout, or Delete)
            #   HostDelete        - host is being removed from the cluster (downsize);
            #                       fires on the dying host before tear-down. Always
            #                       emitted alongside HostShutdown.
            #
            # Cluster-scope events (used by cluster-level hooks under
            # configuration.clusters[].reconcile.hooks — see further below):
            #
            #   Always           - every cluster reconcile (incl. cluster delete sweep)
            #   ClusterCreate    - cluster's first-time reconcile (ALL hosts new — no
            #                      ancestor on any of them)
            #   ClusterReconcile - ongoing reconcile pass over an existing cluster (at
            #                      least one host has prior state). Fires on every
            #                      operator reconcile cycle, including taskID-only
            #                      force-reconciles. NOT a "spec changed" signal.
            #   ClusterDelete    - cluster being removed (delete sweep — wiring follow-up)
            #
            # HostShutdown below is the right choice for "graceful drain before any
            # restart-class event" patterns (Alex's swarm-leave use case).
            events:
              - HostShutdown
        post:
          - sql:
              queries:
                - "SYSTEM START REPLICATION QUEUES"
            events:
              - HostUpdate
    # Optional, CHI-level CLUSTER hooks. Inherited by every cluster's
    # spec.configuration.clusters[N].reconcile.hooks and merged with any per-cluster
    # additions via dedup'd MergeFrom (so defining a hook here PLUS the same hook
    # per cluster does not double-fire).
    #
    # Cluster scope is independent of host scope above:
    #   - host hooks fire per host, around each host reconcile
    #   - cluster hooks fire once per cluster reconcile, around the host iteration
    # Cluster hooks accept ONLY cluster-scope events (Always, ClusterCreate,
    # ClusterReconcile, ClusterDelete) — Host* events are rejected at apply time.
    #
    # Use this top-level form when the same cluster-scope hook should apply to
    # every cluster of the CHI. Override per cluster only when you genuinely need
    # cluster-specific behavior (those hooks append to the inherited set).
    #
    # `target` (optional, default: FirstHost) controls which host(s) execute the SQL:
    #   FirstHost — one host only; correct when the SQL itself fans out via
    #               `ON CLUSTER '{cluster}'` (ClickHouse propagates the DDL).
    #   AllHosts  — every host individually; required for SQL that does NOT
    #               support ON CLUSTER (e.g. SYSTEM FLUSH LOGS).
    #   AllShards — first replica of each shard.
    # Both patterns are demonstrated below.
    cluster:
      hooks:
        pre:
          # ON CLUSTER DDL → target defaults to FirstHost (omitted)
          - sql:
              queries:
                - "DETACH TABLE IF EXISTS db.kafka_table ON CLUSTER '{cluster}' PERMANENTLY"
            events:
              - ClusterReconcile
        post:
          # ON CLUSTER DDL → target defaults to FirstHost (omitted)
          - sql:
              queries:
                - "ATTACH TABLE db.kafka_table ON CLUSTER '{cluster}'"
            events:
              - ClusterReconcile
          # SYSTEM FLUSH LOGS does NOT support ON CLUSTER — fan out via target: AllHosts
          - sql:
              queries:
                - "SYSTEM FLUSH LOGS"
            target: AllHosts
            events:
              - ClusterReconcile

    macros:
      sections:
        users:
          enabled: "yes"
        profiles:
          enabled: "yes"
        quotas:
          enabled: "yes"
        settings:
          enabled: "yes"
        files:
          enabled: "yes"


  # List of templates used by a CHI
  useTemplates:
    - name: template1
      namespace: ns1
      useType: merge
    - name: template2
      # No namespace specified - use CHI namespace

  # CHI-level security defaults. Inherited by every cluster that does not override.
  # Operator-wide defaults live in ClickHouseOperatorConfiguration (chopconf) and
  # cascade chopconf → CHI spec → cluster spec; lower level wins.
  # All fields default to nil/empty so an upgrade preserves identical behavior.
  security:
    clickhouse:
      # Outbound TLS knobs for the operator's ClickHouse client.
      tls:
        # Strict | None | "" (unset: legacy InsecureSkipVerify=true preserved).
        verify: ""
        # "1.2" | "1.3" | "" (Go stdlib default).
        minVersion: ""
        # SNI / cert-verify name override. Empty derives from dial host.
        serverName: ""
        # Inline PEM CA bundle. Mutually exclusive with rootCASecretRef below.
        rootCA: ""
        # Alternate source: read PEM CA from a Kubernetes Secret in this CHI's
        # namespace. When Key is empty, the operator tries "ca.crt" then "tls.crt".
        rootCASecretRef:
          name: ""
          key: ""
    zookeeper:
      # Outbound TLS knobs for the operator's ZooKeeper / Keeper client.
      tls:
        verify: ""
        minVersion: ""
    # Note: Kubernetes-API-client TLS is operator-process-scoped — there is no
    # per-CHI override. Configure it in ClickHouseOperatorConfiguration only.

  defaults:
    replicasUseFQDN: "no"
    distributedDDL:
      profile: default
    storageManagement:
      # Specify PVC provisioner.
      # 1. StatefulSet. PVC would be provisioned by the StatefulSet
      # 2. Operator. PVC would be provisioned by the operator
      provisioner: StatefulSet

      # Specify PVC reclaim policy.
      # 1. Retain. Keep PVC from being deleted
      #    Retaining PVC will also keep backing PV from deletion. This is useful in case we need to keep data intact.
      # 2. Delete
      reclaimPolicy: Retain

    templates:
      hostTemplate: host-template-custom-ports
      podTemplate: clickhouse-v23.8
      dataVolumeClaimTemplate: default-volume-claim
      logVolumeClaimTemplate: default-volume-claim
      serviceTemplate: chi-service-template
      clusterServiceTemplate: cluster-service-template
      shardServiceTemplate: shard-service-template
      replicaServiceTemplate: replica-service-template

  configuration:
    zookeeper:
      # Option 1: Reference a ClickHouseKeeperInstallation (CHK) by name.
      # The operator resolves this to ZooKeeper node addresses automatically.
      # keeper:
      #   name: my-keeper
      #   namespace: my-namespace    # optional, defaults to CHI namespace
      #   serviceType: replicas      # "replicas" (default) or "service"

      # Option 2: Specify ZooKeeper nodes explicitly.
      nodes:
        - host: zookeeper-0.zookeepers.zoo3ns.svc.cluster.local
          port: 2181
          secure: yes
          availabilityZone: az1
        - host: zookeeper-1.zookeepers.zoo3ns.svc.cluster.local
          port: 2181
          secure: yes
          availabilityZone: az2
        - host: zookeeper-2.zookeepers.zoo3ns.svc.cluster.local
          port: 2181
          secure: yes
          availabilityZone: az3
      session_timeout_ms: 30000
      operation_timeout_ms: 10000
      root: "/path/to/zookeeper/root/node"
      identity: "user:password"
    users:
      readonly/profile: readonly
      #     <users>
      #        <readonly>
      #          <profile>readonly</profile>
      #        </readonly>
      #     </users>
      test/networks/ip:
        - "127.0.0.1"
        - "::/0"
      #     <users>
      #        <test>
      #          <networks>
      #            <ip>127.0.0.1</ip>
      #            <ip>::/0</ip>
      #          </networks>
      #        </test>
      #     </users>
      test/profile: default
      test/quotas: default
    profiles:
      readonly/readonly: "1"
      #      <profiles>
      #        <readonly>
      #          <readonly>1</readonly>
      #        </readonly>
      #      </profiles>
      default/max_memory_usage: "1000000000"
    quotas:
      default/interval/duration: "3600"
      #     <quotas>
      #       <default>
      #         <interval>
      #           <duration>3600</duration>
      #         </interval>
      #       </default>
      #     </quotas>
    settings:
      compression/case/method: zstd
      #      <compression>
      #        <case>
      #          <method>zstd</method>
      #        </case>
      #      </compression>
      disable_internal_dns_cache: 1
      #      <disable_internal_dns_cache>1</disable_internal_dns_cache>
    files:
      dict1.xml: |
        <yandex>
            <!-- ref to file /etc/clickhouse-data/config.d/source1.csv -->
        </yandex>
      source1.csv: |
        a1,b1,c1,d1
        a2,b2,c2,d2

    clusters:

      - name: all-counts
        templates:
          podTemplate: clickhouse-v23.8
          dataVolumeClaimTemplate: default-volume-claim
          logVolumeClaimTemplate: default-volume-claim
        schemaPolicy:
          replica: All
          shard: All
        layout:
          shardsCount: 3
          replicasCount: 2

        # Cluster-level security knobs. Same shape as spec.security above; values
        # here win over CHI-level and chopconf-level defaults (fill-empty merge).
        security:
          clickhouse:
            tls:
              # Strict | None | "" (inherit / legacy InsecureSkipVerify=true).
              verify: ""
              # "1.2" | "1.3" | "" (Go stdlib default).
              minVersion: ""
              # SNI / cert-verify name override. Empty derives from dial host.
              serverName: ""
              # Inline PEM CA bundle. Mutually exclusive with rootCASecretRef below.
              rootCA: ""
              # Alternate source: read PEM CA from a Kubernetes Secret in the CHI's
              # namespace. When Key is empty, the operator tries "ca.crt" then "tls.crt".
              rootCASecretRef:
                name: ""
                key: ""
          zookeeper:
            tls:
              verify: ""
              minVersion: ""
          # Kubernetes-API-client TLS is operator-process-scoped — no per-cluster
          # override exists. Configure it in ClickHouseOperatorConfiguration only.
        reconcile:
          # Optional, overwrites reconcile.runtime from the operator's config
          runtime:
            # Max number of concurrent shard reconciles within one cluster in progress
            reconcileShardsThreadsNumber: 5
            # Max percentage of concurrent shard reconciles within one cluster in progress
            reconcileShardsMaxConcurrencyPercent: 50

          # Optional, cluster-level hooks to execute before/after cluster reconcile.
          # Cluster hooks run ONCE per cluster (not per host).
          # For cluster hooks, `target` controls where SQL executes:
          #   - FirstHost (default): one host, suitable for ON CLUSTER queries
          #   - AllHosts:            every host in the cluster
          #   - AllShards:           first replica of each shard
          # Hooks are skipped on first cluster creation (no live hosts yet).
          hooks:
            # Cluster-level hooks listen to CLUSTER scope events only:
            #   Always           - every cluster reconcile (incl. cluster delete sweep)
            #   ClusterCreate    - cluster's first-time reconcile (ALL hosts new)
            #   ClusterReconcile - ongoing reconcile pass; fires on every operator
            #                      reconcile cycle, NOT only on spec changes
            #   ClusterDelete    - cluster being removed (delete sweep)
            # Host* events are NOT valid here — they belong to host-level hooks
            # (spec.reconcile.host.hooks above), which are evaluated per host.
            pre:
              - sql:
                  queries:
                    - "DETACH TABLE IF EXISTS db.kafka_table ON CLUSTER '{cluster}' PERMANENTLY"
                events:
                  - ClusterReconcile
            post:
              - sql:
                  queries:
                    - "ATTACH TABLE db.kafka_table ON CLUSTER '{cluster}'"
                events:
                  - ClusterReconcile
              - sql:
                  queries:
                    - "SYSTEM FLUSH LOGS"
                # target and events are HookAction fields (siblings of sql:), not SQLHookAction
                # fields — they must be at the same indent level as `sql:` above.
                target: AllHosts
                events:
                  - ClusterReconcile

          host:
            # Whether the operator during reconcile procedure should wait for a ClickHouse host:
            #   - to be excluded from a ClickHouse cluster
            #   - to complete all running queries
            #   - to be included into a ClickHouse cluster
            # respectfully before moving forward with host reconcile
            wait:
              exclude: true
              queries: true
              include: false
              # The operator during reconcile procedure should wait for replicas to catch-up
              # replication delay a.k.a replication lag for the following replicas
              replicas:
                # All replicas (new and known earlier) are explicitly requested to wait for replication to catch-up
                all: no
                # New replicas only are requested to wait for replication to catch-up
                new: yes
                # Replication catch-up is considered to be completed as soon as replication delay
                # a.k.a replication lag - calculated as "MAX(absolute_delay) FROM system.replicas"
                # is within this specified delay (in seconds)
                delay: 10
              probes:
                # Whether the operator during host launch procedure should wait for startup probe to succeed.
                # In case probe is unspecified wait is assumed to be completed successfully.
                # Default option value is to do not wait.
                startup: no
                # Whether the operator during host launch procedure should wait for readiness probe to succeed.
                # In case probe is unspecified wait is assumed to be completed successfully.
                # Default option value is to wait.
                readiness: yes

      - name: shards-only
        templates:
          podTemplate: clickhouse-v23.8
          dataVolumeClaimTemplate: default-volume-claim
          logVolumeClaimTemplate: default-volume-claim
        # Cluster-level keeper reference overrides top-level zookeeper config for this cluster.
        # If a cluster has its own zookeeper config, the top-level config is ignored for this cluster.
        zookeeper:
          keeper:
            name: cluster-specific-keeper
            namespace: keeper-namespace
            serviceType: service
        layout:
          shardsCount: 3
          # replicasCount not specified, assumed = 1, by default

      - name: replicas-only
        templates:
          podTemplate: clickhouse-v23.8
          dataVolumeClaimTemplate: default-volume-claim
          logVolumeClaimTemplate: default-volume-claim
        layout:
          # shardsCount not specified, assumed = 1, by default
          replicasCount: 3

      - name: customized
        templates:
          podTemplate: clickhouse-v23.8
          dataVolumeClaimTemplate: default-volume-claim
          logVolumeClaimTemplate: default-volume-claim
        schemaPolicy:
          replica: None
          shard: None
        layout:
          shards:
            - name: shard0
              replicasCount: 3
              weight: 1
              internalReplication: Disabled
              templates:
                podTemplate: clickhouse-v23.8
                dataVolumeClaimTemplate: default-volume-claim
                logVolumeClaimTemplate: default-volume-claim

            - name: shard1
              templates:
                podTemplate: clickhouse-v23.8
                dataVolumeClaimTemplate: default-volume-claim
                logVolumeClaimTemplate: default-volume-claim
              replicas:
                - name: replica0
                - name: replica1
                - name: replica2

            - name: shard2
              replicasCount: 3
              templates:
                podTemplate: clickhouse-v23.8
                dataVolumeClaimTemplate: default-volume-claim
                logVolumeClaimTemplate: default-volume-claim
                replicaServiceTemplate: replica-service-template
              replicas:
                - name: replica0
                  tcpPort: 9000
                  httpPort: 8123
                  interserverHTTPPort: 9009
                  templates:
                    podTemplate: clickhouse-v23.8
                    dataVolumeClaimTemplate: default-volume-claim
                    logVolumeClaimTemplate: default-volume-claim
                    replicaServiceTemplate: replica-service-template

      - name: with-secret
        # Insecure communication.
        # Opens/Closes insecure ports
        insecure: "yes"
        # Secure communication.
        # Opens/Closes secure ports
        # Translates into <secure>1</secure> ClickHouse setting for remote replicas
        secure: "yes"
        # Shared secret value to secure cluster communications
        secret:
          # Auto-generate shared secret value to secure cluster communications
          auto: "True"
          # Cluster shared secret value in plain text
          value: "plaintext secret"
          # Cluster shared secret source
          valueFrom:
            secretKeyRef:
              name: "SecretName"
              key: "Key"
        layout:
          shardsCount: 2

  templates:
    hostTemplates:
      - name: host-template-custom-ports
        portDistribution:
          - type: "ClusterScopeIndex"
        spec:
          # Insecure communication.
          # Opens/Closes insecure ports
          insecure: "yes"
          # Secure communication.
          # Opens/Closes secure ports
          # Translates into <secure>1</secure> ClickHouse setting for remote replicas
          secure: "no"
          tcpPort: 7000
          httpPort: 7001
          interserverHTTPPort: 7002
          #settings:
          #files:
          #templates:

    serviceTemplates:
      - name: chi-service-template
        # generateName understands different sets of macroses,
        # depending on the level of the object, for which Service is being created:
        #
        # For CHI-level Service:
        # 1. {chi} - ClickHouseInstallation name
        # 2. {chiID} - short hashed ClickHouseInstallation name (BEWARE, this is an experimental feature)
        #
        # For Cluster-level Service:
        # 1. {chi} - ClickHouseInstallation name
        # 2. {chiID} - short hashed ClickHouseInstallation name (BEWARE, this is an experimental feature)
        # 3. {cluster} - cluster name
        # 4. {clusterID} - short hashed cluster name (BEWARE, this is an experimental feature)
        # 5. {clusterIndex} - 0-based index of the cluster in the CHI (BEWARE, this is an experimental feature)
        #
        # For Shard-level Service:
        # 1. {chi} - ClickHouseInstallation name
        # 2. {chiID} - short hashed ClickHouseInstallation name (BEWARE, this is an experimental feature)
        # 3. {cluster} - cluster name
        # 4. {clusterID} - short hashed cluster name (BEWARE, this is an experimental feature)
        # 5. {clusterIndex} - 0-based index of the cluster in the CHI (BEWARE, this is an experimental feature)
        # 6. {shard} - shard name
        # 7. {shardID} - short hashed shard name (BEWARE, this is an experimental feature)
        # 8. {shardIndex} - 0-based index of the shard in the cluster (BEWARE, this is an experimental feature)
        #
        # For Replica-level Service:
        # 1. {chi} - ClickHouseInstallation name
        # 2. {chiID} - short hashed ClickHouseInstallation name (BEWARE, this is an experimental feature)
        # 3. {cluster} - cluster name
        # 4. {clusterID} - short hashed cluster name (BEWARE, this is an experimental feature)
        # 5. {clusterIndex} - 0-based index of the cluster in the CHI (BEWARE, this is an experimental feature)
        # 6. {shard} - shard name
        # 7. {shardID} - short hashed shard name (BEWARE, this is an experimental feature)
        # 8. {shardIndex} - 0-based index of the shard in the cluster (BEWARE, this is an experimental feature)
        # 9. {replica} - replica name
        # 10. {replicaID} - short hashed replica name (BEWARE, this is an experimental feature)
        # 11. {replicaIndex} - 0-based index of the replica in the shard (BEWARE, this is an experimental feature)
        # 12. {chiScopeIndex} - 0-based index of the host in the chi (BEWARE, this is an experimental feature)
        # 13. {chiScopeCycleIndex} - 0-based index of the host's cycle in the chi-scope (BEWARE, this is an experimental feature)
        # 14. {chiScopeCycleOffset} - 0-based offset of the host in the chi-scope cycle (BEWARE, this is an experimental feature)
        # 15. {clusterScopeIndex} - 0-based index of the host in the cluster (BEWARE, this is an experimental feature)
        # 16. {clusterScopeCycleIndex} - 0-based index of the host's cycle in the cluster-scope (BEWARE, this is an experimental feature)
        # 17. {clusterScopeCycleOffset} - 0-based offset of the host in the cluster-scope cycle (BEWARE, this is an experimental feature)
        # 18. {shardScopeIndex} - 0-based index of the host in the shard (BEWARE, this is an experimental feature)
        # 19. {replicaScopeIndex} - 0-based index of the host in the replica (BEWARE, this is an experimental feature)
        # 20. {clusterScopeCycleHeadPointsToPreviousCycleTail} - 0-based cluster-scope index of previous cycle tail
        generateName: "service-{chi}"
        # type ObjectMeta struct from k8s.io/meta/v1
        metadata:
          labels:
            custom.label: "custom.value"
          annotations:
            # For more details on Internal Load Balancer check
            # https://kubernetes.io/docs/concepts/services-networking/service/#internal-load-balancer
            cloud.google.com/load-balancer-type: "Internal"
            service.beta.kubernetes.io/aws-load-balancer-internal: "true"
            service.beta.kubernetes.io/azure-load-balancer-internal: "true"
            service.beta.kubernetes.io/openstack-internal-load-balancer: "true"
            service.beta.kubernetes.io/cce-load-balancer-internal-vpc: "true"

            # NLB Load Balancer
            service.beta.kubernetes.io/aws-load-balancer-type: "nlb"

        # type ServiceSpec struct from k8s.io/core/v1
        spec:
          ports:
            - name: http
              port: 8123
            - name: tcp
              port: 9000
          type: LoadBalancer

      - name: replica-service-template
        # type ServiceSpec struct from k8s.io/core/v1
        spec:
          ports:
            - name: http
              port: 8123
            - name: tcp
              port: 9000
            - name: interserver
              port: 9009
          type: ClusterIP
          clusterIP: None

      - name: preserve-client-source-ip
        # For more details on Preserving Client Source IP check
        # https://kubernetes.io/docs/tasks/access-application-cluster/create-external-load-balancer/#preserving-the-client-source-ip
        spec:
          selector:
            app: example
          ports:
            - name: http
              port: 8123
            - name: tcp
              port: 9000
            - name: interserver
              port: 9009
          externalTrafficPolicy: Local
          type: LoadBalancer

    volumeClaimTemplates:
      - name: default-volume-claim
        # Specify PVC provisioner.
        # 1. StatefulSet. PVC would be provisioned by the StatefulSet
        # 2. Operator. PVC would be provisioned by the operator
        provisioner: StatefulSet

        # Specify PVC reclaim policy.
        # 1. Retain. Keep PVC from being deleted
        #    Retaining PVC will also keep backing PV from deletion. This is useful in case we need to keep data intact.
        # 2. Delete
        reclaimPolicy: Retain

        # type ObjectMeta struct {} from k8s.io/meta/v1
        metadata:
          labels:
            a: "b"
        # type PersistentVolumeClaimSpec struct from k8s.io/core/v1
        spec:
          # 1. If storageClassName is not specified, default StorageClass
          # (must be specified by cluster administrator) would be used for provisioning
          # 2. If storageClassName is set to an empty string (‘’), no storage class will be used
          # dynamic provisioning is disabled for this PVC. Existing, “Available”, PVs
          # (that do not have a specified storageClassName) will be considered for binding to the PVC
          #storageClassName: gold
          accessModes:
            - ReadWriteOnce
          resources:
            requests:
              storage: 1Gi

      - name: volume-claim-retain-pvc
        # Specify PVC provisioner.
        # 1. StatefulSet. PVC would be provisioned by the StatefulSet
        # 2. Operator. PVC would be provisioned by the operator
        provisioner: StatefulSet

        # Specify PVC reclaim policy.
        # 1. Retain. Keep PVC from being deleted
        #    Retaining PVC will also keep backing PV from deletion. This is useful in case we need to keep data intact.
        # 2. Delete
        reclaimPolicy: Retain

        # type ObjectMeta struct {} from k8s.io/meta/v1
        metadata:
          labels:
            a: "b"
        # type PersistentVolumeClaimSpec struct from k8s.io/core/v1
        spec:
          # 1. If storageClassName is not specified, default StorageClass
          # (must be specified by cluster administrator) would be used for provisioning
          # 2. If storageClassName is set to an empty string (‘’), no storage class will be used
          # dynamic provisioning is disabled for this PVC. Existing, “Available”, PVs
          # (that do not have a specified storageClassName) will be considered for binding to the PVC
          #storageClassName: gold
          accessModes:
            - ReadWriteOnce
          resources:
            requests:
              storage: 1Gi

    podTemplates:
      # multiple pod templates makes possible to update version smoothly
      # pod template for ClickHouse v18.16.1
      - name: clickhouse-v23.8
        # We may need to label nodes with clickhouse=allow label for this example to run
        # See ./label_nodes.sh for this purpose
        zone:
          key: "clickhouse"
          values:
            - "allow"
        # Shortcut version for AWS installations
        #zone:
        #  values:
        #    - "us-east-1a"

        # Possible values for podDistribution are:
        # Unspecified - empty value
        # ClickHouseAntiAffinity - AntiAffinity by ClickHouse instances.
        #   Pod pushes away other ClickHouse pods, which allows one ClickHouse instance per topologyKey-specified unit
        #   CH - (push away) - CH - (push away) - CH
        # ShardAntiAffinity - AntiAffinity by shard name.
        #   Pod pushes away other pods of the same shard (replicas of this shard),
        #   which allows one replica of a shard instance per topologyKey-specified unit.
        #   Other shards are allowed - it does not push all shards away, but CH-instances of the same shard only.
        #   Used for data loss avoidance - keeps all copies of the shard on different topologyKey-specified units.
        #   shard1,replica1 - (push away) - shard1,replica2 - (push away) - shard1,replica3
        # ReplicaAntiAffinity - AntiAffinity by replica name.
        #   Pod pushes away other pods of the same replica (shards of this replica),
        #   which allows one shard of a replica per topologyKey-specified unit.
        #   Other replicas are allowed - it does not push all replicas away, but CH-instances of the same replica only.
        #   Used to evenly distribute load from "full cluster scan" queries.
        #   shard1,replica1 - (push away) - shard2,replica1 - (push away) - shard3,replica1
        # AnotherNamespaceAntiAffinity - AntiAffinity by "another" namespace.
        #   Pod pushes away pods from another namespace, which allows same-namespace pods per topologyKey-specified unit.
        #   ns1 - (push away) - ns2 - (push away) - ns3
        # AnotherClickHouseInstallationAntiAffinity - AntiAffinity by "another" ClickHouseInstallation name.
        #   Pod pushes away pods from another ClickHouseInstallation,
        #   which allows same-ClickHouseInstallation pods per topologyKey-specified unit.
        #   CHI1 - (push away) - CHI2 - (push away) - CHI3
        # AnotherClusterAntiAffinity - AntiAffinity by "another" cluster name.
        #   Pod pushes away pods from another Cluster,
        #   which allows same-cluster pods per topologyKey-specified unit.
        #   cluster1 - (push away) - cluster2 - (push away) - cluster3
        # MaxNumberPerNode - AntiAffinity by cycle index.
        #   Pod pushes away pods from the same cycle,
        #   which allows to specify maximum number of ClickHouse instances per topologyKey-specified unit.
        #   Used to setup circular replication.
        # NamespaceAffinity - Affinity by namespace.
        #   Pod attracts pods from the same namespace,
        #   which allows pods from same namespace per topologyKey-specified unit.
        #   ns1 + (attracts) + ns1
        # ClickHouseInstallationAffinity - Affinity by ClickHouseInstallation name.
        #   Pod attracts pods from the same ClickHouseInstallation,
        #   which allows pods from the same CHI per topologyKey-specified unit.
        #   CHI1 + (attracts) + CHI1
        # ClusterAffinity - Affinity by cluster name.
        #   Pod attracts pods from the same cluster,
        #   which allows pods from the same Cluster per topologyKey-specified unit.
        #   cluster1 + (attracts) + cluster1
        # ShardAffinity - Affinity by shard name.
        #   Pod attracts pods from the same shard,
        #   which allows pods from the same Shard per topologyKey-specified unit.
        #   shard1 + (attracts) + shard1
        # ReplicaAffinity - Affinity by replica name.
        #   Pod attracts pods from the same replica,
        #   which allows pods from the same Replica per topologyKey-specified unit.
        #   replica1 + (attracts) + replica1
        # PreviousTailAffinity - Affinity to overlap cycles. Used to make cycle pod distribution
        #   cycle head + (attracts to) + previous cycle tail
        podDistribution:
          - type: ShardAntiAffinity
          - type: MaxNumberPerNode
            number: 2
            # Apply podDistribution on per-host basis
            topologyKey: "kubernetes.io/hostname"
            # Apply podDistribution on per-zone basis
            #topologyKey: "kubernetes.io/zone"

        # type ObjectMeta struct {} from k8s.io/meta/v1
        metadata:
          labels:
            a: "b"
        # type PodSpec struct {} from k8s.io/core/v1
        spec:
          containers:
            - name: clickhouse
              image: clickhouse/clickhouse-server:23.8
              volumeMounts:
                - name: default-volume-claim
                  mountPath: /var/lib/clickhouse
              resources:
                requests:
                  memory: "64Mi"
                  cpu: "100m"
                limits:
                  memory: "64Mi"
                  cpu: "100m"
            - name: clickhouse-log
              image: clickhouse/clickhouse-server:23.8
              command:
                - "/bin/sh"
                - "-c"
                - "--"
              args:
                - "while true; do sleep 30; done;"

      # pod template for ClickHouse v23.8
      - name: clickhouse-v23.8
        # type ObjectMeta struct {} from k8s.io/meta/v1
        metadata:
          labels:
            a: "b"
        # type PodSpec struct {} from k8s.io/core/v1
        spec:
          containers:
            - name: clickhouse
              image: clickhouse/clickhouse-server:23.8
              volumeMounts:
                - name: default-volume-claim
                  mountPath: /var/lib/clickhouse
              resources:
                requests:
                  memory: "64Mi"
                  cpu: "100m"
                limits:
                  memory: "64Mi"
                  cpu: "100m"
            - name: clickhouse-log
              image: clickhouse/clickhouse-server:23.8
              command:
                - "/bin/sh"
                - "-c"
                - "--"
              args:
                - "while true; do sleep 30; done;"