# Template Parameters: # # KIND=ClickHouseInstallation # SINGULAR=clickhouseinstallation # PLURAL=clickhouseinstallations # SHORT=chi # OPERATOR_VERSION=0.24.2 # apiVersion: apiextensions.k8s.io/v1 kind: CustomResourceDefinition metadata: name: clickhouseinstallations.clickhouse.altinity.com labels: clickhouse.altinity.com/chop: 0.24.2 spec: group: clickhouse.altinity.com scope: Namespaced names: kind: ClickHouseInstallation singular: clickhouseinstallation plural: clickhouseinstallations shortNames: - chi versions: - name: v1 served: true storage: true additionalPrinterColumns: - name: version type: string description: Operator version priority: 1 # show in wide view jsonPath: .status.chop-version - name: clusters type: integer description: Clusters count jsonPath: .status.clusters - name: shards type: integer description: Shards count priority: 1 # show in wide view jsonPath: .status.shards - name: hosts type: integer description: Hosts count jsonPath: .status.hosts - name: taskID type: string description: TaskID priority: 1 # show in wide view jsonPath: .status.taskID - name: status type: string description: Resource status jsonPath: .status.status - name: hosts-unchanged type: integer description: Unchanged hosts count priority: 1 # show in wide view jsonPath: .status.hostsUnchanged - name: hosts-updated type: integer description: Updated hosts count priority: 1 # show in wide view jsonPath: .status.hostsUpdated - name: hosts-added type: integer description: Added hosts count priority: 1 # show in wide view jsonPath: .status.hostsAdded - name: hosts-completed type: integer description: Completed hosts count jsonPath: .status.hostsCompleted - name: hosts-deleted type: integer description: Hosts deleted count priority: 1 # show in wide view jsonPath: .status.hostsDeleted - name: hosts-delete type: integer description: Hosts to be deleted count priority: 1 # show in wide view jsonPath: .status.hostsDelete - name: endpoint type: string description: Client access endpoint priority: 1 # show in wide view jsonPath: .status.endpoint - name: age type: date description: Age of the resource # Displayed in all priorities jsonPath: .metadata.creationTimestamp subresources: status: {} schema: openAPIV3Schema: description: "define a set of Kubernetes resources (StatefulSet, PVC, Service, ConfigMap) which describe behavior one or more clusters" type: object required: - spec properties: apiVersion: description: | APIVersion defines the versioned schema of this representation of an object. Servers should convert recognized schemas to the latest internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources type: string kind: description: | Kind is a string value representing the REST resource this object represents. Servers may infer this from the endpoint the client submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds type: string metadata: type: object status: type: object description: | Status contains many fields like a normalized configuration, clickhouse-operator version, current action and all applied action list, current taskID and all applied taskIDs and other properties: chop-version: type: string description: "Operator version" chop-commit: type: string description: "Operator git commit SHA" chop-date: type: string description: "Operator build date" chop-ip: type: string description: "IP address of the operator's pod which managed this resource" clusters: type: integer minimum: 0 description: "Clusters count" shards: type: integer minimum: 0 description: "Shards count" replicas: type: integer minimum: 0 description: "Replicas count" hosts: type: integer minimum: 0 description: "Hosts count" status: type: string description: "Status" taskID: type: string description: "Current task id" taskIDsStarted: type: array description: "Started task ids" nullable: true items: type: string taskIDsCompleted: type: array description: "Completed task ids" nullable: true items: type: string action: type: string description: "Action" actions: type: array description: "Actions" nullable: true items: type: string error: type: string description: "Last error" errors: type: array description: "Errors" nullable: true items: type: string hostsUnchanged: type: integer minimum: 0 description: "Unchanged Hosts count" hostsUpdated: type: integer minimum: 0 description: "Updated Hosts count" hostsAdded: type: integer minimum: 0 description: "Added Hosts count" hostsCompleted: type: integer minimum: 0 description: "Completed Hosts count" hostsDeleted: type: integer minimum: 0 description: "Deleted Hosts count" hostsDelete: type: integer minimum: 0 description: "About to delete Hosts count" pods: type: array description: "Pods" nullable: true items: type: string pod-ips: type: array description: "Pod IPs" nullable: true items: type: string fqdns: type: array description: "Pods FQDNs" nullable: true items: type: string endpoint: type: string description: "Endpoint" generation: type: integer minimum: 0 description: "Generation" normalized: type: object description: "Normalized resource requested" x-kubernetes-preserve-unknown-fields: true normalizedCompleted: type: object description: "Normalized resource completed" x-kubernetes-preserve-unknown-fields: true hostsWithTablesCreated: type: array description: "List of hosts with tables created by the operator" nullable: true items: type: string usedTemplates: type: array description: "List of templates used to build this CHI" nullable: true x-kubernetes-preserve-unknown-fields: true items: type: object x-kubernetes-preserve-unknown-fields: true spec: type: object # x-kubernetes-preserve-unknown-fields: true description: | Specification of the desired behavior of one or more ClickHouse clusters More info: https://github.com/Altinity/clickhouse-operator/blob/master/docs/custom_resource_explained.md properties: taskID: type: string description: | Allows to define custom taskID for CHI update and watch status of this update execution. Displayed in all .status.taskID* fields. By default (if not filled) every update of CHI manifest will generate random taskID stop: &TypeStringBool type: string description: | Allows to stop all ClickHouse clusters defined in a CHI. Works as the following: - When `stop` is `1` operator sets `Replicas: 0` in each StatefulSet. Thie leads to having all `Pods` and `Service` deleted. All PVCs are kept intact. - When `stop` is `0` operator sets `Replicas: 1` and `Pod`s and `Service`s will created again and all retained PVCs will be attached to `Pod`s. enum: # List StringBoolXXX constants from model - "" - "0" - "1" - "False" - "false" - "True" - "true" - "No" - "no" - "Yes" - "yes" - "Off" - "off" - "On" - "on" - "Disable" - "disable" - "Enable" - "enable" - "Disabled" - "disabled" - "Enabled" - "enabled" restart: type: string description: | In case 'RollingUpdate' specified, the operator will always restart ClickHouse pods during reconcile. This options is used in rare cases when force restart is required and is typically removed after the use in order to avoid unneeded restarts. enum: - "" - "RollingUpdate" troubleshoot: <<: *TypeStringBool description: | Allows to troubleshoot Pods during CrashLoopBack state. This may happen when wrong configuration applied, in this case `clickhouse-server` wouldn't start. Command within ClickHouse container is modified with `sleep` in order to avoid quick restarts and give time to troubleshoot via CLI. Liveness and Readiness probes are disabled as well. namespaceDomainPattern: type: string description: | Custom domain pattern which will be used for DNS names of `Service` or `Pod`. Typical use scenario - custom cluster domain in Kubernetes cluster Example: %s.svc.my.test templating: type: object # nullable: true description: | Optional, applicable inside ClickHouseInstallationTemplate only. Defines current ClickHouseInstallationTemplate application options to target ClickHouseInstallation(s)." properties: policy: type: string description: | When defined as `auto` inside ClickhouseInstallationTemplate, this ClickhouseInstallationTemplate will be auto-added into ClickHouseInstallation, selectable by `chiSelector`. Default value is `manual`, meaning ClickHouseInstallation should request this ClickhouseInstallationTemplate explicitly. enum: - "" - "auto" - "manual" chiSelector: type: object description: "Optional, defines selector for ClickHouseInstallation(s) to be templated with ClickhouseInstallationTemplate" # nullable: true x-kubernetes-preserve-unknown-fields: true reconciling: type: object description: "Optional, allows tuning reconciling cycle for ClickhouseInstallation from clickhouse-operator side" # nullable: true properties: policy: type: string description: | DISCUSSED TO BE DEPRECATED Syntax sugar Overrides all three 'reconcile.host.wait.{exclude, queries, include}' values from the operator's config Possible values: - wait - should wait to exclude host, complete queries and include host back into the cluster - nowait - should NOT wait to exclude host, complete queries and include host back into the cluster enum: - "" - "wait" - "nowait" configMapPropagationTimeout: type: integer description: | Timeout in seconds for `clickhouse-operator` to wait for modified `ConfigMap` to propagate into the `Pod` More details: https://kubernetes.io/docs/concepts/configuration/configmap/#mounted-configmaps-are-updated-automatically minimum: 0 maximum: 3600 cleanup: type: object description: "Optional, defines behavior for cleanup Kubernetes resources during reconcile cycle" # nullable: true properties: unknownObjects: type: object description: | Describes what clickhouse-operator should do with found Kubernetes resources which should be managed by clickhouse-operator, but do not have `ownerReference` to any currently managed `ClickHouseInstallation` resource. Default behavior is `Delete`" # nullable: true properties: statefulSet: &TypeObjectsCleanup type: string description: "Behavior policy for unknown StatefulSet, `Delete` by default" enum: # List ObjectsCleanupXXX constants from model - "" - "Retain" - "Delete" pvc: type: string <<: *TypeObjectsCleanup description: "Behavior policy for unknown PVC, `Delete` by default" configMap: <<: *TypeObjectsCleanup description: "Behavior policy for unknown ConfigMap, `Delete` by default" service: <<: *TypeObjectsCleanup description: "Behavior policy for unknown Service, `Delete` by default" reconcileFailedObjects: type: object description: | Describes what clickhouse-operator should do with Kubernetes resources which are failed during reconcile. Default behavior is `Retain`" # nullable: true properties: statefulSet: <<: *TypeObjectsCleanup description: "Behavior policy for failed StatefulSet, `Retain` by default" pvc: <<: *TypeObjectsCleanup description: "Behavior policy for failed PVC, `Retain` by default" configMap: <<: *TypeObjectsCleanup description: "Behavior policy for failed ConfigMap, `Retain` by default" service: <<: *TypeObjectsCleanup description: "Behavior policy for failed Service, `Retain` by default" defaults: type: object description: | define default behavior for whole ClickHouseInstallation, some behavior can be re-define on cluster, shard and replica level More info: https://github.com/Altinity/clickhouse-operator/blob/master/docs/custom_resource_explained.md#specdefaults # nullable: true properties: replicasUseFQDN: <<: *TypeStringBool description: | define should replicas be specified by FQDN in ``. In case of "no" will use short hostname and clickhouse-server will use kubernetes default suffixes for DNS lookup "yes" by default distributedDDL: type: object description: | allows change `` settings More info: https://clickhouse.tech/docs/en/operations/server-configuration-parameters/settings/#server-settings-distributed_ddl # nullable: true properties: profile: type: string description: "Settings from this profile will be used to execute DDL queries" storageManagement: type: object description: default storage management options properties: provisioner: &TypePVCProvisioner type: string description: "defines `PVC` provisioner - be it StatefulSet or the Operator" enum: - "" - "StatefulSet" - "Operator" reclaimPolicy: &TypePVCReclaimPolicy type: string description: | defines behavior of `PVC` deletion. `Delete` by default, if `Retain` specified then `PVC` will be kept when deleting StatefulSet enum: - "" - "Retain" - "Delete" templates: &TypeTemplateNames type: object description: "optional, configuration of the templates names which will use for generate Kubernetes resources according to one or more ClickHouse clusters described in current ClickHouseInstallation (chi) resource" # nullable: true properties: hostTemplate: type: string description: "optional, template name from chi.spec.templates.hostTemplates, which will apply to configure every `clickhouse-server` instance during render ConfigMap resources which will mount into `Pod`" podTemplate: type: string description: "optional, template name from chi.spec.templates.podTemplates, allows customization each `Pod` resource during render and reconcile each StatefulSet.spec resource described in `chi.spec.configuration.clusters`" dataVolumeClaimTemplate: type: string description: "optional, template name from chi.spec.templates.volumeClaimTemplates, allows customization each `PVC` which will mount for clickhouse data directory in each `Pod` during render and reconcile every StatefulSet.spec resource described in `chi.spec.configuration.clusters`" logVolumeClaimTemplate: type: string description: "optional, template name from chi.spec.templates.volumeClaimTemplates, allows customization each `PVC` which will mount for clickhouse log directory in each `Pod` during render and reconcile every StatefulSet.spec resource described in `chi.spec.configuration.clusters`" serviceTemplate: type: string description: "optional, template name from chi.spec.templates.serviceTemplates, allows customization for one `Service` resource which will created by `clickhouse-operator` which cover all clusters in whole `chi` resource" clusterServiceTemplate: type: string description: "optional, template name from chi.spec.templates.serviceTemplates, allows customization for each `Service` resource which will created by `clickhouse-operator` which cover each clickhouse cluster described in `chi.spec.configuration.clusters`" shardServiceTemplate: type: string description: "optional, template name from chi.spec.templates.serviceTemplates, allows customization for each `Service` resource which will created by `clickhouse-operator` which cover each shard inside clickhouse cluster described in `chi.spec.configuration.clusters`" replicaServiceTemplate: type: string description: "optional, template name from chi.spec.templates.serviceTemplates, allows customization for each `Service` resource which will created by `clickhouse-operator` which cover each replica inside each shard inside each clickhouse cluster described in `chi.spec.configuration.clusters`" volumeClaimTemplate: type: string description: "optional, alias for dataVolumeClaimTemplate, template name from chi.spec.templates.volumeClaimTemplates, allows customization each `PVC` which will mount for clickhouse data directory in each `Pod` during render and reconcile every StatefulSet.spec resource described in `chi.spec.configuration.clusters`" configuration: type: object description: "allows configure multiple aspects and behavior for `clickhouse-server` instance and also allows describe multiple `clickhouse-server` clusters inside one `chi` resource" # nullable: true properties: zookeeper: &TypeZookeeperConfig type: object description: | allows configure .. section in each `Pod` during generate `ConfigMap` which will mounted in `/etc/clickhouse-server/config.d/` `clickhouse-operator` itself doesn't manage Zookeeper, please install Zookeeper separatelly look examples on https://github.com/Altinity/clickhouse-operator/tree/master/deploy/zookeeper/ currently, zookeeper (or clickhouse-keeper replacement) used for *ReplicatedMergeTree table engines and for `distributed_ddl` More details: https://clickhouse.tech/docs/en/operations/server-configuration-parameters/settings/#server-settings_zookeeper # nullable: true properties: nodes: type: array description: "describe every available zookeeper cluster node for interaction" # nullable: true items: type: object #required: # - host properties: host: type: string description: "dns name or ip address for Zookeeper node" port: type: integer description: "TCP port which used to connect to Zookeeper node" minimum: 0 maximum: 65535 secure: <<: *TypeStringBool description: "if a secure connection to Zookeeper is required" session_timeout_ms: type: integer description: "session timeout during connect to Zookeeper" operation_timeout_ms: type: integer description: "one operation timeout during Zookeeper transactions" root: type: string description: "optional root znode path inside zookeeper to store ClickHouse related data (replication queue or distributed DDL)" identity: type: string description: "optional access credentials string with `user:password` format used when use digest authorization in Zookeeper" users: type: object description: | allows configure .. section in each `Pod` during generate `ConfigMap` which will mount in `/etc/clickhouse-server/users.d/` you can configure password hashed, authorization restrictions, database level security row filters etc. More details: https://clickhouse.tech/docs/en/operations/settings/settings-users/ Your yaml code will convert to XML, see examples https://github.com/Altinity/clickhouse-operator/blob/master/docs/custom_resource_explained.md#specconfigurationusers any key could contains `valueFrom` with `secretKeyRef` which allow pass password from kubernetes secrets secret value will pass in `pod.spec.containers.evn`, and generate with from_env=XXX in XML in /etc/clickhouse-server/users.d/chop-generated-users.xml it not allow automatically updates when updates `secret`, change spec.taskID for manually trigger reconcile cycle look into https://github.com/Altinity/clickhouse-operator/blob/master/docs/chi-examples/05-settings-01-overview.yaml for examples any key with prefix `k8s_secret_` shall has value with format namespace/secret/key or secret/key in this case value from secret will write directly into XML tag during render *-usersd ConfigMap any key with prefix `k8s_secret_env` shall has value with format namespace/secret/key or secret/key in this case value from secret will write into environment variable and write to XML tag via from_env=XXX look into https://github.com/Altinity/clickhouse-operator/blob/master/docs/chi-examples/05-settings-01-overview.yaml for examples # nullable: true x-kubernetes-preserve-unknown-fields: true profiles: type: object description: | allows configure .. section in each `Pod` during generate `ConfigMap` which will mount in `/etc/clickhouse-server/users.d/` you can configure any aspect of settings profile More details: https://clickhouse.tech/docs/en/operations/settings/settings-profiles/ Your yaml code will convert to XML, see examples https://github.com/Altinity/clickhouse-operator/blob/master/docs/custom_resource_explained.md#specconfigurationprofiles # nullable: true x-kubernetes-preserve-unknown-fields: true quotas: type: object description: | allows configure .. section in each `Pod` during generate `ConfigMap` which will mount in `/etc/clickhouse-server/users.d/` you can configure any aspect of resource quotas More details: https://clickhouse.tech/docs/en/operations/quotas/ Your yaml code will convert to XML, see examples https://github.com/Altinity/clickhouse-operator/blob/master/docs/custom_resource_explained.md#specconfigurationquotas # nullable: true x-kubernetes-preserve-unknown-fields: true settings: &TypeSettings type: object description: | allows configure `clickhouse-server` settings inside ... tag in each `Pod` during generate `ConfigMap` which will mount in `/etc/clickhouse-server/config.d/` More details: https://clickhouse.tech/docs/en/operations/settings/settings/ Your yaml code will convert to XML, see examples https://github.com/Altinity/clickhouse-operator/blob/master/docs/custom_resource_explained.md#specconfigurationsettings any key could contains `valueFrom` with `secretKeyRef` which allow pass password from kubernetes secrets look into https://github.com/Altinity/clickhouse-operator/blob/master/docs/chi-examples/05-settings-01-overview.yaml for examples secret value will pass in `pod.spec.evn`, and generate with from_env=XXX in XML in /etc/clickhouse-server/config.d/chop-generated-settings.xml it not allow automatically updates when updates `secret`, change spec.taskID for manually trigger reconcile cycle # nullable: true x-kubernetes-preserve-unknown-fields: true files: &TypeFiles type: object description: | allows define content of any setting file inside each `Pod` during generate `ConfigMap` which will mount in `/etc/clickhouse-server/config.d/` or `/etc/clickhouse-server/conf.d/` or `/etc/clickhouse-server/users.d/` every key in this object is the file name every value in this object is the file content you can use `!!binary |` and base64 for binary files, see details here https://yaml.org/type/binary.html each key could contains prefix like USERS, COMMON, HOST or config.d, users.d, cond.d, wrong prefixes will ignored, subfolders also will ignored More details: https://github.com/Altinity/clickhouse-operator/blob/master/docs/chi-examples/05-settings-05-files-nested.yaml any key could contains `valueFrom` with `secretKeyRef` which allow pass values from kubernetes secrets secrets will mounted into pod as separate volume in /etc/clickhouse-server/secrets.d/ and will automatically update when update secret it useful for pass SSL certificates from cert-manager or similar tool look into https://github.com/Altinity/clickhouse-operator/blob/master/docs/chi-examples/05-settings-01-overview.yaml for examples # nullable: true x-kubernetes-preserve-unknown-fields: true clusters: type: array description: | describes clusters layout and allows change settings on cluster-level, shard-level and replica-level every cluster is a set of StatefulSet, one StatefulSet contains only one Pod with `clickhouse-server` all Pods will rendered in part of ClickHouse configs, mounted from ConfigMap as `/etc/clickhouse-server/config.d/chop-generated-remote_servers.xml` Clusters will use for Distributed table engine, more details: https://clickhouse.tech/docs/en/engines/table-engines/special/distributed/ If `cluster` contains zookeeper settings (could be inherited from top `chi` level), when you can create *ReplicatedMergeTree tables # nullable: true items: type: object #required: # - name properties: name: type: string description: "cluster name, used to identify set of servers and wide used during generate names of related Kubernetes resources" minLength: 1 # See namePartClusterMaxLen const maxLength: 15 pattern: "^[a-zA-Z0-9-]{0,15}$" zookeeper: <<: *TypeZookeeperConfig description: | optional, allows configure .. section in each `Pod` only in current ClickHouse cluster, during generate `ConfigMap` which will mounted in `/etc/clickhouse-server/config.d/` override top-level `chi.spec.configuration.zookeeper` settings settings: <<: *TypeSettings description: | optional, allows configure `clickhouse-server` settings inside ... tag in each `Pod` only in one cluster during generate `ConfigMap` which will mount in `/etc/clickhouse-server/config.d/` override top-level `chi.spec.configuration.settings` More details: https://clickhouse.tech/docs/en/operations/settings/settings/ files: <<: *TypeFiles description: | optional, allows define content of any setting file inside each `Pod` on current cluster during generate `ConfigMap` which will mount in `/etc/clickhouse-server/config.d/` or `/etc/clickhouse-server/conf.d/` or `/etc/clickhouse-server/users.d/` override top-level `chi.spec.configuration.files` templates: <<: *TypeTemplateNames description: | optional, configuration of the templates names which will use for generate Kubernetes resources according to selected cluster override top-level `chi.spec.configuration.templates` schemaPolicy: type: object description: | describes how schema is propagated within replicas and shards properties: replica: type: string description: "how schema is propagated within a replica" enum: # List SchemaPolicyReplicaXXX constants from model - "" - "None" - "All" shard: type: string description: "how schema is propagated between shards" enum: # List SchemaPolicyShardXXX constants from model - "" - "None" - "All" - "DistributedTablesOnly" insecure: <<: *TypeStringBool description: optional, open insecure ports for cluster, defaults to "yes" secure: <<: *TypeStringBool description: optional, open secure ports for cluster secret: type: object description: "optional, shared secret value to secure cluster communications" properties: auto: <<: *TypeStringBool description: "Auto-generate shared secret value to secure cluster communications" value: description: "Cluster shared secret value in plain text" type: string valueFrom: description: "Cluster shared secret source" type: object properties: secretKeyRef: description: | Selects a key of a secret in the clickhouse installation namespace. Should not be used if value is not empty. type: object properties: name: description: | Name of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names type: string key: description: The key of the secret to select from. Must be a valid secret key. type: string optional: description: Specify whether the Secret or its key must be defined type: boolean required: - name - key pdbMaxUnavailable: type: integer description: | Pod eviction is allowed if at most "pdbMaxUnavailable" pods are unavailable after the eviction, i.e. even in absence of the evicted pod. For example, one can prevent all voluntary evictions by specifying 0. This is a mutually exclusive setting with "minAvailable". minimum: 0 maximum: 65535 layout: type: object description: | describe current cluster layout, how much shards in cluster, how much replica in shard allows override settings on each shard and replica separatelly # nullable: true properties: shardsCount: type: integer description: | how much shards for current ClickHouse cluster will run in Kubernetes, each shard contains shared-nothing part of data and contains set of replicas, cluster contains 1 shard by default" replicasCount: type: integer description: | how much replicas in each shards for current cluster will run in Kubernetes, each replica is a separate `StatefulSet` which contains only one `Pod` with `clickhouse-server` instance, every shard contains 1 replica by default" shards: type: array description: | optional, allows override top-level `chi.spec.configuration`, cluster-level `chi.spec.configuration.clusters` settings for each shard separately, use it only if you fully understand what you do" # nullable: true items: type: object properties: name: type: string description: "optional, by default shard name is generated, but you can override it and setup custom name" minLength: 1 # See namePartShardMaxLen const maxLength: 15 pattern: "^[a-zA-Z0-9-]{0,15}$" definitionType: type: string description: "DEPRECATED - to be removed soon" weight: type: integer description: | optional, 1 by default, allows setup shard setting which will use during insert into tables with `Distributed` engine, will apply in inside ConfigMap which will mount in /etc/clickhouse-server/config.d/chop-generated-remote_servers.xml More details: https://clickhouse.tech/docs/en/engines/table-engines/special/distributed/ internalReplication: <<: *TypeStringBool description: | optional, `true` by default when `chi.spec.configuration.clusters[].layout.ReplicaCount` > 1 and 0 otherwise allows setup setting which will use during insert into tables with `Distributed` engine for insert only in one live replica and other replicas will download inserted data during replication, will apply in inside ConfigMap which will mount in /etc/clickhouse-server/config.d/chop-generated-remote_servers.xml More details: https://clickhouse.tech/docs/en/engines/table-engines/special/distributed/ settings: <<: *TypeSettings description: | optional, allows configure `clickhouse-server` settings inside ... tag in each `Pod` only in one shard during generate `ConfigMap` which will mount in `/etc/clickhouse-server/config.d/` override top-level `chi.spec.configuration.settings` and cluster-level `chi.spec.configuration.clusters.settings` More details: https://clickhouse.tech/docs/en/operations/settings/settings/ files: <<: *TypeFiles description: | optional, allows define content of any setting file inside each `Pod` only in one shard during generate `ConfigMap` which will mount in `/etc/clickhouse-server/config.d/` or `/etc/clickhouse-server/conf.d/` or `/etc/clickhouse-server/users.d/` override top-level `chi.spec.configuration.files` and cluster-level `chi.spec.configuration.clusters.files` templates: <<: *TypeTemplateNames description: | optional, configuration of the templates names which will use for generate Kubernetes resources according to selected shard override top-level `chi.spec.configuration.templates` and cluster-level `chi.spec.configuration.clusters.templates` replicasCount: type: integer description: | optional, how much replicas in selected shard for selected ClickHouse cluster will run in Kubernetes, each replica is a separate `StatefulSet` which contains only one `Pod` with `clickhouse-server` instance, shard contains 1 replica by default override cluster-level `chi.spec.configuration.clusters.layout.replicasCount` minimum: 1 replicas: type: array description: | optional, allows override behavior for selected replicas from cluster-level `chi.spec.configuration.clusters` and shard-level `chi.spec.configuration.clusters.layout.shards` # nullable: true items: # Host type: object properties: name: type: string description: "optional, by default replica name is generated, but you can override it and setup custom name" minLength: 1 # See namePartReplicaMaxLen const maxLength: 15 pattern: "^[a-zA-Z0-9-]{0,15}$" insecure: <<: *TypeStringBool description: | optional, open insecure ports for cluster, defaults to "yes" secure: <<: *TypeStringBool description: | optional, open secure ports tcpPort: type: integer description: | optional, setup `Pod.spec.containers.ports` with name `tcp` for selected replica, override `chi.spec.templates.hostTemplates.spec.tcpPort` allows connect to `clickhouse-server` via TCP Native protocol via kubernetes `Service` minimum: 1 maximum: 65535 tlsPort: type: integer minimum: 1 maximum: 65535 httpPort: type: integer description: | optional, setup `Pod.spec.containers.ports` with name `http` for selected replica, override `chi.spec.templates.hostTemplates.spec.httpPort` allows connect to `clickhouse-server` via HTTP protocol via kubernetes `Service` minimum: 1 maximum: 65535 httpsPort: type: integer minimum: 1 maximum: 65535 interserverHTTPPort: type: integer description: | optional, setup `Pod.spec.containers.ports` with name `interserver` for selected replica, override `chi.spec.templates.hostTemplates.spec.interserverHTTPPort` allows connect between replicas inside same shard during fetch replicated data parts HTTP protocol minimum: 1 maximum: 65535 settings: <<: *TypeSettings description: | optional, allows configure `clickhouse-server` settings inside ... tag in `Pod` only in one replica during generate `ConfigMap` which will mount in `/etc/clickhouse-server/conf.d/` override top-level `chi.spec.configuration.settings`, cluster-level `chi.spec.configuration.clusters.settings` and shard-level `chi.spec.configuration.clusters.layout.shards.settings` More details: https://clickhouse.tech/docs/en/operations/settings/settings/ files: <<: *TypeFiles description: | optional, allows define content of any setting file inside `Pod` only in one replica during generate `ConfigMap` which will mount in `/etc/clickhouse-server/config.d/` or `/etc/clickhouse-server/conf.d/` or `/etc/clickhouse-server/users.d/` override top-level `chi.spec.configuration.files`, cluster-level `chi.spec.configuration.clusters.files` and shard-level `chi.spec.configuration.clusters.layout.shards.files` templates: <<: *TypeTemplateNames description: | optional, configuration of the templates names which will use for generate Kubernetes resources according to selected replica override top-level `chi.spec.configuration.templates`, cluster-level `chi.spec.configuration.clusters.templates` and shard-level `chi.spec.configuration.clusters.layout.shards.templates` replicas: type: array description: "optional, allows override top-level `chi.spec.configuration` and cluster-level `chi.spec.configuration.clusters` configuration for each replica and each shard relates to selected replica, use it only if you fully understand what you do" # nullable: true items: type: object properties: name: type: string description: "optional, by default replica name is generated, but you can override it and setup custom name" minLength: 1 # See namePartShardMaxLen const maxLength: 15 pattern: "^[a-zA-Z0-9-]{0,15}$" settings: <<: *TypeSettings description: | optional, allows configure `clickhouse-server` settings inside ... tag in `Pod` only in one replica during generate `ConfigMap` which will mount in `/etc/clickhouse-server/conf.d/` override top-level `chi.spec.configuration.settings`, cluster-level `chi.spec.configuration.clusters.settings` and will ignore if shard-level `chi.spec.configuration.clusters.layout.shards` present More details: https://clickhouse.tech/docs/en/operations/settings/settings/ files: <<: *TypeFiles description: | optional, allows define content of any setting file inside each `Pod` only in one replica during generate `ConfigMap` which will mount in `/etc/clickhouse-server/config.d/` or `/etc/clickhouse-server/conf.d/` or `/etc/clickhouse-server/users.d/` override top-level `chi.spec.configuration.files` and cluster-level `chi.spec.configuration.clusters.files`, will ignore if `chi.spec.configuration.clusters.layout.shards` presents templates: <<: *TypeTemplateNames description: | optional, configuration of the templates names which will use for generate Kubernetes resources according to selected replica override top-level `chi.spec.configuration.templates`, cluster-level `chi.spec.configuration.clusters.templates` shardsCount: type: integer description: "optional, count of shards related to current replica, you can override each shard behavior on low-level `chi.spec.configuration.clusters.layout.replicas.shards`" minimum: 1 shards: type: array description: "optional, list of shards related to current replica, will ignore if `chi.spec.configuration.clusters.layout.shards` presents" # nullable: true items: # Host type: object properties: name: type: string description: "optional, by default shard name is generated, but you can override it and setup custom name" minLength: 1 # See namePartReplicaMaxLen const maxLength: 15 pattern: "^[a-zA-Z0-9-]{0,15}$" insecure: <<: *TypeStringBool description: | optional, open insecure ports for cluster, defaults to "yes" secure: <<: *TypeStringBool description: | optional, open secure ports tcpPort: type: integer description: | optional, setup `Pod.spec.containers.ports` with name `tcp` for selected shard, override `chi.spec.templates.hostTemplates.spec.tcpPort` allows connect to `clickhouse-server` via TCP Native protocol via kubernetes `Service` minimum: 1 maximum: 65535 tlsPort: type: integer minimum: 1 maximum: 65535 httpPort: type: integer description: | optional, setup `Pod.spec.containers.ports` with name `http` for selected shard, override `chi.spec.templates.hostTemplates.spec.httpPort` allows connect to `clickhouse-server` via HTTP protocol via kubernetes `Service` minimum: 1 maximum: 65535 httpsPort: type: integer minimum: 1 maximum: 65535 interserverHTTPPort: type: integer description: | optional, setup `Pod.spec.containers.ports` with name `interserver` for selected shard, override `chi.spec.templates.hostTemplates.spec.interserverHTTPPort` allows connect between replicas inside same shard during fetch replicated data parts HTTP protocol minimum: 1 maximum: 65535 settings: <<: *TypeSettings description: | optional, allows configure `clickhouse-server` settings inside ... tag in `Pod` only in one shard related to current replica during generate `ConfigMap` which will mount in `/etc/clickhouse-server/conf.d/` override top-level `chi.spec.configuration.settings`, cluster-level `chi.spec.configuration.clusters.settings` and replica-level `chi.spec.configuration.clusters.layout.replicas.settings` More details: https://clickhouse.tech/docs/en/operations/settings/settings/ files: <<: *TypeFiles description: | optional, allows define content of any setting file inside each `Pod` only in one shard related to current replica during generate `ConfigMap` which will mount in `/etc/clickhouse-server/config.d/` or `/etc/clickhouse-server/conf.d/` or `/etc/clickhouse-server/users.d/` override top-level `chi.spec.configuration.files` and cluster-level `chi.spec.configuration.clusters.files`, will ignore if `chi.spec.configuration.clusters.layout.shards` presents templates: <<: *TypeTemplateNames description: | optional, configuration of the templates names which will use for generate Kubernetes resources according to selected replica override top-level `chi.spec.configuration.templates`, cluster-level `chi.spec.configuration.clusters.templates`, replica-level `chi.spec.configuration.clusters.layout.replicas.templates` templates: type: object description: "allows define templates which will use for render Kubernetes resources like StatefulSet, ConfigMap, Service, PVC, by default, clickhouse-operator have own templates, but you can override it" # nullable: true properties: hostTemplates: type: array description: "hostTemplate will use during apply to generate `clickhose-server` config files" # nullable: true items: type: object #required: # - name properties: name: description: "template name, could use to link inside top-level `chi.spec.defaults.templates.hostTemplate`, cluster-level `chi.spec.configuration.clusters.templates.hostTemplate`, shard-level `chi.spec.configuration.clusters.layout.shards.temlates.hostTemplate`, replica-level `chi.spec.configuration.clusters.layout.replicas.templates.hostTemplate`" type: string portDistribution: type: array description: "define how will distribute numeric values of named ports in `Pod.spec.containers.ports` and clickhouse-server configs" # nullable: true items: type: object #required: # - type properties: type: type: string description: "type of distribution, when `Unspecified` (default value) then all listen ports on clickhouse-server configuration in all Pods will have the same value, when `ClusterScopeIndex` then ports will increment to offset from base value depends on shard and replica index inside cluster with combination of `chi.spec.templates.podTemlates.spec.HostNetwork` it allows setup ClickHouse cluster inside Kubernetes and provide access via external network bypass Kubernetes internal network" enum: # List PortDistributionXXX constants - "" - "Unspecified" - "ClusterScopeIndex" spec: # Host type: object properties: name: type: string description: "by default, hostname will generate, but this allows define custom name for each `clickhuse-server`" minLength: 1 # See namePartReplicaMaxLen const maxLength: 15 pattern: "^[a-zA-Z0-9-]{0,15}$" insecure: <<: *TypeStringBool description: | optional, open insecure ports for cluster, defaults to "yes" secure: <<: *TypeStringBool description: | optional, open secure ports tcpPort: type: integer description: | optional, setup `tcp_port` inside `clickhouse-server` settings for each Pod where current template will apply if specified, should have equal value with `chi.spec.templates.podTemplates.spec.containers.ports[name=tcp]` More info: https://clickhouse.tech/docs/en/interfaces/tcp/ minimum: 1 maximum: 65535 tlsPort: type: integer minimum: 1 maximum: 65535 httpPort: type: integer description: | optional, setup `http_port` inside `clickhouse-server` settings for each Pod where current template will apply if specified, should have equal value with `chi.spec.templates.podTemplates.spec.containers.ports[name=http]` More info: https://clickhouse.tech/docs/en/interfaces/http/ minimum: 1 maximum: 65535 httpsPort: type: integer minimum: 1 maximum: 65535 interserverHTTPPort: type: integer description: | optional, setup `interserver_http_port` inside `clickhouse-server` settings for each Pod where current template will apply if specified, should have equal value with `chi.spec.templates.podTemplates.spec.containers.ports[name=interserver]` More info: https://clickhouse.tech/docs/en/operations/server-configuration-parameters/settings/#interserver-http-port minimum: 1 maximum: 65535 settings: <<: *TypeSettings description: | optional, allows configure `clickhouse-server` settings inside ... tag in each `Pod` where this template will apply during generate `ConfigMap` which will mount in `/etc/clickhouse-server/conf.d/` More details: https://clickhouse.tech/docs/en/operations/settings/settings/ files: <<: *TypeFiles description: | optional, allows define content of any setting file inside each `Pod` where this template will apply during generate `ConfigMap` which will mount in `/etc/clickhouse-server/config.d/` or `/etc/clickhouse-server/conf.d/` or `/etc/clickhouse-server/users.d/` templates: <<: *TypeTemplateNames description: "be careful, this part of CRD allows override template inside template, don't use it if you don't understand what you do" podTemplates: type: array description: | podTemplate will use during render `Pod` inside `StatefulSet.spec` and allows define rendered `Pod.spec`, pod scheduling distribution and pod zone More information: https://github.com/Altinity/clickhouse-operator/blob/master/docs/custom_resource_explained.md#spectemplatespodtemplates # nullable: true items: type: object #required: # - name properties: name: type: string description: "template name, could use to link inside top-level `chi.spec.defaults.templates.podTemplate`, cluster-level `chi.spec.configuration.clusters.templates.podTemplate`, shard-level `chi.spec.configuration.clusters.layout.shards.temlates.podTemplate`, replica-level `chi.spec.configuration.clusters.layout.replicas.templates.podTemplate`" generateName: type: string description: "allows define format for generated `Pod` name, look to https://github.com/Altinity/clickhouse-operator/blob/master/docs/custom_resource_explained.md#spectemplatesservicetemplates for details about available template variables" zone: type: object description: "allows define custom zone name and will separate ClickHouse `Pods` between nodes, shortcut for `chi.spec.templates.podTemplates.spec.affinity.podAntiAffinity`" #required: # - values properties: key: type: string description: "optional, if defined, allows select kubernetes nodes by label with `name` equal `key`" values: type: array description: "optional, if defined, allows select kubernetes nodes by label with `value` in `values`" # nullable: true items: type: string distribution: type: string description: "DEPRECATED, shortcut for `chi.spec.templates.podTemplates.spec.affinity.podAntiAffinity`" enum: - "" - "Unspecified" - "OnePerHost" podDistribution: type: array description: "define ClickHouse Pod distribution policy between Kubernetes Nodes inside Shard, Replica, Namespace, CHI, another ClickHouse cluster" # nullable: true items: type: object #required: # - type properties: type: type: string description: "you can define multiple affinity policy types" enum: # List PodDistributionXXX constants - "" - "Unspecified" - "ClickHouseAntiAffinity" - "ShardAntiAffinity" - "ReplicaAntiAffinity" - "AnotherNamespaceAntiAffinity" - "AnotherClickHouseInstallationAntiAffinity" - "AnotherClusterAntiAffinity" - "MaxNumberPerNode" - "NamespaceAffinity" - "ClickHouseInstallationAffinity" - "ClusterAffinity" - "ShardAffinity" - "ReplicaAffinity" - "PreviousTailAffinity" - "CircularReplication" scope: type: string description: "scope for apply each podDistribution" enum: # list PodDistributionScopeXXX constants - "" - "Unspecified" - "Shard" - "Replica" - "Cluster" - "ClickHouseInstallation" - "Namespace" number: type: integer description: "define, how much ClickHouse Pods could be inside selected scope with selected distribution type" minimum: 0 maximum: 65535 topologyKey: type: string description: | use for inter-pod affinity look to `pod.spec.affinity.podAntiAffinity.preferredDuringSchedulingIgnoredDuringExecution.podAffinityTerm.topologyKey`, more info: https://kubernetes.io/docs/concepts/scheduling-eviction/assign-pod-node/#inter-pod-affinity-and-anti-affinity" metadata: type: object description: | allows pass standard object's metadata from template to Pod More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#metadata # nullable: true x-kubernetes-preserve-unknown-fields: true spec: # TODO specify PodSpec type: object description: "allows define whole Pod.spec inside StaefulSet.spec, look to https://kubernetes.io/docs/concepts/workloads/pods/#pod-templates for details" # nullable: true x-kubernetes-preserve-unknown-fields: true volumeClaimTemplates: type: array description: | allows define template for rendering `PVC` kubernetes resource, which would use inside `Pod` for mount clickhouse `data`, clickhouse `logs` or something else # nullable: true items: type: object #required: # - name # - spec properties: name: type: string description: | template name, could use to link inside top-level `chi.spec.defaults.templates.dataVolumeClaimTemplate` or `chi.spec.defaults.templates.logVolumeClaimTemplate`, cluster-level `chi.spec.configuration.clusters.templates.dataVolumeClaimTemplate` or `chi.spec.configuration.clusters.templates.logVolumeClaimTemplate`, shard-level `chi.spec.configuration.clusters.layout.shards.temlates.dataVolumeClaimTemplate` or `chi.spec.configuration.clusters.layout.shards.temlates.logVolumeClaimTemplate` replica-level `chi.spec.configuration.clusters.layout.replicas.templates.dataVolumeClaimTemplate` or `chi.spec.configuration.clusters.layout.replicas.templates.logVolumeClaimTemplate` provisioner: *TypePVCProvisioner reclaimPolicy: *TypePVCReclaimPolicy metadata: type: object description: | allows to pass standard object's metadata from template to PVC More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#metadata # nullable: true x-kubernetes-preserve-unknown-fields: true spec: type: object description: | allows define all aspects of `PVC` resource More info: https://kubernetes.io/docs/concepts/storage/persistent-volumes/#persistentvolumeclaims # nullable: true x-kubernetes-preserve-unknown-fields: true serviceTemplates: type: array description: | allows define template for rendering `Service` which would get endpoint from Pods which scoped chi-wide, cluster-wide, shard-wide, replica-wide level # nullable: true items: type: object #required: # - name # - spec properties: name: type: string description: | template name, could use to link inside chi-level `chi.spec.defaults.templates.serviceTemplate` cluster-level `chi.spec.configuration.clusters.templates.clusterServiceTemplate` shard-level `chi.spec.configuration.clusters.layout.shards.temlates.shardServiceTemplate` replica-level `chi.spec.configuration.clusters.layout.replicas.templates.replicaServiceTemplate` or `chi.spec.configuration.clusters.layout.shards.replicas.replicaServiceTemplate` generateName: type: string description: | allows define format for generated `Service` name, look to https://github.com/Altinity/clickhouse-operator/blob/master/docs/custom_resource_explained.md#spectemplatesservicetemplates for details about available template variables" metadata: # TODO specify ObjectMeta type: object description: | allows pass standard object's metadata from template to Service Could be use for define specificly for Cloud Provider metadata which impact to behavior of service More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#metadata # nullable: true x-kubernetes-preserve-unknown-fields: true spec: # TODO specify ServiceSpec type: object description: | describe behavior of generated Service More info: https://kubernetes.io/docs/concepts/services-networking/service/ # nullable: true x-kubernetes-preserve-unknown-fields: true useTemplates: type: array description: | list of `ClickHouseInstallationTemplate` (chit) resource names which will merge with current `CHI` manifest during render Kubernetes resources to create related ClickHouse clusters" # nullable: true items: type: object #required: # - name properties: name: type: string description: "name of `ClickHouseInstallationTemplate` (chit) resource" namespace: type: string description: "Kubernetes namespace where need search `chit` resource, depending on `watchNamespaces` settings in `clichouse-operator`" useType: type: string description: "optional, current strategy is only merge, and current `chi` settings have more priority than merged template `chit`" enum: # List useTypeXXX constants from model - "" - "merge" --- # Template Parameters: # # KIND=ClickHouseInstallationTemplate # SINGULAR=clickhouseinstallationtemplate # PLURAL=clickhouseinstallationtemplates # SHORT=chit # OPERATOR_VERSION=0.24.2 # apiVersion: apiextensions.k8s.io/v1 kind: CustomResourceDefinition metadata: name: clickhouseinstallationtemplates.clickhouse.altinity.com labels: clickhouse.altinity.com/chop: 0.24.2 spec: group: clickhouse.altinity.com scope: Namespaced names: kind: ClickHouseInstallationTemplate singular: clickhouseinstallationtemplate plural: clickhouseinstallationtemplates shortNames: - chit versions: - name: v1 served: true storage: true additionalPrinterColumns: - name: version type: string description: Operator version priority: 1 # show in wide view jsonPath: .status.chop-version - name: clusters type: integer description: Clusters count jsonPath: .status.clusters - name: shards type: integer description: Shards count priority: 1 # show in wide view jsonPath: .status.shards - name: hosts type: integer description: Hosts count jsonPath: .status.hosts - name: taskID type: string description: TaskID priority: 1 # show in wide view jsonPath: .status.taskID - name: status type: string description: Resource status jsonPath: .status.status - name: hosts-unchanged type: integer description: Unchanged hosts count priority: 1 # show in wide view jsonPath: .status.hostsUnchanged - name: hosts-updated type: integer description: Updated hosts count priority: 1 # show in wide view jsonPath: .status.hostsUpdated - name: hosts-added type: integer description: Added hosts count priority: 1 # show in wide view jsonPath: .status.hostsAdded - name: hosts-completed type: integer description: Completed hosts count jsonPath: .status.hostsCompleted - name: hosts-deleted type: integer description: Hosts deleted count priority: 1 # show in wide view jsonPath: .status.hostsDeleted - name: hosts-delete type: integer description: Hosts to be deleted count priority: 1 # show in wide view jsonPath: .status.hostsDelete - name: endpoint type: string description: Client access endpoint priority: 1 # show in wide view jsonPath: .status.endpoint - name: age type: date description: Age of the resource # Displayed in all priorities jsonPath: .metadata.creationTimestamp subresources: status: {} schema: openAPIV3Schema: description: "define a set of Kubernetes resources (StatefulSet, PVC, Service, ConfigMap) which describe behavior one or more clusters" type: object required: - spec properties: apiVersion: description: | APIVersion defines the versioned schema of this representation of an object. Servers should convert recognized schemas to the latest internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources type: string kind: description: | Kind is a string value representing the REST resource this object represents. Servers may infer this from the endpoint the client submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds type: string metadata: type: object status: type: object description: | Status contains many fields like a normalized configuration, clickhouse-operator version, current action and all applied action list, current taskID and all applied taskIDs and other properties: chop-version: type: string description: "Operator version" chop-commit: type: string description: "Operator git commit SHA" chop-date: type: string description: "Operator build date" chop-ip: type: string description: "IP address of the operator's pod which managed this resource" clusters: type: integer minimum: 0 description: "Clusters count" shards: type: integer minimum: 0 description: "Shards count" replicas: type: integer minimum: 0 description: "Replicas count" hosts: type: integer minimum: 0 description: "Hosts count" status: type: string description: "Status" taskID: type: string description: "Current task id" taskIDsStarted: type: array description: "Started task ids" nullable: true items: type: string taskIDsCompleted: type: array description: "Completed task ids" nullable: true items: type: string action: type: string description: "Action" actions: type: array description: "Actions" nullable: true items: type: string error: type: string description: "Last error" errors: type: array description: "Errors" nullable: true items: type: string hostsUnchanged: type: integer minimum: 0 description: "Unchanged Hosts count" hostsUpdated: type: integer minimum: 0 description: "Updated Hosts count" hostsAdded: type: integer minimum: 0 description: "Added Hosts count" hostsCompleted: type: integer minimum: 0 description: "Completed Hosts count" hostsDeleted: type: integer minimum: 0 description: "Deleted Hosts count" hostsDelete: type: integer minimum: 0 description: "About to delete Hosts count" pods: type: array description: "Pods" nullable: true items: type: string pod-ips: type: array description: "Pod IPs" nullable: true items: type: string fqdns: type: array description: "Pods FQDNs" nullable: true items: type: string endpoint: type: string description: "Endpoint" generation: type: integer minimum: 0 description: "Generation" normalized: type: object description: "Normalized resource requested" x-kubernetes-preserve-unknown-fields: true normalizedCompleted: type: object description: "Normalized resource completed" x-kubernetes-preserve-unknown-fields: true hostsWithTablesCreated: type: array description: "List of hosts with tables created by the operator" nullable: true items: type: string usedTemplates: type: array description: "List of templates used to build this CHI" nullable: true x-kubernetes-preserve-unknown-fields: true items: type: object x-kubernetes-preserve-unknown-fields: true spec: type: object # x-kubernetes-preserve-unknown-fields: true description: | Specification of the desired behavior of one or more ClickHouse clusters More info: https://github.com/Altinity/clickhouse-operator/blob/master/docs/custom_resource_explained.md properties: taskID: type: string description: | Allows to define custom taskID for CHI update and watch status of this update execution. Displayed in all .status.taskID* fields. By default (if not filled) every update of CHI manifest will generate random taskID stop: &TypeStringBool type: string description: | Allows to stop all ClickHouse clusters defined in a CHI. Works as the following: - When `stop` is `1` operator sets `Replicas: 0` in each StatefulSet. Thie leads to having all `Pods` and `Service` deleted. All PVCs are kept intact. - When `stop` is `0` operator sets `Replicas: 1` and `Pod`s and `Service`s will created again and all retained PVCs will be attached to `Pod`s. enum: # List StringBoolXXX constants from model - "" - "0" - "1" - "False" - "false" - "True" - "true" - "No" - "no" - "Yes" - "yes" - "Off" - "off" - "On" - "on" - "Disable" - "disable" - "Enable" - "enable" - "Disabled" - "disabled" - "Enabled" - "enabled" restart: type: string description: | In case 'RollingUpdate' specified, the operator will always restart ClickHouse pods during reconcile. This options is used in rare cases when force restart is required and is typically removed after the use in order to avoid unneeded restarts. enum: - "" - "RollingUpdate" troubleshoot: <<: *TypeStringBool description: | Allows to troubleshoot Pods during CrashLoopBack state. This may happen when wrong configuration applied, in this case `clickhouse-server` wouldn't start. Command within ClickHouse container is modified with `sleep` in order to avoid quick restarts and give time to troubleshoot via CLI. Liveness and Readiness probes are disabled as well. namespaceDomainPattern: type: string description: | Custom domain pattern which will be used for DNS names of `Service` or `Pod`. Typical use scenario - custom cluster domain in Kubernetes cluster Example: %s.svc.my.test templating: type: object # nullable: true description: | Optional, applicable inside ClickHouseInstallationTemplate only. Defines current ClickHouseInstallationTemplate application options to target ClickHouseInstallation(s)." properties: policy: type: string description: | When defined as `auto` inside ClickhouseInstallationTemplate, this ClickhouseInstallationTemplate will be auto-added into ClickHouseInstallation, selectable by `chiSelector`. Default value is `manual`, meaning ClickHouseInstallation should request this ClickhouseInstallationTemplate explicitly. enum: - "" - "auto" - "manual" chiSelector: type: object description: "Optional, defines selector for ClickHouseInstallation(s) to be templated with ClickhouseInstallationTemplate" # nullable: true x-kubernetes-preserve-unknown-fields: true reconciling: type: object description: "Optional, allows tuning reconciling cycle for ClickhouseInstallation from clickhouse-operator side" # nullable: true properties: policy: type: string description: | DISCUSSED TO BE DEPRECATED Syntax sugar Overrides all three 'reconcile.host.wait.{exclude, queries, include}' values from the operator's config Possible values: - wait - should wait to exclude host, complete queries and include host back into the cluster - nowait - should NOT wait to exclude host, complete queries and include host back into the cluster enum: - "" - "wait" - "nowait" configMapPropagationTimeout: type: integer description: | Timeout in seconds for `clickhouse-operator` to wait for modified `ConfigMap` to propagate into the `Pod` More details: https://kubernetes.io/docs/concepts/configuration/configmap/#mounted-configmaps-are-updated-automatically minimum: 0 maximum: 3600 cleanup: type: object description: "Optional, defines behavior for cleanup Kubernetes resources during reconcile cycle" # nullable: true properties: unknownObjects: type: object description: | Describes what clickhouse-operator should do with found Kubernetes resources which should be managed by clickhouse-operator, but do not have `ownerReference` to any currently managed `ClickHouseInstallation` resource. Default behavior is `Delete`" # nullable: true properties: statefulSet: &TypeObjectsCleanup type: string description: "Behavior policy for unknown StatefulSet, `Delete` by default" enum: # List ObjectsCleanupXXX constants from model - "" - "Retain" - "Delete" pvc: type: string <<: *TypeObjectsCleanup description: "Behavior policy for unknown PVC, `Delete` by default" configMap: <<: *TypeObjectsCleanup description: "Behavior policy for unknown ConfigMap, `Delete` by default" service: <<: *TypeObjectsCleanup description: "Behavior policy for unknown Service, `Delete` by default" reconcileFailedObjects: type: object description: | Describes what clickhouse-operator should do with Kubernetes resources which are failed during reconcile. Default behavior is `Retain`" # nullable: true properties: statefulSet: <<: *TypeObjectsCleanup description: "Behavior policy for failed StatefulSet, `Retain` by default" pvc: <<: *TypeObjectsCleanup description: "Behavior policy for failed PVC, `Retain` by default" configMap: <<: *TypeObjectsCleanup description: "Behavior policy for failed ConfigMap, `Retain` by default" service: <<: *TypeObjectsCleanup description: "Behavior policy for failed Service, `Retain` by default" defaults: type: object description: | define default behavior for whole ClickHouseInstallation, some behavior can be re-define on cluster, shard and replica level More info: https://github.com/Altinity/clickhouse-operator/blob/master/docs/custom_resource_explained.md#specdefaults # nullable: true properties: replicasUseFQDN: <<: *TypeStringBool description: | define should replicas be specified by FQDN in ``. In case of "no" will use short hostname and clickhouse-server will use kubernetes default suffixes for DNS lookup "yes" by default distributedDDL: type: object description: | allows change `` settings More info: https://clickhouse.tech/docs/en/operations/server-configuration-parameters/settings/#server-settings-distributed_ddl # nullable: true properties: profile: type: string description: "Settings from this profile will be used to execute DDL queries" storageManagement: type: object description: default storage management options properties: provisioner: &TypePVCProvisioner type: string description: "defines `PVC` provisioner - be it StatefulSet or the Operator" enum: - "" - "StatefulSet" - "Operator" reclaimPolicy: &TypePVCReclaimPolicy type: string description: | defines behavior of `PVC` deletion. `Delete` by default, if `Retain` specified then `PVC` will be kept when deleting StatefulSet enum: - "" - "Retain" - "Delete" templates: &TypeTemplateNames type: object description: "optional, configuration of the templates names which will use for generate Kubernetes resources according to one or more ClickHouse clusters described in current ClickHouseInstallation (chi) resource" # nullable: true properties: hostTemplate: type: string description: "optional, template name from chi.spec.templates.hostTemplates, which will apply to configure every `clickhouse-server` instance during render ConfigMap resources which will mount into `Pod`" podTemplate: type: string description: "optional, template name from chi.spec.templates.podTemplates, allows customization each `Pod` resource during render and reconcile each StatefulSet.spec resource described in `chi.spec.configuration.clusters`" dataVolumeClaimTemplate: type: string description: "optional, template name from chi.spec.templates.volumeClaimTemplates, allows customization each `PVC` which will mount for clickhouse data directory in each `Pod` during render and reconcile every StatefulSet.spec resource described in `chi.spec.configuration.clusters`" logVolumeClaimTemplate: type: string description: "optional, template name from chi.spec.templates.volumeClaimTemplates, allows customization each `PVC` which will mount for clickhouse log directory in each `Pod` during render and reconcile every StatefulSet.spec resource described in `chi.spec.configuration.clusters`" serviceTemplate: type: string description: "optional, template name from chi.spec.templates.serviceTemplates, allows customization for one `Service` resource which will created by `clickhouse-operator` which cover all clusters in whole `chi` resource" clusterServiceTemplate: type: string description: "optional, template name from chi.spec.templates.serviceTemplates, allows customization for each `Service` resource which will created by `clickhouse-operator` which cover each clickhouse cluster described in `chi.spec.configuration.clusters`" shardServiceTemplate: type: string description: "optional, template name from chi.spec.templates.serviceTemplates, allows customization for each `Service` resource which will created by `clickhouse-operator` which cover each shard inside clickhouse cluster described in `chi.spec.configuration.clusters`" replicaServiceTemplate: type: string description: "optional, template name from chi.spec.templates.serviceTemplates, allows customization for each `Service` resource which will created by `clickhouse-operator` which cover each replica inside each shard inside each clickhouse cluster described in `chi.spec.configuration.clusters`" volumeClaimTemplate: type: string description: "optional, alias for dataVolumeClaimTemplate, template name from chi.spec.templates.volumeClaimTemplates, allows customization each `PVC` which will mount for clickhouse data directory in each `Pod` during render and reconcile every StatefulSet.spec resource described in `chi.spec.configuration.clusters`" configuration: type: object description: "allows configure multiple aspects and behavior for `clickhouse-server` instance and also allows describe multiple `clickhouse-server` clusters inside one `chi` resource" # nullable: true properties: zookeeper: &TypeZookeeperConfig type: object description: | allows configure .. section in each `Pod` during generate `ConfigMap` which will mounted in `/etc/clickhouse-server/config.d/` `clickhouse-operator` itself doesn't manage Zookeeper, please install Zookeeper separatelly look examples on https://github.com/Altinity/clickhouse-operator/tree/master/deploy/zookeeper/ currently, zookeeper (or clickhouse-keeper replacement) used for *ReplicatedMergeTree table engines and for `distributed_ddl` More details: https://clickhouse.tech/docs/en/operations/server-configuration-parameters/settings/#server-settings_zookeeper # nullable: true properties: nodes: type: array description: "describe every available zookeeper cluster node for interaction" # nullable: true items: type: object #required: # - host properties: host: type: string description: "dns name or ip address for Zookeeper node" port: type: integer description: "TCP port which used to connect to Zookeeper node" minimum: 0 maximum: 65535 secure: <<: *TypeStringBool description: "if a secure connection to Zookeeper is required" session_timeout_ms: type: integer description: "session timeout during connect to Zookeeper" operation_timeout_ms: type: integer description: "one operation timeout during Zookeeper transactions" root: type: string description: "optional root znode path inside zookeeper to store ClickHouse related data (replication queue or distributed DDL)" identity: type: string description: "optional access credentials string with `user:password` format used when use digest authorization in Zookeeper" users: type: object description: | allows configure .. section in each `Pod` during generate `ConfigMap` which will mount in `/etc/clickhouse-server/users.d/` you can configure password hashed, authorization restrictions, database level security row filters etc. More details: https://clickhouse.tech/docs/en/operations/settings/settings-users/ Your yaml code will convert to XML, see examples https://github.com/Altinity/clickhouse-operator/blob/master/docs/custom_resource_explained.md#specconfigurationusers any key could contains `valueFrom` with `secretKeyRef` which allow pass password from kubernetes secrets secret value will pass in `pod.spec.containers.evn`, and generate with from_env=XXX in XML in /etc/clickhouse-server/users.d/chop-generated-users.xml it not allow automatically updates when updates `secret`, change spec.taskID for manually trigger reconcile cycle look into https://github.com/Altinity/clickhouse-operator/blob/master/docs/chi-examples/05-settings-01-overview.yaml for examples any key with prefix `k8s_secret_` shall has value with format namespace/secret/key or secret/key in this case value from secret will write directly into XML tag during render *-usersd ConfigMap any key with prefix `k8s_secret_env` shall has value with format namespace/secret/key or secret/key in this case value from secret will write into environment variable and write to XML tag via from_env=XXX look into https://github.com/Altinity/clickhouse-operator/blob/master/docs/chi-examples/05-settings-01-overview.yaml for examples # nullable: true x-kubernetes-preserve-unknown-fields: true profiles: type: object description: | allows configure .. section in each `Pod` during generate `ConfigMap` which will mount in `/etc/clickhouse-server/users.d/` you can configure any aspect of settings profile More details: https://clickhouse.tech/docs/en/operations/settings/settings-profiles/ Your yaml code will convert to XML, see examples https://github.com/Altinity/clickhouse-operator/blob/master/docs/custom_resource_explained.md#specconfigurationprofiles # nullable: true x-kubernetes-preserve-unknown-fields: true quotas: type: object description: | allows configure .. section in each `Pod` during generate `ConfigMap` which will mount in `/etc/clickhouse-server/users.d/` you can configure any aspect of resource quotas More details: https://clickhouse.tech/docs/en/operations/quotas/ Your yaml code will convert to XML, see examples https://github.com/Altinity/clickhouse-operator/blob/master/docs/custom_resource_explained.md#specconfigurationquotas # nullable: true x-kubernetes-preserve-unknown-fields: true settings: &TypeSettings type: object description: | allows configure `clickhouse-server` settings inside ... tag in each `Pod` during generate `ConfigMap` which will mount in `/etc/clickhouse-server/config.d/` More details: https://clickhouse.tech/docs/en/operations/settings/settings/ Your yaml code will convert to XML, see examples https://github.com/Altinity/clickhouse-operator/blob/master/docs/custom_resource_explained.md#specconfigurationsettings any key could contains `valueFrom` with `secretKeyRef` which allow pass password from kubernetes secrets look into https://github.com/Altinity/clickhouse-operator/blob/master/docs/chi-examples/05-settings-01-overview.yaml for examples secret value will pass in `pod.spec.evn`, and generate with from_env=XXX in XML in /etc/clickhouse-server/config.d/chop-generated-settings.xml it not allow automatically updates when updates `secret`, change spec.taskID for manually trigger reconcile cycle # nullable: true x-kubernetes-preserve-unknown-fields: true files: &TypeFiles type: object description: | allows define content of any setting file inside each `Pod` during generate `ConfigMap` which will mount in `/etc/clickhouse-server/config.d/` or `/etc/clickhouse-server/conf.d/` or `/etc/clickhouse-server/users.d/` every key in this object is the file name every value in this object is the file content you can use `!!binary |` and base64 for binary files, see details here https://yaml.org/type/binary.html each key could contains prefix like USERS, COMMON, HOST or config.d, users.d, cond.d, wrong prefixes will ignored, subfolders also will ignored More details: https://github.com/Altinity/clickhouse-operator/blob/master/docs/chi-examples/05-settings-05-files-nested.yaml any key could contains `valueFrom` with `secretKeyRef` which allow pass values from kubernetes secrets secrets will mounted into pod as separate volume in /etc/clickhouse-server/secrets.d/ and will automatically update when update secret it useful for pass SSL certificates from cert-manager or similar tool look into https://github.com/Altinity/clickhouse-operator/blob/master/docs/chi-examples/05-settings-01-overview.yaml for examples # nullable: true x-kubernetes-preserve-unknown-fields: true clusters: type: array description: | describes clusters layout and allows change settings on cluster-level, shard-level and replica-level every cluster is a set of StatefulSet, one StatefulSet contains only one Pod with `clickhouse-server` all Pods will rendered in part of ClickHouse configs, mounted from ConfigMap as `/etc/clickhouse-server/config.d/chop-generated-remote_servers.xml` Clusters will use for Distributed table engine, more details: https://clickhouse.tech/docs/en/engines/table-engines/special/distributed/ If `cluster` contains zookeeper settings (could be inherited from top `chi` level), when you can create *ReplicatedMergeTree tables # nullable: true items: type: object #required: # - name properties: name: type: string description: "cluster name, used to identify set of servers and wide used during generate names of related Kubernetes resources" minLength: 1 # See namePartClusterMaxLen const maxLength: 15 pattern: "^[a-zA-Z0-9-]{0,15}$" zookeeper: <<: *TypeZookeeperConfig description: | optional, allows configure .. section in each `Pod` only in current ClickHouse cluster, during generate `ConfigMap` which will mounted in `/etc/clickhouse-server/config.d/` override top-level `chi.spec.configuration.zookeeper` settings settings: <<: *TypeSettings description: | optional, allows configure `clickhouse-server` settings inside ... tag in each `Pod` only in one cluster during generate `ConfigMap` which will mount in `/etc/clickhouse-server/config.d/` override top-level `chi.spec.configuration.settings` More details: https://clickhouse.tech/docs/en/operations/settings/settings/ files: <<: *TypeFiles description: | optional, allows define content of any setting file inside each `Pod` on current cluster during generate `ConfigMap` which will mount in `/etc/clickhouse-server/config.d/` or `/etc/clickhouse-server/conf.d/` or `/etc/clickhouse-server/users.d/` override top-level `chi.spec.configuration.files` templates: <<: *TypeTemplateNames description: | optional, configuration of the templates names which will use for generate Kubernetes resources according to selected cluster override top-level `chi.spec.configuration.templates` schemaPolicy: type: object description: | describes how schema is propagated within replicas and shards properties: replica: type: string description: "how schema is propagated within a replica" enum: # List SchemaPolicyReplicaXXX constants from model - "" - "None" - "All" shard: type: string description: "how schema is propagated between shards" enum: # List SchemaPolicyShardXXX constants from model - "" - "None" - "All" - "DistributedTablesOnly" insecure: <<: *TypeStringBool description: optional, open insecure ports for cluster, defaults to "yes" secure: <<: *TypeStringBool description: optional, open secure ports for cluster secret: type: object description: "optional, shared secret value to secure cluster communications" properties: auto: <<: *TypeStringBool description: "Auto-generate shared secret value to secure cluster communications" value: description: "Cluster shared secret value in plain text" type: string valueFrom: description: "Cluster shared secret source" type: object properties: secretKeyRef: description: | Selects a key of a secret in the clickhouse installation namespace. Should not be used if value is not empty. type: object properties: name: description: | Name of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names type: string key: description: The key of the secret to select from. Must be a valid secret key. type: string optional: description: Specify whether the Secret or its key must be defined type: boolean required: - name - key pdbMaxUnavailable: type: integer description: | Pod eviction is allowed if at most "pdbMaxUnavailable" pods are unavailable after the eviction, i.e. even in absence of the evicted pod. For example, one can prevent all voluntary evictions by specifying 0. This is a mutually exclusive setting with "minAvailable". minimum: 0 maximum: 65535 layout: type: object description: | describe current cluster layout, how much shards in cluster, how much replica in shard allows override settings on each shard and replica separatelly # nullable: true properties: shardsCount: type: integer description: | how much shards for current ClickHouse cluster will run in Kubernetes, each shard contains shared-nothing part of data and contains set of replicas, cluster contains 1 shard by default" replicasCount: type: integer description: | how much replicas in each shards for current cluster will run in Kubernetes, each replica is a separate `StatefulSet` which contains only one `Pod` with `clickhouse-server` instance, every shard contains 1 replica by default" shards: type: array description: | optional, allows override top-level `chi.spec.configuration`, cluster-level `chi.spec.configuration.clusters` settings for each shard separately, use it only if you fully understand what you do" # nullable: true items: type: object properties: name: type: string description: "optional, by default shard name is generated, but you can override it and setup custom name" minLength: 1 # See namePartShardMaxLen const maxLength: 15 pattern: "^[a-zA-Z0-9-]{0,15}$" definitionType: type: string description: "DEPRECATED - to be removed soon" weight: type: integer description: | optional, 1 by default, allows setup shard setting which will use during insert into tables with `Distributed` engine, will apply in inside ConfigMap which will mount in /etc/clickhouse-server/config.d/chop-generated-remote_servers.xml More details: https://clickhouse.tech/docs/en/engines/table-engines/special/distributed/ internalReplication: <<: *TypeStringBool description: | optional, `true` by default when `chi.spec.configuration.clusters[].layout.ReplicaCount` > 1 and 0 otherwise allows setup setting which will use during insert into tables with `Distributed` engine for insert only in one live replica and other replicas will download inserted data during replication, will apply in inside ConfigMap which will mount in /etc/clickhouse-server/config.d/chop-generated-remote_servers.xml More details: https://clickhouse.tech/docs/en/engines/table-engines/special/distributed/ settings: <<: *TypeSettings description: | optional, allows configure `clickhouse-server` settings inside ... tag in each `Pod` only in one shard during generate `ConfigMap` which will mount in `/etc/clickhouse-server/config.d/` override top-level `chi.spec.configuration.settings` and cluster-level `chi.spec.configuration.clusters.settings` More details: https://clickhouse.tech/docs/en/operations/settings/settings/ files: <<: *TypeFiles description: | optional, allows define content of any setting file inside each `Pod` only in one shard during generate `ConfigMap` which will mount in `/etc/clickhouse-server/config.d/` or `/etc/clickhouse-server/conf.d/` or `/etc/clickhouse-server/users.d/` override top-level `chi.spec.configuration.files` and cluster-level `chi.spec.configuration.clusters.files` templates: <<: *TypeTemplateNames description: | optional, configuration of the templates names which will use for generate Kubernetes resources according to selected shard override top-level `chi.spec.configuration.templates` and cluster-level `chi.spec.configuration.clusters.templates` replicasCount: type: integer description: | optional, how much replicas in selected shard for selected ClickHouse cluster will run in Kubernetes, each replica is a separate `StatefulSet` which contains only one `Pod` with `clickhouse-server` instance, shard contains 1 replica by default override cluster-level `chi.spec.configuration.clusters.layout.replicasCount` minimum: 1 replicas: type: array description: | optional, allows override behavior for selected replicas from cluster-level `chi.spec.configuration.clusters` and shard-level `chi.spec.configuration.clusters.layout.shards` # nullable: true items: # Host type: object properties: name: type: string description: "optional, by default replica name is generated, but you can override it and setup custom name" minLength: 1 # See namePartReplicaMaxLen const maxLength: 15 pattern: "^[a-zA-Z0-9-]{0,15}$" insecure: <<: *TypeStringBool description: | optional, open insecure ports for cluster, defaults to "yes" secure: <<: *TypeStringBool description: | optional, open secure ports tcpPort: type: integer description: | optional, setup `Pod.spec.containers.ports` with name `tcp` for selected replica, override `chi.spec.templates.hostTemplates.spec.tcpPort` allows connect to `clickhouse-server` via TCP Native protocol via kubernetes `Service` minimum: 1 maximum: 65535 tlsPort: type: integer minimum: 1 maximum: 65535 httpPort: type: integer description: | optional, setup `Pod.spec.containers.ports` with name `http` for selected replica, override `chi.spec.templates.hostTemplates.spec.httpPort` allows connect to `clickhouse-server` via HTTP protocol via kubernetes `Service` minimum: 1 maximum: 65535 httpsPort: type: integer minimum: 1 maximum: 65535 interserverHTTPPort: type: integer description: | optional, setup `Pod.spec.containers.ports` with name `interserver` for selected replica, override `chi.spec.templates.hostTemplates.spec.interserverHTTPPort` allows connect between replicas inside same shard during fetch replicated data parts HTTP protocol minimum: 1 maximum: 65535 settings: <<: *TypeSettings description: | optional, allows configure `clickhouse-server` settings inside ... tag in `Pod` only in one replica during generate `ConfigMap` which will mount in `/etc/clickhouse-server/conf.d/` override top-level `chi.spec.configuration.settings`, cluster-level `chi.spec.configuration.clusters.settings` and shard-level `chi.spec.configuration.clusters.layout.shards.settings` More details: https://clickhouse.tech/docs/en/operations/settings/settings/ files: <<: *TypeFiles description: | optional, allows define content of any setting file inside `Pod` only in one replica during generate `ConfigMap` which will mount in `/etc/clickhouse-server/config.d/` or `/etc/clickhouse-server/conf.d/` or `/etc/clickhouse-server/users.d/` override top-level `chi.spec.configuration.files`, cluster-level `chi.spec.configuration.clusters.files` and shard-level `chi.spec.configuration.clusters.layout.shards.files` templates: <<: *TypeTemplateNames description: | optional, configuration of the templates names which will use for generate Kubernetes resources according to selected replica override top-level `chi.spec.configuration.templates`, cluster-level `chi.spec.configuration.clusters.templates` and shard-level `chi.spec.configuration.clusters.layout.shards.templates` replicas: type: array description: "optional, allows override top-level `chi.spec.configuration` and cluster-level `chi.spec.configuration.clusters` configuration for each replica and each shard relates to selected replica, use it only if you fully understand what you do" # nullable: true items: type: object properties: name: type: string description: "optional, by default replica name is generated, but you can override it and setup custom name" minLength: 1 # See namePartShardMaxLen const maxLength: 15 pattern: "^[a-zA-Z0-9-]{0,15}$" settings: <<: *TypeSettings description: | optional, allows configure `clickhouse-server` settings inside ... tag in `Pod` only in one replica during generate `ConfigMap` which will mount in `/etc/clickhouse-server/conf.d/` override top-level `chi.spec.configuration.settings`, cluster-level `chi.spec.configuration.clusters.settings` and will ignore if shard-level `chi.spec.configuration.clusters.layout.shards` present More details: https://clickhouse.tech/docs/en/operations/settings/settings/ files: <<: *TypeFiles description: | optional, allows define content of any setting file inside each `Pod` only in one replica during generate `ConfigMap` which will mount in `/etc/clickhouse-server/config.d/` or `/etc/clickhouse-server/conf.d/` or `/etc/clickhouse-server/users.d/` override top-level `chi.spec.configuration.files` and cluster-level `chi.spec.configuration.clusters.files`, will ignore if `chi.spec.configuration.clusters.layout.shards` presents templates: <<: *TypeTemplateNames description: | optional, configuration of the templates names which will use for generate Kubernetes resources according to selected replica override top-level `chi.spec.configuration.templates`, cluster-level `chi.spec.configuration.clusters.templates` shardsCount: type: integer description: "optional, count of shards related to current replica, you can override each shard behavior on low-level `chi.spec.configuration.clusters.layout.replicas.shards`" minimum: 1 shards: type: array description: "optional, list of shards related to current replica, will ignore if `chi.spec.configuration.clusters.layout.shards` presents" # nullable: true items: # Host type: object properties: name: type: string description: "optional, by default shard name is generated, but you can override it and setup custom name" minLength: 1 # See namePartReplicaMaxLen const maxLength: 15 pattern: "^[a-zA-Z0-9-]{0,15}$" insecure: <<: *TypeStringBool description: | optional, open insecure ports for cluster, defaults to "yes" secure: <<: *TypeStringBool description: | optional, open secure ports tcpPort: type: integer description: | optional, setup `Pod.spec.containers.ports` with name `tcp` for selected shard, override `chi.spec.templates.hostTemplates.spec.tcpPort` allows connect to `clickhouse-server` via TCP Native protocol via kubernetes `Service` minimum: 1 maximum: 65535 tlsPort: type: integer minimum: 1 maximum: 65535 httpPort: type: integer description: | optional, setup `Pod.spec.containers.ports` with name `http` for selected shard, override `chi.spec.templates.hostTemplates.spec.httpPort` allows connect to `clickhouse-server` via HTTP protocol via kubernetes `Service` minimum: 1 maximum: 65535 httpsPort: type: integer minimum: 1 maximum: 65535 interserverHTTPPort: type: integer description: | optional, setup `Pod.spec.containers.ports` with name `interserver` for selected shard, override `chi.spec.templates.hostTemplates.spec.interserverHTTPPort` allows connect between replicas inside same shard during fetch replicated data parts HTTP protocol minimum: 1 maximum: 65535 settings: <<: *TypeSettings description: | optional, allows configure `clickhouse-server` settings inside ... tag in `Pod` only in one shard related to current replica during generate `ConfigMap` which will mount in `/etc/clickhouse-server/conf.d/` override top-level `chi.spec.configuration.settings`, cluster-level `chi.spec.configuration.clusters.settings` and replica-level `chi.spec.configuration.clusters.layout.replicas.settings` More details: https://clickhouse.tech/docs/en/operations/settings/settings/ files: <<: *TypeFiles description: | optional, allows define content of any setting file inside each `Pod` only in one shard related to current replica during generate `ConfigMap` which will mount in `/etc/clickhouse-server/config.d/` or `/etc/clickhouse-server/conf.d/` or `/etc/clickhouse-server/users.d/` override top-level `chi.spec.configuration.files` and cluster-level `chi.spec.configuration.clusters.files`, will ignore if `chi.spec.configuration.clusters.layout.shards` presents templates: <<: *TypeTemplateNames description: | optional, configuration of the templates names which will use for generate Kubernetes resources according to selected replica override top-level `chi.spec.configuration.templates`, cluster-level `chi.spec.configuration.clusters.templates`, replica-level `chi.spec.configuration.clusters.layout.replicas.templates` templates: type: object description: "allows define templates which will use for render Kubernetes resources like StatefulSet, ConfigMap, Service, PVC, by default, clickhouse-operator have own templates, but you can override it" # nullable: true properties: hostTemplates: type: array description: "hostTemplate will use during apply to generate `clickhose-server` config files" # nullable: true items: type: object #required: # - name properties: name: description: "template name, could use to link inside top-level `chi.spec.defaults.templates.hostTemplate`, cluster-level `chi.spec.configuration.clusters.templates.hostTemplate`, shard-level `chi.spec.configuration.clusters.layout.shards.temlates.hostTemplate`, replica-level `chi.spec.configuration.clusters.layout.replicas.templates.hostTemplate`" type: string portDistribution: type: array description: "define how will distribute numeric values of named ports in `Pod.spec.containers.ports` and clickhouse-server configs" # nullable: true items: type: object #required: # - type properties: type: type: string description: "type of distribution, when `Unspecified` (default value) then all listen ports on clickhouse-server configuration in all Pods will have the same value, when `ClusterScopeIndex` then ports will increment to offset from base value depends on shard and replica index inside cluster with combination of `chi.spec.templates.podTemlates.spec.HostNetwork` it allows setup ClickHouse cluster inside Kubernetes and provide access via external network bypass Kubernetes internal network" enum: # List PortDistributionXXX constants - "" - "Unspecified" - "ClusterScopeIndex" spec: # Host type: object properties: name: type: string description: "by default, hostname will generate, but this allows define custom name for each `clickhuse-server`" minLength: 1 # See namePartReplicaMaxLen const maxLength: 15 pattern: "^[a-zA-Z0-9-]{0,15}$" insecure: <<: *TypeStringBool description: | optional, open insecure ports for cluster, defaults to "yes" secure: <<: *TypeStringBool description: | optional, open secure ports tcpPort: type: integer description: | optional, setup `tcp_port` inside `clickhouse-server` settings for each Pod where current template will apply if specified, should have equal value with `chi.spec.templates.podTemplates.spec.containers.ports[name=tcp]` More info: https://clickhouse.tech/docs/en/interfaces/tcp/ minimum: 1 maximum: 65535 tlsPort: type: integer minimum: 1 maximum: 65535 httpPort: type: integer description: | optional, setup `http_port` inside `clickhouse-server` settings for each Pod where current template will apply if specified, should have equal value with `chi.spec.templates.podTemplates.spec.containers.ports[name=http]` More info: https://clickhouse.tech/docs/en/interfaces/http/ minimum: 1 maximum: 65535 httpsPort: type: integer minimum: 1 maximum: 65535 interserverHTTPPort: type: integer description: | optional, setup `interserver_http_port` inside `clickhouse-server` settings for each Pod where current template will apply if specified, should have equal value with `chi.spec.templates.podTemplates.spec.containers.ports[name=interserver]` More info: https://clickhouse.tech/docs/en/operations/server-configuration-parameters/settings/#interserver-http-port minimum: 1 maximum: 65535 settings: <<: *TypeSettings description: | optional, allows configure `clickhouse-server` settings inside ... tag in each `Pod` where this template will apply during generate `ConfigMap` which will mount in `/etc/clickhouse-server/conf.d/` More details: https://clickhouse.tech/docs/en/operations/settings/settings/ files: <<: *TypeFiles description: | optional, allows define content of any setting file inside each `Pod` where this template will apply during generate `ConfigMap` which will mount in `/etc/clickhouse-server/config.d/` or `/etc/clickhouse-server/conf.d/` or `/etc/clickhouse-server/users.d/` templates: <<: *TypeTemplateNames description: "be careful, this part of CRD allows override template inside template, don't use it if you don't understand what you do" podTemplates: type: array description: | podTemplate will use during render `Pod` inside `StatefulSet.spec` and allows define rendered `Pod.spec`, pod scheduling distribution and pod zone More information: https://github.com/Altinity/clickhouse-operator/blob/master/docs/custom_resource_explained.md#spectemplatespodtemplates # nullable: true items: type: object #required: # - name properties: name: type: string description: "template name, could use to link inside top-level `chi.spec.defaults.templates.podTemplate`, cluster-level `chi.spec.configuration.clusters.templates.podTemplate`, shard-level `chi.spec.configuration.clusters.layout.shards.temlates.podTemplate`, replica-level `chi.spec.configuration.clusters.layout.replicas.templates.podTemplate`" generateName: type: string description: "allows define format for generated `Pod` name, look to https://github.com/Altinity/clickhouse-operator/blob/master/docs/custom_resource_explained.md#spectemplatesservicetemplates for details about available template variables" zone: type: object description: "allows define custom zone name and will separate ClickHouse `Pods` between nodes, shortcut for `chi.spec.templates.podTemplates.spec.affinity.podAntiAffinity`" #required: # - values properties: key: type: string description: "optional, if defined, allows select kubernetes nodes by label with `name` equal `key`" values: type: array description: "optional, if defined, allows select kubernetes nodes by label with `value` in `values`" # nullable: true items: type: string distribution: type: string description: "DEPRECATED, shortcut for `chi.spec.templates.podTemplates.spec.affinity.podAntiAffinity`" enum: - "" - "Unspecified" - "OnePerHost" podDistribution: type: array description: "define ClickHouse Pod distribution policy between Kubernetes Nodes inside Shard, Replica, Namespace, CHI, another ClickHouse cluster" # nullable: true items: type: object #required: # - type properties: type: type: string description: "you can define multiple affinity policy types" enum: # List PodDistributionXXX constants - "" - "Unspecified" - "ClickHouseAntiAffinity" - "ShardAntiAffinity" - "ReplicaAntiAffinity" - "AnotherNamespaceAntiAffinity" - "AnotherClickHouseInstallationAntiAffinity" - "AnotherClusterAntiAffinity" - "MaxNumberPerNode" - "NamespaceAffinity" - "ClickHouseInstallationAffinity" - "ClusterAffinity" - "ShardAffinity" - "ReplicaAffinity" - "PreviousTailAffinity" - "CircularReplication" scope: type: string description: "scope for apply each podDistribution" enum: # list PodDistributionScopeXXX constants - "" - "Unspecified" - "Shard" - "Replica" - "Cluster" - "ClickHouseInstallation" - "Namespace" number: type: integer description: "define, how much ClickHouse Pods could be inside selected scope with selected distribution type" minimum: 0 maximum: 65535 topologyKey: type: string description: | use for inter-pod affinity look to `pod.spec.affinity.podAntiAffinity.preferredDuringSchedulingIgnoredDuringExecution.podAffinityTerm.topologyKey`, more info: https://kubernetes.io/docs/concepts/scheduling-eviction/assign-pod-node/#inter-pod-affinity-and-anti-affinity" metadata: type: object description: | allows pass standard object's metadata from template to Pod More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#metadata # nullable: true x-kubernetes-preserve-unknown-fields: true spec: # TODO specify PodSpec type: object description: "allows define whole Pod.spec inside StaefulSet.spec, look to https://kubernetes.io/docs/concepts/workloads/pods/#pod-templates for details" # nullable: true x-kubernetes-preserve-unknown-fields: true volumeClaimTemplates: type: array description: | allows define template for rendering `PVC` kubernetes resource, which would use inside `Pod` for mount clickhouse `data`, clickhouse `logs` or something else # nullable: true items: type: object #required: # - name # - spec properties: name: type: string description: | template name, could use to link inside top-level `chi.spec.defaults.templates.dataVolumeClaimTemplate` or `chi.spec.defaults.templates.logVolumeClaimTemplate`, cluster-level `chi.spec.configuration.clusters.templates.dataVolumeClaimTemplate` or `chi.spec.configuration.clusters.templates.logVolumeClaimTemplate`, shard-level `chi.spec.configuration.clusters.layout.shards.temlates.dataVolumeClaimTemplate` or `chi.spec.configuration.clusters.layout.shards.temlates.logVolumeClaimTemplate` replica-level `chi.spec.configuration.clusters.layout.replicas.templates.dataVolumeClaimTemplate` or `chi.spec.configuration.clusters.layout.replicas.templates.logVolumeClaimTemplate` provisioner: *TypePVCProvisioner reclaimPolicy: *TypePVCReclaimPolicy metadata: type: object description: | allows to pass standard object's metadata from template to PVC More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#metadata # nullable: true x-kubernetes-preserve-unknown-fields: true spec: type: object description: | allows define all aspects of `PVC` resource More info: https://kubernetes.io/docs/concepts/storage/persistent-volumes/#persistentvolumeclaims # nullable: true x-kubernetes-preserve-unknown-fields: true serviceTemplates: type: array description: | allows define template for rendering `Service` which would get endpoint from Pods which scoped chi-wide, cluster-wide, shard-wide, replica-wide level # nullable: true items: type: object #required: # - name # - spec properties: name: type: string description: | template name, could use to link inside chi-level `chi.spec.defaults.templates.serviceTemplate` cluster-level `chi.spec.configuration.clusters.templates.clusterServiceTemplate` shard-level `chi.spec.configuration.clusters.layout.shards.temlates.shardServiceTemplate` replica-level `chi.spec.configuration.clusters.layout.replicas.templates.replicaServiceTemplate` or `chi.spec.configuration.clusters.layout.shards.replicas.replicaServiceTemplate` generateName: type: string description: | allows define format for generated `Service` name, look to https://github.com/Altinity/clickhouse-operator/blob/master/docs/custom_resource_explained.md#spectemplatesservicetemplates for details about available template variables" metadata: # TODO specify ObjectMeta type: object description: | allows pass standard object's metadata from template to Service Could be use for define specificly for Cloud Provider metadata which impact to behavior of service More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#metadata # nullable: true x-kubernetes-preserve-unknown-fields: true spec: # TODO specify ServiceSpec type: object description: | describe behavior of generated Service More info: https://kubernetes.io/docs/concepts/services-networking/service/ # nullable: true x-kubernetes-preserve-unknown-fields: true useTemplates: type: array description: | list of `ClickHouseInstallationTemplate` (chit) resource names which will merge with current `CHI` manifest during render Kubernetes resources to create related ClickHouse clusters" # nullable: true items: type: object #required: # - name properties: name: type: string description: "name of `ClickHouseInstallationTemplate` (chit) resource" namespace: type: string description: "Kubernetes namespace where need search `chit` resource, depending on `watchNamespaces` settings in `clichouse-operator`" useType: type: string description: "optional, current strategy is only merge, and current `chi` settings have more priority than merged template `chit`" enum: # List useTypeXXX constants from model - "" - "merge" --- # Template Parameters: # # NONE # apiVersion: apiextensions.k8s.io/v1 kind: CustomResourceDefinition metadata: name: clickhouseoperatorconfigurations.clickhouse.altinity.com labels: clickhouse.altinity.com/chop: 0.24.2 spec: group: clickhouse.altinity.com scope: Namespaced names: kind: ClickHouseOperatorConfiguration singular: clickhouseoperatorconfiguration plural: clickhouseoperatorconfigurations shortNames: - chopconf versions: - name: v1 served: true storage: true additionalPrinterColumns: - name: namespaces type: string description: Watch namespaces jsonPath: .status - name: age type: date description: Age of the resource # Displayed in all priorities jsonPath: .metadata.creationTimestamp schema: openAPIV3Schema: type: object description: "allows customize `clickhouse-operator` settings, need restart clickhouse-operator pod after adding, more details https://github.com/Altinity/clickhouse-operator/blob/master/docs/operator_configuration.md" x-kubernetes-preserve-unknown-fields: true properties: status: type: object x-kubernetes-preserve-unknown-fields: true spec: type: object description: | Allows to define settings of the clickhouse-operator. More info: https://github.com/Altinity/clickhouse-operator/blob/master/config/config.yaml Check into etc-clickhouse-operator* ConfigMaps if you need more control x-kubernetes-preserve-unknown-fields: true properties: watch: type: object description: "Parameters for watch kubernetes resources which used by clickhouse-operator deployment" properties: namespaces: type: array description: "List of namespaces where clickhouse-operator watches for events." items: type: string clickhouse: type: object description: "Clickhouse related parameters used by clickhouse-operator" properties: configuration: type: object properties: file: type: object properties: path: type: object description: | Each 'path' can be either absolute or relative. In case path is absolute - it is used as is. In case path is relative - it is relative to the folder where configuration file you are reading right now is located. properties: common: type: string description: | Path to the folder where ClickHouse configuration files common for all instances within a CHI are located. Default value - config.d host: type: string description: | Path to the folder where ClickHouse configuration files unique for each instance (host) within a CHI are located. Default value - conf.d user: type: string description: | Path to the folder where ClickHouse configuration files with users settings are located. Files are common for all instances within a CHI. Default value - users.d user: type: object description: "Default parameters for any user which will create" properties: default: type: object properties: profile: type: string description: "ClickHouse server configuration `...` for any " quota: type: string description: "ClickHouse server configuration `...` for any " networksIP: type: array description: "ClickHouse server configuration `...` for any " items: type: string password: type: string description: "ClickHouse server configuration `...` for any " network: type: object description: "Default network parameters for any user which will create" properties: hostRegexpTemplate: type: string description: "ClickHouse server configuration `...` for any " configurationRestartPolicy: type: object description: "Configuration restart policy describes what configuration changes require ClickHouse restart" properties: rules: type: array description: "Array of set of rules per specified ClickHouse versions" items: type: object properties: version: type: string description: "ClickHouse version expression" rules: type: array description: "Set of configuration rules for specified ClickHouse version" items: type: object description: "setting: value pairs for configuration restart policy" access: type: object description: "parameters which use for connect to clickhouse from clickhouse-operator deployment" properties: scheme: type: string description: "The scheme to user for connecting to ClickHouse. Possible values: http, https, auto" username: type: string description: "ClickHouse username to be used by operator to connect to ClickHouse instances, deprecated, use chCredentialsSecretName" password: type: string description: "ClickHouse password to be used by operator to connect to ClickHouse instances, deprecated, use chCredentialsSecretName" rootCA: type: string description: "Root certificate authority that clients use when verifying server certificates. Used for https connection to ClickHouse" secret: type: object properties: namespace: type: string description: "Location of k8s Secret with username and password to be used by operator to connect to ClickHouse instances" name: type: string description: "Name of k8s Secret with username and password to be used by operator to connect to ClickHouse instances" port: type: integer minimum: 1 maximum: 65535 description: "Port to be used by operator to connect to ClickHouse instances" timeouts: type: object description: "Timeouts used to limit connection and queries from the operator to ClickHouse instances, In seconds" properties: connect: type: integer minimum: 1 maximum: 10 description: "Timout to setup connection from the operator to ClickHouse instances. In seconds." query: type: integer minimum: 1 maximum: 600 description: "Timout to perform SQL query from the operator to ClickHouse instances. In seconds." metrics: type: object description: "parameters which use for connect to fetch metrics from clickhouse by clickhouse-operator" properties: timeouts: type: object description: | Timeouts used to limit connection and queries from the metrics exporter to ClickHouse instances Specified in seconds. properties: collect: type: integer minimum: 1 maximum: 600 description: | Timeout used to limit metrics collection request. In seconds. Upon reaching this timeout metrics collection is aborted and no more metrics are collected in this cycle. All collected metrics are returned. template: type: object description: "Parameters which are used if you want to generate ClickHouseInstallationTemplate custom resources from files which are stored inside clickhouse-operator deployment" properties: chi: type: object properties: policy: type: string description: | CHI template updates handling policy Possible policy values: - ReadOnStart. Accept CHIT updates on the operators start only. - ApplyOnNextReconcile. Accept CHIT updates at all time. Apply news CHITs on next regular reconcile of the CHI enum: - "" - "ReadOnStart" - "ApplyOnNextReconcile" path: type: string description: "Path to folder where ClickHouseInstallationTemplate .yaml manifests are located." reconcile: type: object description: "allow tuning reconciling process" properties: runtime: type: object description: "runtime parameters for clickhouse-operator process which are used during reconcile cycle" properties: reconcileCHIsThreadsNumber: type: integer minimum: 1 maximum: 65535 description: "How many goroutines will be used to reconcile CHIs in parallel, 10 by default" reconcileShardsThreadsNumber: type: integer minimum: 1 maximum: 65535 description: "How many goroutines will be used to reconcile shards of a cluster in parallel, 1 by default" reconcileShardsMaxConcurrencyPercent: type: integer minimum: 0 maximum: 100 description: "The maximum percentage of cluster shards that may be reconciled in parallel, 50 percent by default." statefulSet: type: object description: "Allow change default behavior for reconciling StatefulSet which generated by clickhouse-operator" properties: create: type: object description: "Behavior during create StatefulSet" properties: onFailure: type: string description: | What to do in case created StatefulSet is not in Ready after `statefulSetUpdateTimeout` seconds Possible options: 1. abort - do nothing, just break the process and wait for admin. 2. delete - delete newly created problematic StatefulSet. 3. ignore (default) - ignore error, pretend nothing happened and move on to the next StatefulSet. update: type: object description: "Behavior during update StatefulSet" properties: timeout: type: integer description: "How many seconds to wait for created/updated StatefulSet to be Ready" pollInterval: type: integer description: "How many seconds to wait between checks for created/updated StatefulSet status" onFailure: type: string description: | What to do in case updated StatefulSet is not in Ready after `statefulSetUpdateTimeout` seconds Possible options: 1. abort - do nothing, just break the process and wait for admin. 2. rollback (default) - delete Pod and rollback StatefulSet to previous Generation. Pod would be recreated by StatefulSet based on rollback-ed configuration. 3. ignore - ignore error, pretend nothing happened and move on to the next StatefulSet. host: type: object description: | Whether the operator during reconcile procedure should wait for a ClickHouse host: - to be excluded from a ClickHouse cluster - to complete all running queries - to be included into a ClickHouse cluster respectfully before moving forward properties: wait: type: object properties: exclude: &TypeStringBool type: string description: "Whether the operator during reconcile procedure should wait for a ClickHouse host to be excluded from a ClickHouse cluster" enum: # List StringBoolXXX constants from model - "" - "0" - "1" - "False" - "false" - "True" - "true" - "No" - "no" - "Yes" - "yes" - "Off" - "off" - "On" - "on" - "Disable" - "disable" - "Enable" - "enable" - "Disabled" - "disabled" - "Enabled" - "enabled" queries: <<: *TypeStringBool description: "Whether the operator during reconcile procedure should wait for a ClickHouse host to complete all running queries" include: <<: *TypeStringBool description: "Whether the operator during reconcile procedure should wait for a ClickHouse host to be included into a ClickHouse cluster" annotation: type: object description: "defines which metadata.annotations items will include or exclude during render StatefulSet, Pod, PVC resources" properties: include: type: array description: | When propagating labels from the chi's `metadata.annotations` section to child objects' `metadata.annotations`, include annotations with names from the following list items: type: string exclude: type: array description: | When propagating labels from the chi's `metadata.annotations` section to child objects' `metadata.annotations`, exclude annotations with names from the following list items: type: string label: type: object description: "defines which metadata.labels will include or exclude during render StatefulSet, Pod, PVC resources" properties: include: type: array description: | When propagating labels from the chi's `metadata.labels` section to child objects' `metadata.labels`, include labels from the following list items: type: string exclude: type: array items: type: string description: | When propagating labels from the chi's `metadata.labels` section to child objects' `metadata.labels`, exclude labels from the following list appendScope: <<: *TypeStringBool description: | Whether to append *Scope* labels to StatefulSet and Pod - "LabelShardScopeIndex" - "LabelReplicaScopeIndex" - "LabelCHIScopeIndex" - "LabelCHIScopeCycleSize" - "LabelCHIScopeCycleIndex" - "LabelCHIScopeCycleOffset" - "LabelClusterScopeIndex" - "LabelClusterScopeCycleSize" - "LabelClusterScopeCycleIndex" - "LabelClusterScopeCycleOffset" metrics: type: object description: "defines metrics exporter options" properties: labels: type: object description: "defines metric labels options" properties: exclude: type: array description: | When adding labels to a metric exclude labels with names from the following list items: type: string status: type: object description: "defines status options" properties: fields: type: object description: "defines status fields options" properties: action: <<: *TypeStringBool description: "Whether the operator should fill status field 'action'" actions: <<: *TypeStringBool description: "Whether the operator should fill status field 'actions'" error: <<: *TypeStringBool description: "Whether the operator should fill status field 'error'" errors: <<: *TypeStringBool description: "Whether the operator should fill status field 'errors'" statefulSet: type: object description: "define StatefulSet-specific parameters" properties: revisionHistoryLimit: type: integer description: | revisionHistoryLimit is the maximum number of revisions that will be maintained in the StatefulSet's revision history. Look details in `statefulset.spec.revisionHistoryLimit` pod: type: object description: "define pod specific parameters" properties: terminationGracePeriod: type: integer description: | Optional duration in seconds the pod needs to terminate gracefully. Look details in `pod.spec.terminationGracePeriodSeconds` logger: type: object description: "allow setup clickhouse-operator logger behavior" properties: logtostderr: type: string description: "boolean, allows logs to stderr" alsologtostderr: type: string description: "boolean allows logs to stderr and files both" v: type: string description: "verbosity level of clickhouse-operator log, default - 1 max - 9" stderrthreshold: type: string vmodule: type: string description: | Comma-separated list of filename=N, where filename (can be a pattern) must have no .go ext, and N is a V level. Ex.: file*=2 sets the 'V' to 2 in all files with names like file*. log_backtrace_at: type: string description: | It can be set to a file and line number with a logging line. Ex.: file.go:123 Each time when this line is being executed, a stack trace will be written to the Info log. --- # Template Parameters: # # OPERATOR_VERSION=0.24.2 # apiVersion: apiextensions.k8s.io/v1 kind: CustomResourceDefinition metadata: name: clickhousekeeperinstallations.clickhouse-keeper.altinity.com labels: clickhouse-keeper.altinity.com/chop: 0.24.2 spec: group: clickhouse-keeper.altinity.com scope: Namespaced names: kind: ClickHouseKeeperInstallation singular: clickhousekeeperinstallation plural: clickhousekeeperinstallations shortNames: - chk versions: - name: v1 served: true storage: true additionalPrinterColumns: - name: version type: string description: Operator version priority: 1 # show in wide view jsonPath: .status.chop-version - name: clusters type: integer description: Clusters count jsonPath: .status.clusters - name: shards type: integer description: Shards count priority: 1 # show in wide view jsonPath: .status.shards - name: hosts type: integer description: Hosts count jsonPath: .status.hosts - name: taskID type: string description: TaskID priority: 1 # show in wide view jsonPath: .status.taskID - name: status type: string description: Resource status jsonPath: .status.status - name: hosts-unchanged type: integer description: Unchanged hosts count priority: 1 # show in wide view jsonPath: .status.hostsUnchanged - name: hosts-updated type: integer description: Updated hosts count priority: 1 # show in wide view jsonPath: .status.hostsUpdated - name: hosts-added type: integer description: Added hosts count priority: 1 # show in wide view jsonPath: .status.hostsAdded - name: hosts-completed type: integer description: Completed hosts count jsonPath: .status.hostsCompleted - name: hosts-deleted type: integer description: Hosts deleted count priority: 1 # show in wide view jsonPath: .status.hostsDeleted - name: hosts-delete type: integer description: Hosts to be deleted count priority: 1 # show in wide view jsonPath: .status.hostsDelete - name: endpoint type: string description: Client access endpoint priority: 1 # show in wide view jsonPath: .status.endpoint - name: age type: date description: Age of the resource # Displayed in all priorities jsonPath: .metadata.creationTimestamp subresources: status: {} schema: openAPIV3Schema: description: "define a set of Kubernetes resources (StatefulSet, PVC, Service, ConfigMap) which describe behavior one or more clusters" type: object required: - spec properties: apiVersion: description: | APIVersion defines the versioned schema of this representation of an object. Servers should convert recognized schemas to the latest internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources type: string kind: description: | Kind is a string value representing the REST resource this object represents. Servers may infer this from the endpoint the client submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds type: string metadata: type: object status: type: object description: | Status contains many fields like a normalized configuration, clickhouse-operator version, current action and all applied action list, current taskID and all applied taskIDs and other properties: chop-version: type: string description: "Operator version" chop-commit: type: string description: "Operator git commit SHA" chop-date: type: string description: "Operator build date" chop-ip: type: string description: "IP address of the operator's pod which managed this resource" clusters: type: integer minimum: 0 description: "Clusters count" shards: type: integer minimum: 0 description: "Shards count" replicas: type: integer minimum: 0 description: "Replicas count" hosts: type: integer minimum: 0 description: "Hosts count" status: type: string description: "Status" taskID: type: string description: "Current task id" taskIDsStarted: type: array description: "Started task ids" nullable: true items: type: string taskIDsCompleted: type: array description: "Completed task ids" nullable: true items: type: string action: type: string description: "Action" actions: type: array description: "Actions" nullable: true items: type: string error: type: string description: "Last error" errors: type: array description: "Errors" nullable: true items: type: string hostsUnchanged: type: integer minimum: 0 description: "Unchanged Hosts count" hostsUpdated: type: integer minimum: 0 description: "Updated Hosts count" hostsAdded: type: integer minimum: 0 description: "Added Hosts count" hostsCompleted: type: integer minimum: 0 description: "Completed Hosts count" hostsDeleted: type: integer minimum: 0 description: "Deleted Hosts count" hostsDelete: type: integer minimum: 0 description: "About to delete Hosts count" pods: type: array description: "Pods" nullable: true items: type: string pod-ips: type: array description: "Pod IPs" nullable: true items: type: string fqdns: type: array description: "Pods FQDNs" nullable: true items: type: string endpoint: type: string description: "Endpoint" generation: type: integer minimum: 0 description: "Generation" normalized: type: object description: "Normalized resource requested" x-kubernetes-preserve-unknown-fields: true normalizedCompleted: type: object description: "Normalized resource completed" x-kubernetes-preserve-unknown-fields: true hostsWithTablesCreated: type: array description: "List of hosts with tables created by the operator" nullable: true items: type: string usedTemplates: type: array description: "List of templates used to build this CHI" nullable: true x-kubernetes-preserve-unknown-fields: true items: type: object x-kubernetes-preserve-unknown-fields: true spec: type: object # x-kubernetes-preserve-unknown-fields: true description: | Specification of the desired behavior of one or more ClickHouse clusters More info: https://github.com/Altinity/clickhouse-operator/blob/master/docs/custom_resource_explained.md properties: taskID: type: string description: | Allows to define custom taskID for CHI update and watch status of this update execution. Displayed in all .status.taskID* fields. By default (if not filled) every update of CHI manifest will generate random taskID stop: &TypeStringBool type: string description: | Allows to stop all ClickHouse clusters defined in a CHI. Works as the following: - When `stop` is `1` operator sets `Replicas: 0` in each StatefulSet. Thie leads to having all `Pods` and `Service` deleted. All PVCs are kept intact. - When `stop` is `0` operator sets `Replicas: 1` and `Pod`s and `Service`s will created again and all retained PVCs will be attached to `Pod`s. enum: # List StringBoolXXX constants from model - "" - "0" - "1" - "False" - "false" - "True" - "true" - "No" - "no" - "Yes" - "yes" - "Off" - "off" - "On" - "on" - "Disable" - "disable" - "Enable" - "enable" - "Disabled" - "disabled" - "Enabled" - "enabled" namespaceDomainPattern: type: string description: | Custom domain pattern which will be used for DNS names of `Service` or `Pod`. Typical use scenario - custom cluster domain in Kubernetes cluster Example: %s.svc.my.test reconciling: type: object description: "Optional, allows tuning reconciling cycle for ClickhouseInstallation from clickhouse-operator side" # nullable: true properties: policy: type: string description: | DISCUSSED TO BE DEPRECATED Syntax sugar Overrides all three 'reconcile.host.wait.{exclude, queries, include}' values from the operator's config Possible values: - wait - should wait to exclude host, complete queries and include host back into the cluster - nowait - should NOT wait to exclude host, complete queries and include host back into the cluster enum: - "" - "wait" - "nowait" configMapPropagationTimeout: type: integer description: | Timeout in seconds for `clickhouse-operator` to wait for modified `ConfigMap` to propagate into the `Pod` More details: https://kubernetes.io/docs/concepts/configuration/configmap/#mounted-configmaps-are-updated-automatically minimum: 0 maximum: 3600 cleanup: type: object description: "Optional, defines behavior for cleanup Kubernetes resources during reconcile cycle" # nullable: true properties: unknownObjects: type: object description: | Describes what clickhouse-operator should do with found Kubernetes resources which should be managed by clickhouse-operator, but do not have `ownerReference` to any currently managed `ClickHouseInstallation` resource. Default behavior is `Delete`" # nullable: true properties: statefulSet: &TypeObjectsCleanup type: string description: "Behavior policy for unknown StatefulSet, `Delete` by default" enum: # List ObjectsCleanupXXX constants from model - "" - "Retain" - "Delete" pvc: type: string <<: *TypeObjectsCleanup description: "Behavior policy for unknown PVC, `Delete` by default" configMap: <<: *TypeObjectsCleanup description: "Behavior policy for unknown ConfigMap, `Delete` by default" service: <<: *TypeObjectsCleanup description: "Behavior policy for unknown Service, `Delete` by default" reconcileFailedObjects: type: object description: | Describes what clickhouse-operator should do with Kubernetes resources which are failed during reconcile. Default behavior is `Retain`" # nullable: true properties: statefulSet: <<: *TypeObjectsCleanup description: "Behavior policy for failed StatefulSet, `Retain` by default" pvc: <<: *TypeObjectsCleanup description: "Behavior policy for failed PVC, `Retain` by default" configMap: <<: *TypeObjectsCleanup description: "Behavior policy for failed ConfigMap, `Retain` by default" service: <<: *TypeObjectsCleanup description: "Behavior policy for failed Service, `Retain` by default" defaults: type: object description: | define default behavior for whole ClickHouseInstallation, some behavior can be re-define on cluster, shard and replica level More info: https://github.com/Altinity/clickhouse-operator/blob/master/docs/custom_resource_explained.md#specdefaults # nullable: true properties: replicasUseFQDN: <<: *TypeStringBool description: | define should replicas be specified by FQDN in ``. In case of "no" will use short hostname and clickhouse-server will use kubernetes default suffixes for DNS lookup "yes" by default distributedDDL: type: object description: | allows change `` settings More info: https://clickhouse.tech/docs/en/operations/server-configuration-parameters/settings/#server-settings-distributed_ddl # nullable: true properties: profile: type: string description: "Settings from this profile will be used to execute DDL queries" storageManagement: type: object description: default storage management options properties: provisioner: &TypePVCProvisioner type: string description: "defines `PVC` provisioner - be it StatefulSet or the Operator" enum: - "" - "StatefulSet" - "Operator" reclaimPolicy: &TypePVCReclaimPolicy type: string description: | defines behavior of `PVC` deletion. `Delete` by default, if `Retain` specified then `PVC` will be kept when deleting StatefulSet enum: - "" - "Retain" - "Delete" templates: &TypeTemplateNames type: object description: "optional, configuration of the templates names which will use for generate Kubernetes resources according to one or more ClickHouse clusters described in current ClickHouseInstallation (chi) resource" # nullable: true properties: hostTemplate: type: string description: "optional, template name from chi.spec.templates.hostTemplates, which will apply to configure every `clickhouse-server` instance during render ConfigMap resources which will mount into `Pod`" podTemplate: type: string description: "optional, template name from chi.spec.templates.podTemplates, allows customization each `Pod` resource during render and reconcile each StatefulSet.spec resource described in `chi.spec.configuration.clusters`" dataVolumeClaimTemplate: type: string description: "optional, template name from chi.spec.templates.volumeClaimTemplates, allows customization each `PVC` which will mount for clickhouse data directory in each `Pod` during render and reconcile every StatefulSet.spec resource described in `chi.spec.configuration.clusters`" logVolumeClaimTemplate: type: string description: "optional, template name from chi.spec.templates.volumeClaimTemplates, allows customization each `PVC` which will mount for clickhouse log directory in each `Pod` during render and reconcile every StatefulSet.spec resource described in `chi.spec.configuration.clusters`" serviceTemplate: type: string description: "optional, template name from chi.spec.templates.serviceTemplates, allows customization for one `Service` resource which will created by `clickhouse-operator` which cover all clusters in whole `chi` resource" clusterServiceTemplate: type: string description: "optional, template name from chi.spec.templates.serviceTemplates, allows customization for each `Service` resource which will created by `clickhouse-operator` which cover each clickhouse cluster described in `chi.spec.configuration.clusters`" shardServiceTemplate: type: string description: "optional, template name from chi.spec.templates.serviceTemplates, allows customization for each `Service` resource which will created by `clickhouse-operator` which cover each shard inside clickhouse cluster described in `chi.spec.configuration.clusters`" replicaServiceTemplate: type: string description: "optional, template name from chi.spec.templates.serviceTemplates, allows customization for each `Service` resource which will created by `clickhouse-operator` which cover each replica inside each shard inside each clickhouse cluster described in `chi.spec.configuration.clusters`" volumeClaimTemplate: type: string description: "optional, alias for dataVolumeClaimTemplate, template name from chi.spec.templates.volumeClaimTemplates, allows customization each `PVC` which will mount for clickhouse data directory in each `Pod` during render and reconcile every StatefulSet.spec resource described in `chi.spec.configuration.clusters`" configuration: type: object description: "allows configure multiple aspects and behavior for `clickhouse-server` instance and also allows describe multiple `clickhouse-server` clusters inside one `chi` resource" # nullable: true properties: settings: &TypeSettings type: object description: | allows configure multiple aspects and behavior for `clickhouse-keeper` instance # nullable: true x-kubernetes-preserve-unknown-fields: true files: &TypeFiles type: object description: | allows define content of any setting # nullable: true x-kubernetes-preserve-unknown-fields: true clusters: type: array description: | describes clusters layout and allows change settings on cluster-level and replica-level # nullable: true items: type: object #required: # - name properties: name: type: string description: "cluster name, used to identify set of servers and wide used during generate names of related Kubernetes resources" minLength: 1 # See namePartClusterMaxLen const maxLength: 15 pattern: "^[a-zA-Z0-9-]{0,15}$" settings: <<: *TypeSettings description: | optional, allows configure `clickhouse-server` settings inside ... tag in each `Pod` only in one cluster during generate `ConfigMap` which will mount in `/etc/clickhouse-server/config.d/` override top-level `chi.spec.configuration.settings` More details: https://clickhouse.tech/docs/en/operations/settings/settings/ files: <<: *TypeFiles description: | optional, allows define content of any setting file inside each `Pod` on current cluster during generate `ConfigMap` which will mount in `/etc/clickhouse-server/config.d/` or `/etc/clickhouse-server/conf.d/` or `/etc/clickhouse-server/users.d/` override top-level `chi.spec.configuration.files` templates: <<: *TypeTemplateNames description: | optional, configuration of the templates names which will use for generate Kubernetes resources according to selected cluster override top-level `chi.spec.configuration.templates` layout: type: object description: | describe current cluster layout, how much shards in cluster, how much replica in shard allows override settings on each shard and replica separatelly # nullable: true properties: replicasCount: type: integer description: | how much replicas in each shards for current cluster will run in Kubernetes, each replica is a separate `StatefulSet` which contains only one `Pod` with `clickhouse-server` instance, every shard contains 1 replica by default" replicas: type: array description: "optional, allows override top-level `chi.spec.configuration` and cluster-level `chi.spec.configuration.clusters` configuration for each replica and each shard relates to selected replica, use it only if you fully understand what you do" # nullable: true items: type: object properties: name: type: string description: "optional, by default replica name is generated, but you can override it and setup custom name" minLength: 1 # See namePartShardMaxLen const maxLength: 15 pattern: "^[a-zA-Z0-9-]{0,15}$" settings: <<: *TypeSettings description: | optional, allows configure `clickhouse-server` settings inside ... tag in `Pod` only in one replica during generate `ConfigMap` which will mount in `/etc/clickhouse-server/conf.d/` override top-level `chi.spec.configuration.settings`, cluster-level `chi.spec.configuration.clusters.settings` and will ignore if shard-level `chi.spec.configuration.clusters.layout.shards` present More details: https://clickhouse.tech/docs/en/operations/settings/settings/ files: <<: *TypeFiles description: | optional, allows define content of any setting file inside each `Pod` only in one replica during generate `ConfigMap` which will mount in `/etc/clickhouse-server/config.d/` or `/etc/clickhouse-server/conf.d/` or `/etc/clickhouse-server/users.d/` override top-level `chi.spec.configuration.files` and cluster-level `chi.spec.configuration.clusters.files`, will ignore if `chi.spec.configuration.clusters.layout.shards` presents templates: <<: *TypeTemplateNames description: | optional, configuration of the templates names which will use for generate Kubernetes resources according to selected replica override top-level `chi.spec.configuration.templates`, cluster-level `chi.spec.configuration.clusters.templates` shardsCount: type: integer description: "optional, count of shards related to current replica, you can override each shard behavior on low-level `chi.spec.configuration.clusters.layout.replicas.shards`" minimum: 1 shards: type: array description: "optional, list of shards related to current replica, will ignore if `chi.spec.configuration.clusters.layout.shards` presents" # nullable: true items: # Host type: object properties: name: type: string description: "optional, by default shard name is generated, but you can override it and setup custom name" minLength: 1 # See namePartReplicaMaxLen const maxLength: 15 pattern: "^[a-zA-Z0-9-]{0,15}$" zkPort: type: integer minimum: 1 maximum: 65535 raftPort: type: integer minimum: 1 maximum: 65535 settings: <<: *TypeSettings description: | optional, allows configure `clickhouse-server` settings inside ... tag in `Pod` only in one shard related to current replica during generate `ConfigMap` which will mount in `/etc/clickhouse-server/conf.d/` override top-level `chi.spec.configuration.settings`, cluster-level `chi.spec.configuration.clusters.settings` and replica-level `chi.spec.configuration.clusters.layout.replicas.settings` More details: https://clickhouse.tech/docs/en/operations/settings/settings/ files: <<: *TypeFiles description: | optional, allows define content of any setting file inside each `Pod` only in one shard related to current replica during generate `ConfigMap` which will mount in `/etc/clickhouse-server/config.d/` or `/etc/clickhouse-server/conf.d/` or `/etc/clickhouse-server/users.d/` override top-level `chi.spec.configuration.files` and cluster-level `chi.spec.configuration.clusters.files`, will ignore if `chi.spec.configuration.clusters.layout.shards` presents templates: <<: *TypeTemplateNames description: | optional, configuration of the templates names which will use for generate Kubernetes resources according to selected replica override top-level `chi.spec.configuration.templates`, cluster-level `chi.spec.configuration.clusters.templates`, replica-level `chi.spec.configuration.clusters.layout.replicas.templates` templates: type: object description: "allows define templates which will use for render Kubernetes resources like StatefulSet, ConfigMap, Service, PVC, by default, clickhouse-operator have own templates, but you can override it" # nullable: true properties: hostTemplates: type: array description: "hostTemplate will use during apply to generate `clickhose-server` config files" # nullable: true items: type: object #required: # - name properties: name: description: "template name, could use to link inside top-level `chi.spec.defaults.templates.hostTemplate`, cluster-level `chi.spec.configuration.clusters.templates.hostTemplate`, shard-level `chi.spec.configuration.clusters.layout.shards.temlates.hostTemplate`, replica-level `chi.spec.configuration.clusters.layout.replicas.templates.hostTemplate`" type: string portDistribution: type: array description: "define how will distribute numeric values of named ports in `Pod.spec.containers.ports` and clickhouse-server configs" # nullable: true items: type: object #required: # - type properties: type: type: string description: "type of distribution, when `Unspecified` (default value) then all listen ports on clickhouse-server configuration in all Pods will have the same value, when `ClusterScopeIndex` then ports will increment to offset from base value depends on shard and replica index inside cluster with combination of `chi.spec.templates.podTemlates.spec.HostNetwork` it allows setup ClickHouse cluster inside Kubernetes and provide access via external network bypass Kubernetes internal network" enum: # List PortDistributionXXX constants - "" - "Unspecified" - "ClusterScopeIndex" spec: # Host type: object properties: name: type: string description: "by default, hostname will generate, but this allows define custom name for each `clickhuse-server`" minLength: 1 # See namePartReplicaMaxLen const maxLength: 15 pattern: "^[a-zA-Z0-9-]{0,15}$" zkPort: type: integer minimum: 1 maximum: 65535 raftPort: type: integer minimum: 1 maximum: 65535 settings: <<: *TypeSettings description: | optional, allows configure `clickhouse-server` settings inside ... tag in each `Pod` where this template will apply during generate `ConfigMap` which will mount in `/etc/clickhouse-server/conf.d/` More details: https://clickhouse.tech/docs/en/operations/settings/settings/ files: <<: *TypeFiles description: | optional, allows define content of any setting file inside each `Pod` where this template will apply during generate `ConfigMap` which will mount in `/etc/clickhouse-server/config.d/` or `/etc/clickhouse-server/conf.d/` or `/etc/clickhouse-server/users.d/` templates: <<: *TypeTemplateNames description: "be careful, this part of CRD allows override template inside template, don't use it if you don't understand what you do" podTemplates: type: array description: | podTemplate will use during render `Pod` inside `StatefulSet.spec` and allows define rendered `Pod.spec`, pod scheduling distribution and pod zone More information: https://github.com/Altinity/clickhouse-operator/blob/master/docs/custom_resource_explained.md#spectemplatespodtemplates # nullable: true items: type: object #required: # - name properties: name: type: string description: "template name, could use to link inside top-level `chi.spec.defaults.templates.podTemplate`, cluster-level `chi.spec.configuration.clusters.templates.podTemplate`, shard-level `chi.spec.configuration.clusters.layout.shards.temlates.podTemplate`, replica-level `chi.spec.configuration.clusters.layout.replicas.templates.podTemplate`" generateName: type: string description: "allows define format for generated `Pod` name, look to https://github.com/Altinity/clickhouse-operator/blob/master/docs/custom_resource_explained.md#spectemplatesservicetemplates for details about available template variables" zone: type: object description: "allows define custom zone name and will separate ClickHouse `Pods` between nodes, shortcut for `chi.spec.templates.podTemplates.spec.affinity.podAntiAffinity`" #required: # - values properties: key: type: string description: "optional, if defined, allows select kubernetes nodes by label with `name` equal `key`" values: type: array description: "optional, if defined, allows select kubernetes nodes by label with `value` in `values`" # nullable: true items: type: string distribution: type: string description: "DEPRECATED, shortcut for `chi.spec.templates.podTemplates.spec.affinity.podAntiAffinity`" enum: - "" - "Unspecified" - "OnePerHost" podDistribution: type: array description: "define ClickHouse Pod distribution policy between Kubernetes Nodes inside Shard, Replica, Namespace, CHI, another ClickHouse cluster" # nullable: true items: type: object #required: # - type properties: type: type: string description: "you can define multiple affinity policy types" enum: # List PodDistributionXXX constants - "" - "Unspecified" - "ClickHouseAntiAffinity" - "ShardAntiAffinity" - "ReplicaAntiAffinity" - "AnotherNamespaceAntiAffinity" - "AnotherClickHouseInstallationAntiAffinity" - "AnotherClusterAntiAffinity" - "MaxNumberPerNode" - "NamespaceAffinity" - "ClickHouseInstallationAffinity" - "ClusterAffinity" - "ShardAffinity" - "ReplicaAffinity" - "PreviousTailAffinity" - "CircularReplication" scope: type: string description: "scope for apply each podDistribution" enum: # list PodDistributionScopeXXX constants - "" - "Unspecified" - "Shard" - "Replica" - "Cluster" - "ClickHouseInstallation" - "Namespace" number: type: integer description: "define, how much ClickHouse Pods could be inside selected scope with selected distribution type" minimum: 0 maximum: 65535 topologyKey: type: string description: | use for inter-pod affinity look to `pod.spec.affinity.podAntiAffinity.preferredDuringSchedulingIgnoredDuringExecution.podAffinityTerm.topologyKey`, more info: https://kubernetes.io/docs/concepts/scheduling-eviction/assign-pod-node/#inter-pod-affinity-and-anti-affinity" metadata: type: object description: | allows pass standard object's metadata from template to Pod More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#metadata # nullable: true x-kubernetes-preserve-unknown-fields: true spec: # TODO specify PodSpec type: object description: "allows define whole Pod.spec inside StaefulSet.spec, look to https://kubernetes.io/docs/concepts/workloads/pods/#pod-templates for details" # nullable: true x-kubernetes-preserve-unknown-fields: true volumeClaimTemplates: type: array description: | allows define template for rendering `PVC` kubernetes resource, which would use inside `Pod` for mount clickhouse `data`, clickhouse `logs` or something else # nullable: true items: type: object #required: # - name # - spec properties: name: type: string description: | template name, could use to link inside top-level `chi.spec.defaults.templates.dataVolumeClaimTemplate` or `chi.spec.defaults.templates.logVolumeClaimTemplate`, cluster-level `chi.spec.configuration.clusters.templates.dataVolumeClaimTemplate` or `chi.spec.configuration.clusters.templates.logVolumeClaimTemplate`, shard-level `chi.spec.configuration.clusters.layout.shards.temlates.dataVolumeClaimTemplate` or `chi.spec.configuration.clusters.layout.shards.temlates.logVolumeClaimTemplate` replica-level `chi.spec.configuration.clusters.layout.replicas.templates.dataVolumeClaimTemplate` or `chi.spec.configuration.clusters.layout.replicas.templates.logVolumeClaimTemplate` provisioner: *TypePVCProvisioner reclaimPolicy: *TypePVCReclaimPolicy metadata: type: object description: | allows to pass standard object's metadata from template to PVC More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#metadata # nullable: true x-kubernetes-preserve-unknown-fields: true spec: type: object description: | allows define all aspects of `PVC` resource More info: https://kubernetes.io/docs/concepts/storage/persistent-volumes/#persistentvolumeclaims # nullable: true x-kubernetes-preserve-unknown-fields: true serviceTemplates: type: array description: | allows define template for rendering `Service` which would get endpoint from Pods which scoped chi-wide, cluster-wide, shard-wide, replica-wide level # nullable: true items: type: object #required: # - name # - spec properties: name: type: string description: | template name, could use to link inside chi-level `chi.spec.defaults.templates.serviceTemplate` cluster-level `chi.spec.configuration.clusters.templates.clusterServiceTemplate` shard-level `chi.spec.configuration.clusters.layout.shards.temlates.shardServiceTemplate` replica-level `chi.spec.configuration.clusters.layout.replicas.templates.replicaServiceTemplate` or `chi.spec.configuration.clusters.layout.shards.replicas.replicaServiceTemplate` generateName: type: string description: | allows define format for generated `Service` name, look to https://github.com/Altinity/clickhouse-operator/blob/master/docs/custom_resource_explained.md#spectemplatesservicetemplates for details about available template variables" metadata: # TODO specify ObjectMeta type: object description: | allows pass standard object's metadata from template to Service Could be use for define specificly for Cloud Provider metadata which impact to behavior of service More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#metadata # nullable: true x-kubernetes-preserve-unknown-fields: true spec: # TODO specify ServiceSpec type: object description: | describe behavior of generated Service More info: https://kubernetes.io/docs/concepts/services-networking/service/ # nullable: true x-kubernetes-preserve-unknown-fields: true --- # Template Parameters: # # COMMENT= # NAMESPACE=${OPERATOR_NAMESPACE} # NAME=clickhouse-operator # # Setup ServiceAccount apiVersion: v1 kind: ServiceAccount metadata: name: clickhouse-operator namespace: ${OPERATOR_NAMESPACE} labels: clickhouse.altinity.com/chop: 0.24.2 --- # Template Parameters: # # NAMESPACE=${OPERATOR_NAMESPACE} # COMMENT=# # ROLE_KIND=ClusterRole # ROLE_NAME=clickhouse-operator-${OPERATOR_NAMESPACE} # ROLE_BINDING_KIND=ClusterRoleBinding # ROLE_BINDING_NAME=clickhouse-operator-${OPERATOR_NAMESPACE} # # Specifies either # ClusterRole # or # Role # to be bound to ServiceAccount. # ClusterRole is namespace-less and must have unique name # Role is namespace-bound apiVersion: rbac.authorization.k8s.io/v1 kind: ClusterRole metadata: name: clickhouse-operator-${OPERATOR_NAMESPACE} #namespace: ${OPERATOR_NAMESPACE} labels: clickhouse.altinity.com/chop: 0.24.2 rules: # # Core API group # - apiGroups: - "" resources: - configmaps - services - persistentvolumeclaims - secrets verbs: - get - list - patch - update - watch - create - delete - apiGroups: - "" resources: - endpoints verbs: - get - list - watch - apiGroups: - "" resources: - events verbs: - create - apiGroups: - "" resources: - persistentvolumes verbs: - get - list - patch - update - watch - apiGroups: - "" resources: - pods verbs: - get - list - patch - update - watch - delete - apiGroups: - "" resources: - secrets verbs: - get - list # # apps.* resources # - apiGroups: - apps resources: - statefulsets verbs: - get - list - patch - update - watch - create - delete - apiGroups: - apps resources: - replicasets verbs: - get - patch - update - delete # The operator deployment personally, identified by name - apiGroups: - apps resources: - deployments resourceNames: - clickhouse-operator verbs: - get - patch - update - delete # # policy.* resources # - apiGroups: - policy resources: - poddisruptionbudgets verbs: - get - list - patch - update - watch - create - delete # # apiextensions # - apiGroups: - apiextensions.k8s.io resources: - customresourcedefinitions verbs: - get - list # # The operator's specific Custom Resources # # clickhouse - related resources - apiGroups: - clickhouse.altinity.com resources: - clickhouseinstallations verbs: - get - list - watch - patch - update - delete - apiGroups: - clickhouse.altinity.com resources: - clickhouseinstallationtemplates - clickhouseoperatorconfigurations verbs: - get - list - watch - apiGroups: - clickhouse.altinity.com resources: - clickhouseinstallations/finalizers - clickhouseinstallationtemplates/finalizers - clickhouseoperatorconfigurations/finalizers verbs: - update - apiGroups: - clickhouse.altinity.com resources: - clickhouseinstallations/status - clickhouseinstallationtemplates/status - clickhouseoperatorconfigurations/status verbs: - get - update - patch - create - delete # clickhouse-keeper - related resources - apiGroups: - clickhouse-keeper.altinity.com resources: - clickhousekeeperinstallations verbs: - get - list - watch - patch - update - delete - apiGroups: - clickhouse-keeper.altinity.com resources: - clickhousekeeperinstallations/finalizers verbs: - update - apiGroups: - clickhouse-keeper.altinity.com resources: - clickhousekeeperinstallations/status verbs: - get - update - patch - create - delete --- # Specifies either # ClusterRoleBinding between ClusterRole and ServiceAccount. # or # RoleBinding between Role and ServiceAccount. # ClusterRoleBinding is namespace-less and must have unique name # RoleBinding is namespace-bound apiVersion: rbac.authorization.k8s.io/v1 kind: ClusterRoleBinding metadata: name: clickhouse-operator-${OPERATOR_NAMESPACE} #namespace: ${OPERATOR_NAMESPACE} labels: clickhouse.altinity.com/chop: 0.24.2 roleRef: apiGroup: rbac.authorization.k8s.io kind: ClusterRole name: clickhouse-operator-${OPERATOR_NAMESPACE} subjects: - kind: ServiceAccount name: clickhouse-operator namespace: ${OPERATOR_NAMESPACE} --- # Template Parameters: # # NAME=etc-clickhouse-operator-files # NAMESPACE=${OPERATOR_NAMESPACE} # COMMENT= # apiVersion: v1 kind: ConfigMap metadata: name: etc-clickhouse-operator-files namespace: ${OPERATOR_NAMESPACE} labels: clickhouse.altinity.com/chop: 0.24.2 app: clickhouse-operator data: config.yaml: | # IMPORTANT # This file is auto-generated # Do not edit this file - all changes would be lost # Edit appropriate template in the following folder: # deploy/builder/templates-config # IMPORTANT # # Template parameters available: # WATCH_NAMESPACES= # CH_USERNAME_PLAIN= # CH_PASSWORD_PLAIN= # CH_CREDENTIALS_SECRET_NAMESPACE= # CH_CREDENTIALS_SECRET_NAME=clickhouse-operator # VERBOSITY=1 ################################################ ## ## Watch section ## ################################################ watch: # List of namespaces where clickhouse-operator watches for events. # Concurrently running operators should watch on different namespaces. # IMPORTANT # Regexp is applicable. #namespaces: ["dev", "test"] namespaces: [] clickhouse: configuration: ################################################ ## ## Configuration files section ## ################################################ file: # Each 'path' can be either absolute or relative. # In case path is absolute - it is used as is # In case path is relative - it is relative to the folder where configuration file you are reading right now is located. path: # Path to the folder where ClickHouse configuration files common for all instances within a CHI are located. common: chi/config.d # Path to the folder where ClickHouse configuration files unique for each instance (host) within a CHI are located. host: chi/conf.d # Path to the folder where ClickHouse configuration files with users' settings are located. # Files are common for all instances within a CHI. user: chi/users.d ################################################ ## ## Configuration users section ## ################################################ user: # Default settings for user accounts, created by the operator. # IMPORTANT. These are not access credentials or settings for 'default' user account, # it is a template for filling out missing fields for all user accounts to be created by the operator, # with the following EXCEPTIONS: # 1. 'default' user account DOES NOT use provided password, but uses all the rest of the fields. # Password for 'default' user account has to be provided explicitly, if to be used. # 2. CHOP user account DOES NOT use: # - profile setting. It uses predefined profile called 'clickhouse_operator' # - quota setting. It uses empty quota name. # - networks IP setting. Operator specifies 'networks/ip' user setting to match operators' pod IP only. # - password setting. Password for CHOP account is used from 'clickhouse.access.*' section default: # Default values for ClickHouse user account(s) created by the operator # 1. user/profile - string # 2. user/quota - string # 3. user/networks/ip - multiple strings # 4. user/password - string # These values can be overwritten on per-user basis. profile: "default" quota: "default" networksIP: - "::1" - "127.0.0.1" password: "default" ################################################ ## ## Configuration network section ## ################################################ network: # Default host_regexp to limit network connectivity from outside hostRegexpTemplate: "(chi-{chi}-[^.]+\\d+-\\d+|clickhouse\\-{chi})\\.{namespace}\\.svc\\.cluster\\.local$" ################################################ ## ## Configuration restart policy section ## Configuration restart policy describes what configuration changes require ClickHouse restart ## ################################################ configurationRestartPolicy: rules: # IMPORTANT! # Special version of "*" - default version - has to satisfy all ClickHouse versions. # Default version will also be used in case ClickHouse version is unknown. # ClickHouse version may be unknown due to host being down - for example, because of incorrect "settings" section. # ClickHouse is not willing to start in case incorrect/unknown settings are provided in config file. - version: "*" rules: # see https://kb.altinity.com/altinity-kb-setup-and-maintenance/altinity-kb-server-config-files/#server-config-configxml-sections-which-dont-require-restart # to be replaced with "select * from system.server_settings where changeable_without_restart = 'No'" - settings/*: "yes" # single values - settings/access_control_path: "no" - settings/dictionaries_config: "no" - settings/max_server_memory_*: "no" - settings/max_*_to_drop: "no" - settings/max_concurrent_queries: "no" - settings/models_config: "no" - settings/user_defined_executable_functions_config: "no" # structured XML - settings/logger/*: "no" - settings/macros/*: "no" - settings/remote_servers/*: "no" - settings/user_directories/*: "no" - zookeeper/*: "yes" - files/*.xml: "yes" - files/config.d/*.xml: "yes" - files/config.d/*dict*.xml: "no" # exceptions in default profile - profiles/default/background_*_pool_size: "yes" - profiles/default/max_*_for_server: "yes" - version: "21.*" rules: - settings/logger: "yes" ################################################# ## ## Access to ClickHouse instances ## ################################################ access: # Possible values for 'scheme' are: # 1. http - force http to be used to connect to ClickHouse instances # 2. https - force https to be used to connect to ClickHouse instances # 3. auto - either http or https is selected based on open ports scheme: "auto" # ClickHouse credentials (username, password and port) to be used by the operator to connect to ClickHouse instances. # These credentials are used for: # 1. Metrics requests # 2. Schema maintenance # User with these credentials can be specified in additional ClickHouse .xml config files, # located in 'clickhouse.configuration.file.path.user' folder username: "" password: "" rootCA: "" # Location of the k8s Secret with username and password to be used by the operator to connect to ClickHouse instances. # Can be used instead of explicitly specified username and password available in sections: # - clickhouse.access.username # - clickhouse.access.password # Secret should have two keys: # 1. username # 2. password secret: # Empty `namespace` means that k8s secret would be looked in the same namespace where operator's pod is running. namespace: "" # Empty `name` means no k8s Secret would be looked for name: "clickhouse-operator" # Port where to connect to ClickHouse instances to port: 8123 # Timeouts used to limit connection and queries from the operator to ClickHouse instances # Specified in seconds. timeouts: # Timout to setup connection from the operator to ClickHouse instances. In seconds. connect: 1 # Timout to perform SQL query from the operator to ClickHouse instances. In seconds. query: 4 ################################################# ## ## Metrics collection ## ################################################ metrics: # Timeouts used to limit connection and queries from the metrics exporter to ClickHouse instances # Specified in seconds. timeouts: # Timeout used to limit metrics collection request. In seconds. # Upon reaching this timeout metrics collection is aborted and no more metrics are collected in this cycle. # All collected metrics are returned. collect: 9 keeper: configuration: ################################################ ## ## Configuration files section ## ################################################ file: # Each 'path' can be either absolute or relative. # In case path is absolute - it is used as is # In case path is relative - it is relative to the folder where configuration file you are reading right now is located. path: # Path to the folder where Keeper configuration files common for all instances within a CHK are located. common: chk/keeper_config.d # Path to the folder where Keeper configuration files unique for each instance (host) within a CHK are located. host: chk/conf.d # Path to the folder where Keeper configuration files with users' settings are located. # Files are common for all instances within a CHI. user: chk/users.d ################################################ ## ## Template(s) management section ## ################################################ template: chi: # CHI template updates handling policy # Possible policy values: # - ReadOnStart. Accept CHIT updates on the operators start only. # - ApplyOnNextReconcile. Accept CHIT updates at all time. Apply news CHITs on next regular reconcile of the CHI policy: ApplyOnNextReconcile # Path to the folder where ClickHouseInstallation templates .yaml manifests are located. # Templates are added to the list of all templates and used when CHI is reconciled. # Templates are applied in sorted alpha-numeric order. path: chi/templates.d chk: # CHK template updates handling policy # Possible policy values: # - ReadOnStart. Accept CHIT updates on the operators start only. # - ApplyOnNextReconcile. Accept CHIT updates at all time. Apply news CHITs on next regular reconcile of the CHI policy: ApplyOnNextReconcile # Path to the folder where ClickHouseInstallation templates .yaml manifests are located. # Templates are added to the list of all templates and used when CHI is reconciled. # Templates are applied in sorted alpha-numeric order. path: chk/templates.d ################################################ ## ## Reconcile section ## ################################################ reconcile: # Reconcile runtime settings runtime: # Max number of concurrent CHI reconciles in progress reconcileCHIsThreadsNumber: 10 # The operator reconciles shards concurrently in each CHI with the following limitations: # 1. Number of shards being reconciled (and thus having hosts down) in each CHI concurrently # can not be greater than 'reconcileShardsThreadsNumber'. # 2. Percentage of shards being reconciled (and thus having hosts down) in each CHI concurrently # can not be greater than 'reconcileShardsMaxConcurrencyPercent'. # 3. The first shard is always reconciled alone. Concurrency starts from the second shard and onward. # Thus limiting number of shards being reconciled (and thus having hosts down) in each CHI by both number and percentage # Max number of concurrent shard reconciles within one CHI in progress reconcileShardsThreadsNumber: 5 # Max percentage of concurrent shard reconciles within one CHI in progress reconcileShardsMaxConcurrencyPercent: 50 # Reconcile StatefulSet scenario statefulSet: # Create StatefulSet scenario create: # What to do in case created StatefulSet is not in 'Ready' after `reconcile.statefulSet.update.timeout` seconds # Possible options: # 1. abort - abort the process, do nothing with the problematic StatefulSet, leave it as it is, # do not try to fix or delete or update it, just abort reconcile cycle. # Do not proceed to the next StatefulSet(s) and wait for an admin to assist. # 2. delete - delete newly created problematic StatefulSet and follow 'abort' path afterwards. # 3. ignore - ignore an error, pretend nothing happened, continue reconcile and move on to the next StatefulSet. onFailure: ignore # Update StatefulSet scenario update: # How many seconds to wait for created/updated StatefulSet to be 'Ready' timeout: 300 # How many seconds to wait between checks/polls for created/updated StatefulSet status pollInterval: 5 # What to do in case updated StatefulSet is not in 'Ready' after `reconcile.statefulSet.update.timeout` seconds # Possible options: # 1. abort - abort the process, do nothing with the problematic StatefulSet, leave it as it is, # do not try to fix or delete or update it, just abort reconcile cycle. # Do not proceed to the next StatefulSet(s) and wait for an admin to assist. # 2. rollback - delete Pod and rollback StatefulSet to previous Generation. # Pod would be recreated by StatefulSet based on rollback-ed StatefulSet configuration. # Follow 'abort' path afterwards. # 3. ignore - ignore an error, pretend nothing happened, continue reconcile and move on to the next StatefulSet. onFailure: abort # Reconcile Host scenario host: # Whether the operator during reconcile procedure should wait for a ClickHouse host: # - to be excluded from a ClickHouse cluster # - to complete all running queries # - to be included into a ClickHouse cluster # respectfully before moving forward wait: exclude: true queries: true include: false ################################################ ## ## Annotations management section ## ################################################ annotation: # Applied when: # 1. Propagating annotations from the CHI's `metadata.annotations` to child objects' `metadata.annotations`, # 2. Propagating annotations from the CHI Template's `metadata.annotations` to CHI's `metadata.annotations`, # Include annotations from the following list: # Applied only when not empty. Empty list means "include all, no selection" include: [] # Exclude annotations from the following list: exclude: [] ################################################ ## ## Labels management section ## ################################################ label: # Applied when: # 1. Propagating labels from the CHI's `metadata.labels` to child objects' `metadata.labels`, # 2. Propagating labels from the CHI Template's `metadata.labels` to CHI's `metadata.labels`, # Include labels from the following list: # Applied only when not empty. Empty list means "include all, no selection" include: [] # Exclude labels from the following list: # Applied only when not empty. Empty list means "nothing to exclude, no selection" exclude: [] # Whether to append *Scope* labels to StatefulSet and Pod. # Full list of available *scope* labels check in 'labeler.go' # LabelShardScopeIndex # LabelReplicaScopeIndex # LabelCHIScopeIndex # LabelCHIScopeCycleSize # LabelCHIScopeCycleIndex # LabelCHIScopeCycleOffset # LabelClusterScopeIndex # LabelClusterScopeCycleSize # LabelClusterScopeCycleIndex # LabelClusterScopeCycleOffset appendScope: "no" ################################################ ## ## Metrics management section ## ################################################ metrics: labels: exclude: [] ################################################ ## ## Status management section ## ################################################ status: fields: action: false actions: false error: false errors: false ################################################ ## ## StatefulSet management section ## ################################################ statefulSet: revisionHistoryLimit: 0 ################################################ ## ## Pod management section ## ################################################ pod: # Grace period for Pod termination. # How many seconds to wait between sending # SIGTERM and SIGKILL during Pod termination process. # Increase this number is case of slow shutdown. terminationGracePeriod: 30 ################################################ ## ## Log parameters section ## ################################################ logger: logtostderr: "true" alsologtostderr: "false" v: "1" stderrthreshold: "" vmodule: "" log_backtrace_at: "" --- # Template Parameters: # # NAME=etc-clickhouse-operator-confd-files # NAMESPACE=${OPERATOR_NAMESPACE} # COMMENT= # apiVersion: v1 kind: ConfigMap metadata: name: etc-clickhouse-operator-confd-files namespace: ${OPERATOR_NAMESPACE} labels: clickhouse.altinity.com/chop: 0.24.2 app: clickhouse-operator data: --- # Template Parameters: # # NAME=etc-clickhouse-operator-configd-files # NAMESPACE=${OPERATOR_NAMESPACE} # COMMENT= # apiVersion: v1 kind: ConfigMap metadata: name: etc-clickhouse-operator-configd-files namespace: ${OPERATOR_NAMESPACE} labels: clickhouse.altinity.com/chop: 0.24.2 app: clickhouse-operator data: 01-clickhouse-01-listen.xml: | :: 0.0.0.0 1 01-clickhouse-02-logger.xml: | debug /var/log/clickhouse-server/clickhouse-server.log /var/log/clickhouse-server/clickhouse-server.err.log 1000M 10 1 01-clickhouse-03-query_log.xml: | system query_log
Engine = MergeTree PARTITION BY event_date ORDER BY event_time TTL event_date + interval 30 day 7500
01-clickhouse-04-part_log.xml: | system part_log
Engine = MergeTree PARTITION BY event_date ORDER BY event_time TTL event_date + interval 30 day 7500
01-clickhouse-05-trace_log.xml: | system trace_log
Engine = MergeTree PARTITION BY event_date ORDER BY event_time TTL event_date + interval 30 day 7500
--- # Template Parameters: # # NAME=etc-clickhouse-operator-templatesd-files # NAMESPACE=${OPERATOR_NAMESPACE} # COMMENT= # apiVersion: v1 kind: ConfigMap metadata: name: etc-clickhouse-operator-templatesd-files namespace: ${OPERATOR_NAMESPACE} labels: clickhouse.altinity.com/chop: 0.24.2 app: clickhouse-operator data: 001-templates.json.example: | { "apiVersion": "clickhouse.altinity.com/v1", "kind": "ClickHouseInstallationTemplate", "metadata": { "name": "01-default-volumeclaimtemplate" }, "spec": { "templates": { "volumeClaimTemplates": [ { "name": "chi-default-volume-claim-template", "spec": { "accessModes": [ "ReadWriteOnce" ], "resources": { "requests": { "storage": "2Gi" } } } } ], "podTemplates": [ { "name": "chi-default-oneperhost-pod-template", "distribution": "OnePerHost", "spec": { "containers" : [ { "name": "clickhouse", "image": "clickhouse/clickhouse-server:23.8", "ports": [ { "name": "http", "containerPort": 8123 }, { "name": "client", "containerPort": 9000 }, { "name": "interserver", "containerPort": 9009 } ] } ] } } ] } } } default-pod-template.yaml.example: | apiVersion: "clickhouse.altinity.com/v1" kind: "ClickHouseInstallationTemplate" metadata: name: "default-oneperhost-pod-template" spec: templates: podTemplates: - name: default-oneperhost-pod-template distribution: "OnePerHost" default-storage-template.yaml.example: | apiVersion: "clickhouse.altinity.com/v1" kind: "ClickHouseInstallationTemplate" metadata: name: "default-storage-template-2Gi" spec: templates: volumeClaimTemplates: - name: default-storage-template-2Gi spec: accessModes: - ReadWriteOnce resources: requests: storage: 2Gi readme: | Templates in this folder are packaged with an operator and available via 'useTemplate' --- # Template Parameters: # # NAME=etc-clickhouse-operator-usersd-files # NAMESPACE=${OPERATOR_NAMESPACE} # COMMENT= # apiVersion: v1 kind: ConfigMap metadata: name: etc-clickhouse-operator-usersd-files namespace: ${OPERATOR_NAMESPACE} labels: clickhouse.altinity.com/chop: 0.24.2 app: clickhouse-operator data: 01-clickhouse-operator-profile.xml: | 0 1 10 0 0 02-clickhouse-default-profile.xml: | 2 1 1000 1 1 1 nearest_hostname 0 --- # Template Parameters: # # NAME=etc-keeper-operator-confd-files # NAMESPACE=${OPERATOR_NAMESPACE} # COMMENT= # apiVersion: v1 kind: ConfigMap metadata: name: etc-keeper-operator-confd-files namespace: ${OPERATOR_NAMESPACE} labels: clickhouse.altinity.com/chop: 0.24.2 app: clickhouse-operator data: --- # Template Parameters: # # NAME=etc-keeper-operator-configd-files # NAMESPACE=${OPERATOR_NAMESPACE} # COMMENT= # apiVersion: v1 kind: ConfigMap metadata: name: etc-keeper-operator-configd-files namespace: ${OPERATOR_NAMESPACE} labels: clickhouse.altinity.com/chop: 0.24.2 app: clickhouse-operator data: 01-keeper-01-default-config.xml: | 10000 10000 information 100000 true /var/lib/clickhouse-keeper/coordination/logs /var/lib/clickhouse-keeper/coordination/snapshots /var/lib/clickhouse-keeper 2181 :: 0.0.0.0 1 1 information 4096 01-keeper-02-readiness.xml: | 9182 /ready 01-keeper-03-enable-reconfig.xml: | false --- # Template Parameters: # # NAME=etc-keeper-operator-templatesd-files # NAMESPACE=${OPERATOR_NAMESPACE} # COMMENT= # apiVersion: v1 kind: ConfigMap metadata: name: etc-keeper-operator-templatesd-files namespace: ${OPERATOR_NAMESPACE} labels: clickhouse.altinity.com/chop: 0.24.2 app: clickhouse-operator data: readme: | Templates in this folder are packaged with an operator and available via 'useTemplate' --- # Template Parameters: # # NAME=etc-keeper-operator-usersd-files # NAMESPACE=${OPERATOR_NAMESPACE} # COMMENT= # apiVersion: v1 kind: ConfigMap metadata: name: etc-keeper-operator-usersd-files namespace: ${OPERATOR_NAMESPACE} labels: clickhouse.altinity.com/chop: 0.24.2 app: clickhouse-operator data: --- # # Template parameters available: # NAMESPACE=${OPERATOR_NAMESPACE} # COMMENT= # OPERATOR_VERSION=0.24.2 # CH_USERNAME_SECRET_PLAIN=clickhouse_operator # CH_PASSWORD_SECRET_PLAIN=clickhouse_operator_password # apiVersion: v1 kind: Secret metadata: name: clickhouse-operator namespace: ${OPERATOR_NAMESPACE} labels: clickhouse.altinity.com/chop: 0.24.2 app: clickhouse-operator type: Opaque stringData: username: clickhouse_operator password: clickhouse_operator_password --- # Template Parameters: # # NAMESPACE=${OPERATOR_NAMESPACE} # COMMENT= # OPERATOR_IMAGE=${OPERATOR_IMAGE} # OPERATOR_IMAGE_PULL_POLICY=${OPERATOR_IMAGE_PULL_POLICY} # METRICS_EXPORTER_IMAGE=${METRICS_EXPORTER_IMAGE} # METRICS_EXPORTER_IMAGE_PULL_POLICY=${METRICS_EXPORTER_IMAGE_PULL_POLICY} # # Setup Deployment for clickhouse-operator # Deployment would be created in kubectl-specified namespace kind: Deployment apiVersion: apps/v1 metadata: name: clickhouse-operator namespace: ${OPERATOR_NAMESPACE} labels: clickhouse.altinity.com/chop: 0.24.2 app: clickhouse-operator spec: replicas: 1 selector: matchLabels: app: clickhouse-operator template: metadata: labels: app: clickhouse-operator annotations: prometheus.io/port: '8888' prometheus.io/scrape: 'true' clickhouse-operator-metrics/port: '9999' clickhouse-operator-metrics/scrape: 'true' spec: serviceAccountName: clickhouse-operator volumes: - name: etc-clickhouse-operator-folder configMap: name: etc-clickhouse-operator-files - name: etc-clickhouse-operator-confd-folder configMap: name: etc-clickhouse-operator-confd-files - name: etc-clickhouse-operator-configd-folder configMap: name: etc-clickhouse-operator-configd-files - name: etc-clickhouse-operator-templatesd-folder configMap: name: etc-clickhouse-operator-templatesd-files - name: etc-clickhouse-operator-usersd-folder configMap: name: etc-clickhouse-operator-usersd-files - name: etc-keeper-operator-confd-folder configMap: name: etc-keeper-operator-confd-files - name: etc-keeper-operator-configd-folder configMap: name: etc-keeper-operator-configd-files - name: etc-keeper-operator-templatesd-folder configMap: name: etc-keeper-operator-templatesd-files - name: etc-keeper-operator-usersd-folder configMap: name: etc-keeper-operator-usersd-files containers: - name: clickhouse-operator image: ${OPERATOR_IMAGE} imagePullPolicy: ${OPERATOR_IMAGE_PULL_POLICY} volumeMounts: - name: etc-clickhouse-operator-folder mountPath: /etc/clickhouse-operator - name: etc-clickhouse-operator-confd-folder mountPath: /etc/clickhouse-operator/chi/conf.d - name: etc-clickhouse-operator-configd-folder mountPath: /etc/clickhouse-operator/chi/config.d - name: etc-clickhouse-operator-templatesd-folder mountPath: /etc/clickhouse-operator/chi/templates.d - name: etc-clickhouse-operator-usersd-folder mountPath: /etc/clickhouse-operator/chi/users.d - name: etc-keeper-operator-confd-folder mountPath: /etc/clickhouse-operator/chk/conf.d - name: etc-keeper-operator-configd-folder mountPath: /etc/clickhouse-operator/chk/keeper_config.d - name: etc-keeper-operator-templatesd-folder mountPath: /etc/clickhouse-operator/chk/templates.d - name: etc-keeper-operator-usersd-folder mountPath: /etc/clickhouse-operator/chk/users.d env: # Pod-specific # spec.nodeName: ip-172-20-52-62.ec2.internal - name: OPERATOR_POD_NODE_NAME valueFrom: fieldRef: fieldPath: spec.nodeName # metadata.name: clickhouse-operator-6f87589dbb-ftcsf - name: OPERATOR_POD_NAME valueFrom: fieldRef: fieldPath: metadata.name # metadata.namespace: kube-system - name: OPERATOR_POD_NAMESPACE valueFrom: fieldRef: fieldPath: metadata.namespace # status.podIP: 100.96.3.2 - name: OPERATOR_POD_IP valueFrom: fieldRef: fieldPath: status.podIP # spec.serviceAccount: clickhouse-operator # spec.serviceAccountName: clickhouse-operator - name: OPERATOR_POD_SERVICE_ACCOUNT valueFrom: fieldRef: fieldPath: spec.serviceAccountName # Container-specific - name: OPERATOR_CONTAINER_CPU_REQUEST valueFrom: resourceFieldRef: containerName: clickhouse-operator resource: requests.cpu - name: OPERATOR_CONTAINER_CPU_LIMIT valueFrom: resourceFieldRef: containerName: clickhouse-operator resource: limits.cpu - name: OPERATOR_CONTAINER_MEM_REQUEST valueFrom: resourceFieldRef: containerName: clickhouse-operator resource: requests.memory - name: OPERATOR_CONTAINER_MEM_LIMIT valueFrom: resourceFieldRef: containerName: clickhouse-operator resource: limits.memory ports: - containerPort: 9999 name: metrics - name: metrics-exporter image: ${METRICS_EXPORTER_IMAGE} imagePullPolicy: ${METRICS_EXPORTER_IMAGE_PULL_POLICY} volumeMounts: - name: etc-clickhouse-operator-folder mountPath: /etc/clickhouse-operator - name: etc-clickhouse-operator-confd-folder mountPath: /etc/clickhouse-operator/chi/conf.d - name: etc-clickhouse-operator-configd-folder mountPath: /etc/clickhouse-operator/chi/config.d - name: etc-clickhouse-operator-templatesd-folder mountPath: /etc/clickhouse-operator/chi/templates.d - name: etc-clickhouse-operator-usersd-folder mountPath: /etc/clickhouse-operator/chi/users.d - name: etc-keeper-operator-confd-folder mountPath: /etc/clickhouse-operator/chk/conf.d - name: etc-keeper-operator-configd-folder mountPath: /etc/clickhouse-operator/chk/keeper_config.d - name: etc-keeper-operator-templatesd-folder mountPath: /etc/clickhouse-operator/chk/templates.d - name: etc-keeper-operator-usersd-folder mountPath: /etc/clickhouse-operator/chk/users.d env: # Pod-specific # spec.nodeName: ip-172-20-52-62.ec2.internal - name: OPERATOR_POD_NODE_NAME valueFrom: fieldRef: fieldPath: spec.nodeName # metadata.name: clickhouse-operator-6f87589dbb-ftcsf - name: OPERATOR_POD_NAME valueFrom: fieldRef: fieldPath: metadata.name # metadata.namespace: kube-system - name: OPERATOR_POD_NAMESPACE valueFrom: fieldRef: fieldPath: metadata.namespace # status.podIP: 100.96.3.2 - name: OPERATOR_POD_IP valueFrom: fieldRef: fieldPath: status.podIP # spec.serviceAccount: clickhouse-operator # spec.serviceAccountName: clickhouse-operator - name: OPERATOR_POD_SERVICE_ACCOUNT valueFrom: fieldRef: fieldPath: spec.serviceAccountName # Container-specific - name: OPERATOR_CONTAINER_CPU_REQUEST valueFrom: resourceFieldRef: containerName: clickhouse-operator resource: requests.cpu - name: OPERATOR_CONTAINER_CPU_LIMIT valueFrom: resourceFieldRef: containerName: clickhouse-operator resource: limits.cpu - name: OPERATOR_CONTAINER_MEM_REQUEST valueFrom: resourceFieldRef: containerName: clickhouse-operator resource: requests.memory - name: OPERATOR_CONTAINER_MEM_LIMIT valueFrom: resourceFieldRef: containerName: clickhouse-operator resource: limits.memory ports: - containerPort: 8888 name: metrics --- # Template Parameters: # # NAMESPACE=${OPERATOR_NAMESPACE} # COMMENT= # # Setup ClusterIP Service to provide monitoring metrics for Prometheus # Service would be created in kubectl-specified namespace # In order to get access outside of k8s it should be exposed as: # kubectl --namespace prometheus port-forward service/prometheus 9090 # and point browser to localhost:9090 kind: Service apiVersion: v1 metadata: name: clickhouse-operator-metrics namespace: ${OPERATOR_NAMESPACE} labels: clickhouse.altinity.com/chop: 0.24.2 app: clickhouse-operator spec: ports: - port: 8888 name: clickhouse-metrics - port: 9999 name: operator-metrics selector: app: clickhouse-operator