################################### # Airflow - Common Configs ################################### airflow: ## configs for the docker image of the web/scheduler/worker ## image: repository: elyra/airflow tag: 1.10.12-python3.6 ## values: Always or IfNotPresent pullPolicy: IfNotPresent pullSecret: "" ## the airflow executor type to use ## ## NOTE: ## - this should be `CeleryExecutor` or `KubernetesExecutor` ## - if set to `KubernetesExecutor`: ## - ensure that `workers.enabled` is `false` ## - ensure that `flower.enabled` is `false` ## - ensure that `redis.enabled` is `false` ## - ensure that K8S configs are set in `airflow.config` ## - we set these configs automatically: ## - `AIRFLOW__KUBERNETES__NAMESPACE` ## - `AIRFLOW__KUBERNETES__WORKER_SERVICE_ACCOUNT_NAME` ## - `AIRFLOW__KUBERNETES__ENV_FROM_CONFIGMAP_REF` ## executor: KubernetesExecutor ## the fernet key used to encrypt the connections/variables in the database ## ## WARNING: ## - you MUST customise this value, otherwise the encryption will be somewhat pointless ## ## NOTE: ## - to prevent this value being stored in your values.yaml (and airflow-env ConfigMap), ## consider using `airflow.extraEnv` to define it from a pre-created secret ## ## GENERATE: ## python -c "from cryptography.fernet import Fernet; FERNET_KEY = Fernet.generate_key().decode(); print(FERNET_KEY)" ## fernetKey: "7T512UXSSmBOkpWimFHIVb8jK6lfmSAvx4mO6Arehnc=" ## environment variables for the web/scheduler/worker Pods (for airflow configs) ## ## WARNING: ## - don't include sensitive variables in here, instead make use of `airflow.extraEnv` with Secrets ## - don't specify `AIRFLOW__CORE__SQL_ALCHEMY_CONN`, `AIRFLOW__CELERY__RESULT_BACKEND`, ## or `AIRFLOW__CELERY__BROKER_URL`, they are dynamically created from chart values ## ## NOTE: ## - airflow allows environment configs to be set as environment variables ## - they take the form: AIRFLOW__
__ ## - see the Airflow documentation: https://airflow.apache.org/docs/stable/howto/set-config.html ## ## EXAMPLE: ## config: ## ## Security ## AIRFLOW__CORE__SECURE_MODE: "True"extraEnv: # - name: AIRFLOW__CORE__FERNET_KEY # valueFrom: # secretKeyRef: # name: airflow # key: fernet-key # - name: AIRFLOW__LDAP__BIND_PASSWORD # valueFrom: # secretKeyRef: # name: ldap # key: passworde ## AIRFLOW__API__AUTH_BACKEND: "airflow.api.auth.backend.deny_all" ## AIRFLOW__WEBSERVER__EXPOSE_CONFIG: "False" ## AIRFLOW__WEBSERVER__RBAC: "False" ## ## ## DAGS ## AIRFLOW__SCHEDULER__DAG_DIR_LIST_INTERVAL: "30" ## AIRFLOW__CORE__LOAD_EXAMPLES: "False" ## ## ## Email (SMTP) ## AIRFLOW__EMAIL__EMAIL_BACKEND: "airflow.utils.email.send_email_smtp" ## AIRFLOW__SMTP__SMTP_HOST: "smtpmail.example.com" ## AIRFLOW__SMTP__SMTP_STARTTLS: "False" ## AIRFLOW__SMTP__SMTP_SSL: "False" ## AIRFLOW__SMTP__SMTP_PORT: "25" ## AIRFLOW__SMTP__SMTP_MAIL_FROM: "admin@example.com" ## ## ## Disable noisy "Handling signal: ttou" Gunicorn log messages ## GUNICORN_CMD_ARGS: "--log-level WARNING" ## ## ## Proxy Config ## HTTP_PROXY: "http://proxy.example.com:8080" ## config: AIRFLOW__KUBERNETES__WORKER_CONTAINER_REPOSITORY: "elyra/airflow" AIRFLOW__KUBERNETES__WORKER_CONTAINER_TAG: "1.10.12" AIRFLOW__KUBERNETES__WORKER_CONTAINER_IMAGE_PULL_POLICY: "IfNotPresent" AIRFLOW__KUBERNETES__LOGS_VOLUME_CLAIM: "airflow-logs" AIRFLOW__KUBERNETES__DAGS_VOLUME_CLAIM: "airflow" AIRFLOW__KUBERNETES__NAMESPACE: "default" AIRFLOW__KUBERNETES__DELETE_WORKER_PODS: "False" AIRFLOW__KUBERNETES__RUN_AS_USER: "50000" AIRFLOW__CORE__LOAD_EXAMPLES: "False" AIRFLOW__SCHEDULER__DAG_DIR_LIST_INTERVAL: "60" AIRFLOW__KUBERNETES__WORKER_SERVICE_ACCOUNT_NAME: "airflow" AIRFLOW__KUBERNETES__GIT_SYNC_ONE_TIME: "false" ## extra annotations for the web/scheduler/worker Pods ## ## EXAMPLE: ## podAnnotations: ## iam.amazonaws.com/role: "airflow-Role" ## podAnnotations: {} ## extra environment variables for the web/scheduler/worker (AND flower) Pods ## ## EXAMPLE: ## extraEnv: ## - name: AIRFLOW__CORE__FERNET_KEY ## valueFrom: ## secretKeyRef: ## name: airflow-fernet-key ## key: value ## - name: AIRFLOW__LDAP__BIND_PASSWORD ## valueFrom: ## secretKeyRef: ## name: airflow-ldap-password ## key: value ## extraEnv: [] ## extra configMap volumeMounts for the web/scheduler/worker Pods ## ## EXAMPLE: ## extraConfigmapMounts: ## - name: airflow-webserver-config ## mountPath: /opt/airflow/webserver_config.py ## configMap: airflow-webserver-config ## readOnly: true ## subPath: webserver_config.py ## extraConfigmapMounts: [] ## extra containers for the web/scheduler/worker Pods ## ## EXAMPLE: (a sidecar that syncs DAGs from object storage) ## extraContainers: ## - name: s3-sync ## image: my-user/s3sync:latest ## volumeMounts: ## - name: synchronised-dags ## mountPath: /dags ## extraContainers: [] ## extra pip packages to install in the web/scheduler/worker Pods ## ## EXAMPLE: ## extraPipPackages: ## - "airflow-exporter==1.3.1" ## #extraPipPackages: ## extra volumeMounts for the web/scheduler/worker Pods ## ## EXAMPLE: ## extraVolumeMounts: ## - name: synchronised-dags ## mountPath: /opt/airflow/dags ## extraVolumeMounts: [] ## extra volumes for the web/scheduler/worker Pods ## ## EXAMPLE: ## extraVolumes: ## - name: synchronised-dags ## emptyDir: {} ## extraVolumes: [] ################################### # Airflow - Scheduler Configs ################################### scheduler: ## resource requests/limits for the scheduler Pod ## ## EXAMPLE: ## resources: ## requests: ## cpu: "1000m" ## memory: "1Gi" ## resources: {} ## the nodeSelector configs for the scheduler Pods ## nodeSelector: {} ## the affinity configs for the scheduler Pods ## affinity: {} ## the toleration configs for the scheduler Pods ## tolerations: [] ## the security context for the scheduler Pods ## securityContext: {} ## labels for the scheduler Deployment ## labels: {} ## Pod labels for the scheduler Deployment ## podLabels: {} ## annotations for the scheduler Deployment ## annotations: {} ## Pod Annotations for the scheduler Deployment ## podAnnotations: {} ## if we should tell Kubernetes Autoscaler that its safe to evict these Pods ## safeToEvict: true ## configs for the PodDisruptionBudget of the scheduler ## podDisruptionBudget: ## if a PodDisruptionBudget resource is created for the scheduler ## enabled: true ## the maximum unavailable pods/percentage for the scheduler ## ## NOTE: ## - as there is only ever a single scheduler Pod, ## this must be 100% for Kubernetes to be able to migrate it ## maxUnavailable: "100%" ## the minimum available pods/percentage for the scheduler ## minAvailable: "" ## custom airflow connections for the airflow scheduler ## ## EXAMPLE: ## connections: ## - id: my_aws ## type: aws ## extra: | ## { ## "aws_access_key_id": "XXXXXXXX", ## "aws_secret_access_key": "XXXXXXXX", ## "region_name":"eu-central-1" ## } ## connections: [] ## if `scheduler.connections` are deleted and re-added after each scheduler restart ## refreshConnections: true ## the name of an existing Secret containing an `add-connections.sh` script to run on scheduler start ## ## NOTE: ## - if this is non-empty, `scheduler.connections` will be ignored ## - use this if you don't want to store connections in your values.yaml ## ## EXAMPLE SECRET: ## apiVersion: v1 ## kind: Secret ## metadata: ## name: my-airflow-connections ## type: Opaque ## stringData: ## add-connections.sh: | ## #!/usr/bin/env bash ## ## # remove any existing connection ## airflow connections --delete \ ## --conn_id "my_aws" ## ## # re-add your custom connection ## airflow connections --add \ ## --conn_id "my_aws" \ ## --conn_type "aws" \ ## --conn_extra "{\"region_name\":\"eu-central-1\"}" ## existingSecretConnections: "" ## custom airflow variables for the airflow scheduler ## ## NOTE: ## - THIS IS A STRING, containing a JSON object, with your variables in it ## ## EXAMPLE: ## variables: | ## { "environment": "dev" } ## variables: | {} ## custom airflow pools for the airflow scheduler ## ## NOTE: ## - THIS IS A STRING, containing a JSON object, with your pools in it ## ## EXAMPLE: ## pools: | ## { ## "example": { ## "description": "This is an example pool with 2 slots.", ## "slots": 2 ## } ## } ## pools: | {} ## the value of the `airflow --num_runs` parameter used to run the airflow scheduler ## ## NOTE: ## - this is the number of 'dag refreshes' before the airflow scheduler process will exit ## - if not set to `-1`, the scheduler Pod will restart regularly ## - for most environments, `-1` will be an acceptable value ## numRuns: -1 ## if we run `airflow initdb` when the scheduler starts ## initdb: true ## if we run `airflow initdb` inside a special initContainer ## ## NOTE: ## - may be needed if you have custom database hooks configured that will be pulled in by git-sync ## preinitdb: false ## the number of seconds to wait (in bash) before starting the scheduler container ## initialStartupDelay: 0 ## configs for the scheduler liveness probe ## ## NOTE: ## - the maximum number of seconds the scheduler can be unhealthy = ## `livenessProbe.periodSeconds` x `livenessProbe.failureThreshold` ## livenessProbe: enabled: true ## the number of seconds to wait before checking pod health ## ## NOTE: ## - make larger if you are installing many packages with: ## `airflow.extraPipPackages`, `web.extraPipPackages`, or `dags.installRequirements` ## initialDelaySeconds: 300 periodSeconds: 30 failureThreshold: 5 ## extra init containers to run before the scheduler Pod ## ## EXAMPLE: ## extraInitContainers: ## - name: volume-mount-hack ## image: busybox ## command: ["sh", "-c", "chown -R 1000:1000 logs"] ## volumeMounts: ## - mountPath: /opt/airflow/logs ## name: logs-data ## extraInitContainers: [] ################################### # Airflow - WebUI Configs ################################### web: ## resource requests/limits for the airflow web Pods ## ## EXAMPLE: ## resources: ## requests: ## cpu: "500m" ## memory: "1Gi" ## resources: {} ## the number of web Pods to run ## replicas: 1 ## the nodeSelector configs for the web Pods ## nodeSelector: {} ## the affinity configs for the web Pods ## affinity: {} ## the toleration configs for the web Pods ## tolerations: [] ## the security context for the web Pods ## securityContext: {} ## labels for the web Deployment ## labels: {} ## Pod labels for the web Deployment ## podLabels: {} ## annotations for the web Deployment ## annotations: {} ## Pod annotations for the web Deployment ## podAnnotations: {} ## if we should tell Kubernetes Autoscaler that its safe to evict these Pods ## safeToEvict: true ## configs for the PodDisruptionBudget of the web Deployment ## podDisruptionBudget: ## if a PodDisruptionBudget resource is created for the web Deployment ## ## WARNING: ## - if you enable a PodDisruptionBudget, you should set `web.replicas` to a value ## large enough for Kubernetes to evict at least 1 Pod at a time ## enabled: false ## the maximum unavailable pods/percentage for the web Deployment ## maxUnavailable: "" ## the minimum available pods/percentage for the web Deployment ## minAvailable: "" ## configs for the Service of the web Pods ## service: ## annotations: {} ##sessionAffinity: "None" ##sessionAffinityConfig: {} type: NodePort ## externalPort: 8080 ## loadBalancerIP: "" ##loadBalancerSourceRanges: [] ##nodePort: ## http: "" ## sets `AIRFLOW__WEBSERVER__BASE_URL` ## ## NOTE: ## - should be compatible with `ingress.web.path` config ## baseUrl: "http://localhost:8080" ## sets `AIRFLOW__CORE__STORE_SERIALIZED_DAGS` ## ## NOTE: ## - setting true will disable `git-sync` and `git-clone` containers in the web Pod ## - Docs: https://airflow.apache.org/docs/stable/dag-serialization.html ## serializeDAGs: false ## extra pip packages to install in the web container ## ## EXAMPLE: ( packages used by RBAC UI for OAuth ) ## extraPipPackages: ## - "apache-airflow[google_auth]==1.10.10" ## extraPipPackages: [] ## the number of seconds to wait (in bash) before starting the web container ## initialStartupDelay: 0 ## the number of seconds to wait before declaring a new Pod available ## minReadySeconds: 5 ## configs for the web Service readiness probe ## readinessProbe: enabled: false scheme: HTTP initialDelaySeconds: 10 periodSeconds: 10 timeoutSeconds: 1 successThreshold: 1 failureThreshold: 3 ## configs for the web Service liveness probe ## livenessProbe: enabled: true scheme: HTTP ## the number of seconds to wait before checking pod health ## ## NOTE: ## - make larger if you are installing many packages with: ## `airflow.extraPipPackages`, `web.extraPipPackages`, or `dags.installRequirements` ## initialDelaySeconds: 300 periodSeconds: 30 timeoutSeconds: 3 successThreshold: 1 failureThreshold: 2 ## the directory in which to mount secrets on web containers ## secretsDir: /var/airflow/secrets ## the names of existing Kubernetes Secrets to mount as files at `{workers.secretsDir}//` ## secrets: [] ## the name of an existing Kubernetes Secret to mount as files to `{web.secretsDir}/` ## ## NOTE: ## - overrides `web.secrets` ## secretsMap: "" ################################### # Airflow - Worker Configs ################################### workers: ## if the airflow workers StatefulSet should be deployed ## enabled: false ## Set to false for use with KubernetesExecutor ## resource requests/limits for the airflow worker Pods ## ## EXAMPLE: ## resources: ## requests: ## cpu: "1000m" ## memory: "2Gi" ## resources: {} ## the number of workers Pods to run ## ## NOTE: ## - when `workers.autoscaling.enabled` is true, this is the minimum ## replicas: 1 ## the nodeSelector configs for the worker Pods ## nodeSelector: {} ## the affinity configs for the worker Pods ## affinity: {} ## the toleration configs for the worker Pods ## tolerations: [] ## the security context for the worker Pods ## securityContext: {} ## labels for the worker StatefulSet ## labels: {} ## Pod labels for the worker StatefulSet ## podLabels: {} ## annotations for the worker StatefulSet ## annotations: {} ## Pod annotations for the worker StatefulSet ## podAnnotations: {} ## if we should tell Kubernetes Autoscaler that its safe to evict these Pods ## safeToEvict: true ## configs for the PodDisruptionBudget of the worker StatefulSet ## podDisruptionBudget: ## if a PodDisruptionBudget resource is created for the worker StatefulSet ## ## WARNING: ## - if you enable a PodDisruptionBudget, you should set `workers.replicas` to a value ## large enough for Kubernetes to evict at least 1 Pod at a time ## - if you enable `workers.celery.gracefullTermination`, you should consider ## specifying a `workers.podDisruptionBudget.minAvailable` to prevent there ## not being enough available workers during graceful termination waiting periods ## enabled: false ## the maximum unavailable pods/percentage for the worker StatefulSet ## maxUnavailable: "" ## the minimum available pods/percentage for the worker StatefulSet ## minAvailable: "" ## configs for the HorizontalPodAutoscaler of the worker Pods ## ## EXAMPLE: ## autoscaling: ## enabled: true ## maxReplicas: 16 ## metrics: ## - type: Resource ## resource: ## name: memory ## target: ## type: Utilization ## averageUtilization: 80 ## autoscaling: enabled: false maxReplicas: 2 metrics: [] ## the number of seconds to wait (in bash) before starting each worker container ## initialStartupDelay: 0 ## configs for the celery worker Pods ## ## NOTE: ## - only takes effect if `airflow.executor` is `CeleryExecutor` ## celery: ## the number of tasks each celery worker can run at a time ## ## NOTE: ## - sets AIRFLOW__CELERY__WORKER_CONCURRENCY ## instances: 16 ## if we should wait for tasks to finish before SIGTERM of the celery worker ## ## WARNING: ## - consider using `workers.podDisruptionBudget.*` to prevent there not being ## enough available workers during graceful termination waiting periods ## gracefullTermination: false ## how many seconds to wait for tasks to finish before SIGTERM of the celery worker ## ## graceful shutdown lifecycle: ## 1. prevent worker accepting new tasks ## 2. wait AT MOST `workers.celery.gracefullTerminationPeriod` for tasks to finish ## 3. send SIGTERM to worker ## 4. wait AT MOST `workers.terminationPeriod` for kill to finish ## 5. send SIGKILL to worker ## gracefullTerminationPeriod: 600 ## how many seconds to wait after SIGTERM before SIGKILL of the celery worker ## ## WARNING: ## - tasks that are still running during SIGKILL will be orphaned, this is important ## to understand with KubernetesPodOperator(), as Pods may continue running ## terminationPeriod: 60 ## directory in which to mount secrets on worker containers ## secretsDir: /var/airflow/secrets ## the names of existing Kubernetes Secrets to mount as files at `{workers.secretsDir}//` ## secrets: [] ## the name of an existing Kubernetes Secret to mount as files to `{web.secretsDir}/` ## ## NOTE: ## - overrides `worker.secrets` ## secretsMap: "" ################################### # Airflow - Flower Configs ################################### flower: ## if the Flower UI should be deployed ## ## NOTE: ## - only takes effect if `airflow.executor` is `CeleryExecutor` ## enabled: true ## resource requests/limits for the flower Pods ## ## EXAMPLE: ## resources: ## requests: ## cpu: "100m" ## memory: "126Mi" ## resources: {} ## the number of flower Pods to run ## replicas: 1 ## the nodeSelector configs for the flower Pods ## nodeSelector: {} ## the affinity configs for the flower Pods ## affinity: {} ## the toleration configs for the flower Pods ## tolerations: [] ## the security context for the flower Pods ## securityContext: {} ## labels for the flower Deployment ## labels: {} ## Pod labels for the flower Deployment ## podLabels: {} ## annotations for the flower Deployment ## annotations: {} ## Pod annotations for the flower Deployment ## podAnnotations: {} ## if we should tell Kubernetes Autoscaler that its safe to evict these Pods ## safeToEvict: true ## configs for the PodDisruptionBudget of the flower Deployment ## podDisruptionBudget: ## if a PodDisruptionBudget resource is created for the flower Deployment ## ## WARNING: ## - if you enable a PodDisruptionBudget, you should set `flower.replicas` to a value ## large enough for Kubernetes to evict at least 1 Pod at a time ## enabled: false ## the maximum unavailable pods/percentage for the flower Deployment ## maxUnavailable: "" ## the minimum available pods/percentage for the flower Deployment ## minAvailable: "" ## the value of the flower `--auth` argument ## ## NOTE: ## - see flower docs: https://flower.readthedocs.io/en/latest/auth.html#google-oauth-2-0 ## oauthDomains: "" ## the name of a pre-created secret containing the basic authentication value for flower ## ## NOTE: ## - This sets `AIRFLOW__CELERY__FLOWER_BASIC_AUTH` ## basicAuthSecret: "" ## the key within `flower.basicAuthSecret` containing the basic authentication string ## basicAuthSecretKey: "" ## sets `AIRFLOW__CELERY__FLOWER_URL_PREFIX` ## ## NOTE: ## - should match `ingress.flower.path` config ## urlPrefix: "" ## configs for the Service of the flower Pods ## service: annotations: {} type: ClusterIP externalPort: 5555 loadBalancerIP: "" loadBalancerSourceRanges: [] nodePort: http: "" ## the number of seconds to wait (in bash) before starting the flower container ## initialStartupDelay: 0 ## the number of seconds to wait before declaring a new Pod available ## minReadySeconds: 5 ## extra ConfigMaps to mount on the flower Pods ## ## EXAMPLE: ## extraConfigmapMounts: ## - name: extra-cert ## mountPath: /etc/ssl/certs/extra-cert.pem ## configMap: extra-certificates ## readOnly: true ## subPath: extra-cert.pem ## extraConfigmapMounts: [] ################################### # Airflow - Logs Configs ################################### logs: ## the airflow logs folder ## path: /opt/airflow/logs ## configs for the logs PVC ## persistence: ## if a persistent volume is mounted at `logs.path` ## enabled: true ## the name of an existing PVC to use ## existingClaim: "" ## sub-path under `logs.persistence.existingClaim` to use ## subPath: "" ## the name of the StorageClass used by the PVC ## ## NOTE: ## - if set to "", then `PersistentVolumeClaim/spec.storageClassName` is omitted ## - if set to "-", then `PersistentVolumeClaim/spec.storageClassName` is set to "" ## storageClass: "standard" ## the access mode of the PVC ## ## WARNING: ## - must be: `ReadWriteMany` ## ## NOTE: ## - different StorageClass support different access modes: ## https://kubernetes.io/docs/concepts/storage/persistent-volumes/#access-modes ## accessMode: ReadWriteMany ## the size of PVC to request ## size: 1Gi ################################### # Airflow - DAGs Configs ################################### dags: ## the airflow dags folder ## path: /opt/airflow/dags ## whether to disable pickling dags from the scheduler to workers ## ## NOTE: ## - sets AIRFLOW__CORE__DONOT_PICKLE ## doNotPickle: false ## install any Python `requirements.txt` at the root of `dags.path` automatically ## ## WARNING: ## - if set to true, and you are using `dags.git.gitSync`, you must also enable ## `dags.initContainer` to ensure the requirements.txt is available at Pod start ## installRequirements: false ## configs for the dags PVC ## persistence: ## if a persistent volume is mounted at `dags.path` ## enabled: true ## the name of an existing PVC to use ## existingClaim: "" ## sub-path under `dags.persistence.existingClaim` to use ## subPath: "" ## the name of the StorageClass used by the PVC ## ## NOTE: ## - if set to "", then `PersistentVolumeClaim/spec.storageClassName` is omitted ## - if set to "-", then `PersistentVolumeClaim/spec.storageClassName` is set to "" ## storageClass: "standard" ## the access mode of the PVC ## ## WARNING: ## - must be one of: `ReadOnlyMany` or `ReadWriteMany` ## ## NOTE: ## - different StorageClass support different access modes: ## https://kubernetes.io/docs/concepts/storage/persistent-volumes/#access-modes ## accessMode: ReadOnlyMany ## the size of PVC to request ## size: 1Gi ## configs for the DAG git repository & sync container ## git: ## url of the git repository ## ## EXAMPLE: (HTTP) ## url: "https://github.com/torvalds/linux.git" ## ## EXAMPLE: (SSH) ## url: "ssh://git@github.com:torvalds/linux.git" ## url: "ssh://git@github.com/your-git-org/your-dag-repo" ## the branch/tag/sha1 which we clone ## ref: "main" ## the name of a pre-created secret containing files for ~/.ssh/ ## ## NOTE: ## - this is ONLY RELEVANT for SSH git repos ## - the secret commonly includes files: id_rsa, id_rsa.pub, known_hosts ## - known_hosts is NOT NEEDED if `git.sshKeyscan` is true ## secret: "airflow-secret" ## if we should implicitly trust [git.repoHost]:git.repoPort, by auto creating a ~/.ssh/known_hosts ## ## WARNING: ## - setting true will increase your vulnerability ot a repo spoofing attack ## ## NOTE: ## - this is ONLY RELEVANT for SSH git repos ## - this is not needed if known_hosts is provided in `git.secret` ## - git.repoHost and git.repoPort ARE REQUIRED for this to work ## sshKeyscan: true ## the name of the private key file in your `git.secret` ## ## NOTE: ## - this is ONLY RELEVANT for PRIVATE SSH git repos ## privateKeyName: id_rsa ## the host name of the git repo ## ## NOTE: ## - this is ONLY REQUIRED for SSH git repos ## ## EXAMPLE: ## repoHost: "github.com" ## repoHost: "github.com" ## the port of the git repo ## ## NOTE: ## - this is ONLY REQUIRED for SSH git repos ## repoPort: 22 ## configs for the git-sync container ## gitSync: ## enable the git-sync sidecar container ## enabled: true ## resource requests/limits for the git-sync container ## ## NOTE: ## - when `workers.autoscaling` is true, YOU MUST SPECIFY a resource request ## ## EXAMPLE: ## resources: ## requests: ## cpu: "50m" ## memory: "64Mi" ## resources: {} ## the docker image for the git-sync container image: repository: alpine/git tag: latest ## values: Always or IfNotPresent pullPolicy: Always ## the git sync interval in seconds ## refreshTime: 10 ## configs for the git-clone container ## ## NOTE: ## - use this container if you want to only clone the external git repo ## at Pod start-time, and not keep it synchronised afterwards ## initContainer: ## enable the git-clone sidecar container ## ## NOTE: ## - this is NOT required for the git-sync sidecar to work ## - this is mostly used for when `dags.installRequirements` is true to ensure that ## requirements.txt is available at Pod start ## enabled: false ## resource requests/limits for the git-clone container ## ## EXAMPLE: ## resources: ## requests: ## cpu: "50m" ## memory: "64Mi" ## resources: {} ## the docker image for the git-clone container image: repository: alpine/git tag: latest ## values: Always or IfNotPresent pullPolicy: Always ## path to mount dags-data volume to ## ## WARNING: ## - this path is also used by the git-sync container ## mountPath: "/dags" ## sub-path under `dags.initContainer.mountPath` to sync dags to ## ## WARNING: ## - this path is also used by the git-sync container ## - this MUST INCLUDE the leading / ## ## EXAMPLE: ## syncSubPath: "/subdirWithDags" ## syncSubPath: "" ################################### # Kubernetes - Ingress Configs ################################### ingress: ## if we should deploy Ingress resources ## ## NOTE: ## - if you want to change url prefix for web ui or flower (even if you do not use this Ingress), ## you can change `web.baseUrl` and `flower.urlPrefix` ## enabled: false ## configs for the Ingress of the web Service ## web: ## annotations for the web Ingress ## annotations: {} ## additional labels for the web Ingress ## labels: {} ## the path for the web Ingress ## ## WARNING: ## - do NOT include the trailing slash (for root, set an empty string) ## ## NOTE: ## - should be compatible with `web.baseUrl` config ## ## EXAMPLE: (if set to "/airflow") ## - UI: http://example.com/airflow/admin ## - API: http://example.com/airflow/api ## - HEALTH: http://example.com/airflow/health ## path: "" ## the hostname for the web Ingress ## host: "" ## the livenessPath for the web Ingress ## ## NOTE: ## - if set to "", defaults to: `{ingress.web.path}/health` ## livenessPath: "" ## configs for web Ingress TLS ## tls: ## enable TLS termination for the web Ingress ## enabled: false ## the name of a pre-created Secret containing a TLS private key and certificate ## ## NOTE: ## - this MUST be specified if `ingress.web.tls.enabled` is true ## secretName: "" ## http paths to add to the web Ingress before the default path ## ## EXAMPLE: ## precedingPaths: ## - path: "/*" ## serviceName: "ssl-redirect" ## servicePort: "use-annotation" ## precedingPaths: [] ## http paths to add to the web Ingress after the default path ## ## EXAMPLE: ## succeedingPaths: ## - path: "/extra-service" ## serviceName: "extra-service" ## servicePort: "use-annotation" ## succeedingPaths: [] ## configs for the Ingress of the flower Service ## flower: ## annotations for the flower Ingress ## annotations: {} ## additional labels for the flower Ingress ## labels: {} ## the path for the flower Ingress ## ## WARNING: ## - do NOT include the trailing slash (for root, set an empty string) ## ## NOTE: ## - should match `flower.urlPrefix` config ## ## EXAMPLE: (if set to "/airflow/flower") ## - UI: http://example.com/airflow/flower ## path: "" ## the hostname for the flower Ingress ## host: "" ## the livenessPath for the flower Ingress ## ## WARNING: ## - keep the trailing slash ## ## NOTE: ## - if set to "", defaults to: `{ingress.flower.path}/` ## livenessPath: "" ## configs for flower Ingress TLS ## tls: ## enable TLS termination for the flower Ingress ## enabled: false ## the name of a pre-created Secret containing a TLS private key and certificate ## ## NOTE: ## - this MUST be specified if `ingress.flower.tls.enabled` is true ## secretName: "" ## http paths to add to the flower Ingress before the default path ## ## EXAMPLE: ## precedingPaths: ## - path: "/*" ## serviceName: "ssl-redirect" ## servicePort: "use-annotation" ## precedingPaths: [] ## http paths to add to the flower Ingress after the default path ## ## EXAMPLE: ## succeedingPaths: ## - path: "/extra-service" ## serviceName: "extra-service" ## servicePort: "use-annotation" ## succeedingPaths: [] ################################### # Kubernetes - RBAC ################################### rbac: ## if Kubernetes RBAC resources are created ## ## NOTE: ## - these allow the service account to create/delete Pods in the airflow namespace, ## which is required for the KubernetesPodOperator() to function ## create: true ## if the created RBAC Role has GET/LIST on Event resources ## ## NOTE: ## - this is needed for KubernetesPodOperator() to use `log_events_on_failure=True` ## events: false ################################### # Kubernetes - Service Account ################################### serviceAccount: ## if a Kubernetes ServiceAccount is created ## ## NOTE: ## - if false, you must create the service account outside of this helm chart, ## with the name: `serviceAccount.name` ## create: true ## the name of the ServiceAccount ## ## NOTE: ## - by default the name is generated using the `airflow.serviceAccountName` template in `_helpers.tpl` ## name: "" ## annotations for the ServiceAccount ## ## EXAMPLE: (to use WorkloadIdentity in Google Cloud) ## annotations: ## iam.gke.io/gcp-service-account: <>@<>.iam.gserviceaccount.com ## annotations: {} ################################### # Kubernetes - Extra Manifests ################################### ## additional Kubernetes manifests to include with this chart ## ## EXAMPLE: ## extraManifests: ## - apiVersion: cloud.google.com/v1beta1 ## kind: BackendConfig ## metadata: ## name: "{{ .Release.Name }}-test" ## spec: ## securityPolicy: ## name: "gcp-cloud-armor-policy-test" ## extraManifests: [] ################################### # Database - PostgreSQL Chart # - https://github.com/helm/charts/tree/master/stable/postgresql ################################### postgresql: ## if the `stable/postgresql` chart is used ## ## WARNING: ## - this is NOT SUITABLE for production deployments of Airflow, ## you should seriously consider using an external database service, ## which can be configured with values under: `externalDatabase` ## ## NOTE: ## - set to `false` if using an external database ## enabled: true ## the postgres database to use ## postgresqlDatabase: airflow ## the postgres user to create ## postgresqlUsername: postgres ## the postgres user's password ## ## WARNING: ## - you should NOT use this, instead specify `postgresql.existingSecret` ## postgresqlPassword: airflow ## the name of a pre-created secret containing the postgres password ## existingSecret: "" ## the key within `postgresql.existingSecret` containing the password string ## existingSecretKey: "postgresql-password" ## configs for the PVC of postgresql ## persistence: ## if postgres will use Persistent Volume Claims to store data ## ## WARNING: ## - if false, data will be LOST as postgres Pods restart ## enabled: true ## the name of the StorageClass used by the PVC ## storageClass: "" ## the access modes of the PVC ## accessModes: - ReadWriteOnce ## the size of PVC to request ## size: 8Gi ## configs for the postgres StatefulSet master: ## annotations for the postgres Pod ## podAnnotations: cluster-autoscaler.kubernetes.io/safe-to-evict: "true" ################################### # Database - External Database # - these configs are only used when `postgresql.enabled` is false ################################### externalDatabase: ## the type of external database: {mysql,postgres} ## type: postgres ## the host of the external database ## host: localhost ## the port of the external database ## port: 5432 ## the database/scheme to use within the the external database ## database: airflow ## the user of the external database ## user: airflow ## the name of a pre-created secret containing the external database password ## passwordSecret: "" ## the key within `externalDatabase.passwordSecret` containing the password string ## passwordSecretKey: "postgresql-password" ## the connection properties for external database, e.g. "?sslmode=require" properties: "" ################################### # Database - Redis Chart # - https://github.com/helm/charts/tree/master/stable/redis ################################### redis: ## if the `stable/redis` chart is used ## ## NOTE: ## - set to `false` if using an external redis database ## - set to `false` if `airflow.executor` is `KubernetesExecutor` ## enabled: true ## the redis password ## ## WARNING: ## - you should NOT use this, instead specify `redis.existingSecret` ## password: airflow ## the name of a pre-created secret containing the redis password ## existingSecret: "" ## the key within `redis.existingSecret` containing the password string ## existingSecretPasswordKey: "redis-password" ## configs for redis cluster mode ## cluster: ## if redis runs in cluster mode ## enabled: false ## the number of redis slaves ## slaveCount: 1 ## configs for the redis master ## master: ## resource requests/limits for the master Pod ## ## EXAMPLE: ## resources: ## requests: ## cpu: "100m" ## memory: "256Mi" ## resources: {} ## annotations for the master Pod ## podAnnotations: cluster-autoscaler.kubernetes.io/safe-to-evict: "true" ## configs for the PVC of the redis master ## persistence: ## use a PVC to persist data ## enabled: false ## the name of the StorageClass used by the PVC ## storageClass: "" ## the access mode of the PVC ## accessModes: - ReadWriteOnce ## the size of PVC to request ## size: 8Gi ## configs for the redis slaves ## slave: ## resource requests/limits for the slave Pods ## ## EXAMPLE: ## resources: ## requests: ## cpu: "100m" ## memory: "256Mi" ## resources: {} ## annotations for the slave Pods ## podAnnotations: cluster-autoscaler.kubernetes.io/safe-to-evict: "true" ## configs for the PVC of the redis slaves ## persistence: ## use a PVC to persist data ## enabled: false ## the name of the StorageClass used by the PVC ## storageClass: "" ## the access mode of the PVC ## accessModes: - ReadWriteOnce ## the size of PVC to request ## size: 8Gi ################################### # Database - External Database # - these configs are only used when `redis.enabled` is false ################################### externalRedis: ## the host of the external redis ## host: localhost ## the port of the external redis ## port: 6379 ## the database number to use within the the external redis ## databaseNumber: 1 ## the name of a pre-created secret containing the external redis password ## passwordSecret: "" ## the key within `externalRedis.passwordSecret` containing the password string ## passwordSecretKey: "redis-password" ################################### # Prometheus - ServiceMonitor ################################### serviceMonitor: ## if the ServiceMonitor resources should be deployed ## ## WARNING: ## - you will need an exporter in your airflow docker container, for example: ## https://github.com/epoch8/airflow-exporter ## ## NOTE: ## - you can install pip packages with `airflow.extraPipPackages` ## - ServiceMonitor is a resource from: https://github.com/coreos/prometheus-operator ## enabled: false ## labels for ServiceMonitor, so that Prometheus can select it ## selector: prometheus: kube-prometheus ## the ServiceMonitor web endpoint path ## path: /admin/metrics ## the ServiceMonitor web endpoint interval ## interval: "30s" ################################### # Prometheus - PrometheusRule ################################### prometheusRule: ## if the PrometheusRule resources should be deployed ## ## WARNING: ## - you will need an exporter in your airflow docker container, for example: ## https://github.com/epoch8/airflow-exporter ## ## NOTE: ## - you can install pip packages with `airflow.extraPipPackages` ## - PrometheusRule a resource from: https://github.com/coreos/prometheus-operator ## enabled: false ## labels for PrometheusRule, so that Prometheus can select it ## additionalLabels: {} ## alerting rules for Prometheus ## ## NOTE: ## - documentation: https://prometheus.io/docs/prometheus/latest/configuration/alerting_rules/ ## groups: []