version: '3' services: koku-base: image: koku_base build: context: . args: GIT_COMMIT: ${GIT_COMMIT-local-dev} PIPENV_DEV: "True" USER_ID: ${USER_ID-1000} command: "echo 'Base Image Should Not Run'" depends_on: - db - redis - unleash koku-server: container_name: koku_server image: koku_base working_dir: /koku entrypoint: - /koku/run_server.sh environment: - DATABASE_SERVICE_NAME=POSTGRES_SQL - DATABASE_ENGINE=postgresql - DATABASE_NAME=${DATABASE_NAME-postgres} - POSTGRES_SQL_SERVICE_HOST=db - POSTGRES_SQL_SERVICE_PORT=5432 - DATABASE_USER=${DATABASE_USER-postgres} - DATABASE_PASSWORD=${DATABASE_PASSWORD-postgres} - KOKU_SOURCES_CLIENT_HOST=${KOKU_SOURCES_CLIENT_HOST-sources-client} - KOKU_SOURCES_CLIENT_PORT=${KOKU_SOURCES_CLIENT_PORT-9000} - AWS_ACCESS_KEY_ID - AWS_SECRET_ACCESS_KEY - DJANGO_READ_DOT_ENV_FILE=True - DEVELOPMENT=${DEVELOPMENT-True} - RBAC_SERVICE_HOST=${RBAC_SERVICE_HOST-rbac-server} - RBAC_SERVICE_PORT=${RBAC_SERVICE_PORT-9000} - RBAC_SERVICE_PATH=${RBAC_SERVICE_PATH-/r/insights/platform/rbac/v1/access/} - REDIS_HOST=${REDIS_HOST-redis} - REDIS_PORT=${REDIS_PORT-6379} - RBAC_CACHE_TTL - PROMETHEUS_MULTIPROC_DIR=/tmp - API_PATH_PREFIX=${API_PATH_PREFIX-/api/cost-management} - MAX_GROUP_BY_OVERRIDE=${MAX_GROUP_BY_OVERRIDE-3} - GOOGLE_APPLICATION_CREDENTIALS=${GOOGLE_APPLICATION_CREDENTIALS-} - DEMO_ACCOUNTS - ACCOUNT_ENHANCED_METRICS=${ACCOUNT_ENHANCED_METRICS-False} - RUN_GUNICORN=${RUN_GUNICORN-} - POD_CPU_LIMIT=${POD_CPU_LIMIT-1} - GUNICORN_WORKERS=${GUNICORN_WORKERS-3} - GUNICORN_THREADS=${GUNICORN_THREADS-False} - RETAIN_NUM_MONTHS=${RETAIN_NUM_MONTHS-4} - NOTIFICATION_CHECK_TIME=${NOTIFICATION_CHECK_TIME-24} - UNLEASH_HOST=${UNLEASH_HOST-unleash} - UNLEASH_PREFIX=${UNLEASH_PREFIX-http} - UNLEASH_PORT=${UNLEASH_PORT-4242} - UNLEASH_TOKEN=${UNLEASH_TOKEN:?} - UNLEASH_LOG_LEVEL=${UNLEASH_LOG_LEVEL-WARNING} - OCI_CLI_USER=${OCI_CLI_USER} - OCI_CLI_FINGERPRINT=${OCI_CLI_FINGERPRINT} - OCI_CLI_TENANCY=${OCI_CLI_TENANCY} - OCI_CLI_KEY_FILE=${OCI_CLI_KEY_FILE} - OCI_PYTHON_SDK_NO_SERVICE_IMPORTS=True - QE_SCHEMA=${QE_SCHEMA} - ENHANCED_ORG_ADMIN=${ENHANCED_ORG_ADMIN-True} - TAG_ENABLED_LIMIT=${TAG_ENABLED_LIMIT-200} ports: - 8000:8000 - 8001:9000 volumes: - './koku:/koku/koku' - './.unleash:/koku/.unleash' - './db_functions:/koku/db_functions' - './scripts:/koku/scripts' - './run_server.sh:/koku/run_server.sh' - './testing/auth_files/:/testing/auth_files/' depends_on: - koku-base masu-server: container_name: masu_server image: koku_base working_dir: /koku entrypoint: - /koku/run_server.sh environment: - MASU=True - DATABASE_SERVICE_NAME=POSTGRES_SQL - DATABASE_ENGINE=postgresql - DATABASE_NAME=${DATABASE_NAME-postgres} - POSTGRES_SQL_SERVICE_HOST=db - POSTGRES_SQL_SERVICE_PORT=5432 - DATABASE_USER=${DATABASE_USER-postgres} - DATABASE_PASSWORD=${DATABASE_PASSWORD-postgres} - AWS_ACCESS_KEY_ID - AWS_SECRET_ACCESS_KEY - DJANGO_READ_DOT_ENV_FILE=True - DEVELOPMENT=${DEVELOPMENT-True} - RBAC_SERVICE_HOST=${RBAC_SERVICE_HOST-rbac-server} - RBAC_SERVICE_PORT=${RBAC_SERVICE_PORT-9000} - RBAC_SERVICE_PATH=${RBAC_SERVICE_PATH-/r/insights/platform/rbac/v1/access/} - REDIS_HOST=${REDIS_HOST-redis} - REDIS_PORT=${REDIS_PORT-6379} - RBAC_CACHE_TTL - PROMETHEUS_MULTIPROC_DIR=/tmp - API_PATH_PREFIX=${API_PATH_PREFIX-/api/cost-management} - ACCOUNT_ENHANCED_METRICS=${ACCOUNT_ENHANCED_METRICS-False} - SOURCES_API_SVC_HOST=${SOURCES_API_HOST-sources} - SOURCES_API_SVC_PORT=${SOURCES_API_PORT-3000} - SOURCES_API_PREFIX=${SOURCES_API_PREFIX-/api/sources/v1.0} - SOURCES_PSK=${SOURCES_PSK-thisMustBeEphemeralOrMinikube} - MASU_DATE_OVERRIDE - RUN_GUNICORN=${RUN_GUNICORN-} - POD_CPU_LIMIT=${POD_CPU_LIMIT-1} - GUNICORN_WORKERS=${GUNICORN_WORKERS-3} - GUNICORN_THREADS=${GUNICORN_THREADS-False} - RETAIN_NUM_MONTHS=${RETAIN_NUM_MONTHS-4} - NOTIFICATION_CHECK_TIME=${NOTIFICATION_CHECK_TIME-24} - UNLEASH_HOST=${UNLEASH_HOST-unleash} - UNLEASH_PREFIX=${UNLEASH_PREFIX-http} - UNLEASH_PORT=${UNLEASH_PORT-4242} - UNLEASH_TOKEN=${UNLEASH_TOKEN:?} - UNLEASH_LOG_LEVEL=${UNLEASH_LOG_LEVEL-WARNING} - GOOGLE_APPLICATION_CREDENTIALS=${GOOGLE_APPLICATION_CREDENTIALS-} - OCI_CLI_USER=${OCI_CLI_USER} - OCI_CLI_FINGERPRINT=${OCI_CLI_FINGERPRINT} - OCI_CLI_TENANCY=${OCI_CLI_TENANCY} - OCI_CLI_KEY_FILE=${OCI_CLI_KEY_FILE} - OCI_PYTHON_SDK_NO_SERVICE_IMPORTS=True - ENHANCED_ORG_ADMIN=${ENHANCED_ORG_ADMIN-True} - TRINO_HOST=${TRINO_HOST-trino} - TRINO_PORT=${TRINO_PORT-8080} - TAG_ENABLED_LIMIT=${TAG_ENABLED_LIMIT-200} ports: - 5042:8000 - 5001:9000 volumes: - './koku:/koku/koku' - './.unleash:/koku/.unleash' - './db_functions:/koku/db_functions' - './scripts:/koku/scripts' - './run_server.sh:/koku/run_server.sh' - './testing/auth_files/:/testing/auth_files/' depends_on: - koku-base koku-worker: hostname: koku-worker-1 image: koku_base working_dir: /koku/koku entrypoint: ['watchmedo', 'auto-restart', '--directory=./', '--pattern=*.py', '--ignore-patterns=*test*', '--recursive', '--', 'celery', '-A', 'koku', 'worker', '--without-gossip', '-l', 'info', '-Q', 'celery,download,hcs,ocp,priority,summary,cost_model,refresh,subs_extraction,subs_transmission'] environment: - AWS_ACCESS_KEY_ID - AWS_SECRET_ACCESS_KEY - DATABASE_SERVICE_NAME=POSTGRES_SQL - DATABASE_ENGINE=postgresql - DATABASE_NAME=${DATABASE_NAME-postgres} - POSTGRES_SQL_SERVICE_HOST=db - POSTGRES_SQL_SERVICE_PORT=5432 - DATABASE_USER=${DATABASE_USER-postgres} - DATABASE_PASSWORD=${DATABASE_PASSWORD-postgres} - REDIS_HOST=${REDIS_HOST-redis} - REDIS_PORT=${REDIS_PORT-6379} - DEVELOPMENT=${DEVELOPMENT-True} - CELERY_LOG_LEVEL=${CELERY_LOG_LEVEL-INFO} - DJANGO_LOG_LEVEL=${DJANGO_LOG_LEVEL-INFO} - UNLEASH_LOG_LEVEL=${UNLEASH_LOG_LEVEL-WARNING} - DJANGO_SETTINGS_MODULE=koku.settings - PROMETHEUS_MULTIPROC_DIR=/tmp - PROMETHEUS_PUSHGATEWAY=${PROMETHEUS_PUSHGATEWAY-pushgateway:9091} - ENABLE_S3_ARCHIVING=${ENABLE_S3_ARCHIVING-False} - ENABLE_HCS_DEBUG=${ENABLE_HCS_DEBUG-False} - ENABLE_SUBS_DEBUG=${ENABLE_SUBS_DEBUG-False} - S3_BUCKET_NAME=${S3_BUCKET_NAME-koku-bucket} - S3_BUCKET_PATH=${S3_BUCKET_PATH-data_archive} - S3_ENDPOINT - S3_ACCESS_KEY - S3_SECRET - DATA_DIR=${DATA_DIR-/testing/data} - GOOGLE_APPLICATION_CREDENTIALS=${GOOGLE_APPLICATION_CREDENTIALS-} - KOKU_CELERY_ENABLE_SENTRY - KOKU_CELERY_SENTRY_DSN - KOKU_SENTRY_ENVIRONMENT=development - DEMO_ACCOUNTS - INITIAL_INGEST_OVERRIDE=${INITIAL_INGEST_OVERRIDE-False} - INITIAL_INGEST_NUM_MONTHS=${INITIAL_INGEST_NUM_MONTHS-2} - POLLING_TIMER=${POLLING_TIMER-60} - POLLING_BATCH_SIZE=${POLLING_BATCH_SIZE-100} - TAG_ENABLED_LIMIT=${TAG_ENABLED_LIMIT-200} - AUTO_DATA_INGEST=${AUTO_DATA_INGEST-True} - REPORT_PROCESSING_BATCH_SIZE=${REPORT_PROCESSING_BATCH_SIZE-100000} - TRINO_HOST=${TRINO_HOST-trino} - TRINO_PORT=${TRINO_PORT-8080} - SOURCES_API_HOST=${SOURCES_API_HOST-sources-server} - SOURCES_API_PORT=${SOURCES_API_PORT-3000} - DATE_OVERRIDE - TRINO_DATE_STEP=${TRINO_DATE_STEP-31} - MAX_CELERY_TASKS_PER_WORKER=${MAX_CELERY_TASKS_PER_WORKER-10} - RETAIN_NUM_MONTHS=${RETAIN_NUM_MONTHS-4} - NOTIFICATION_CHECK_TIME=${NOTIFICATION_CHECK_TIME-24} - UNLEASH_HOST=${UNLEASH_HOST-unleash} - UNLEASH_PREFIX=${UNLEASH_PREFIX-http} - UNLEASH_PORT=${UNLEASH_PORT-4242} - UNLEASH_TOKEN=${UNLEASH_TOKEN:?} - OCI_CLI_USER=${OCI_CLI_USER} - OCI_CLI_FINGERPRINT=${OCI_CLI_FINGERPRINT} - OCI_CLI_TENANCY=${OCI_CLI_TENANCY} - OCI_CLI_KEY_FILE=${OCI_CLI_KEY_FILE} - OCI_PYTHON_SDK_NO_SERVICE_IMPORTS=True - ENHANCED_ORG_ADMIN=${ENHANCED_ORG_ADMIN-True} ports: - 6001-6020:9000 volumes: - './koku:/koku/koku' - './.unleash:/koku/.unleash' - './db_functions:/koku/db_functions' - './scripts:/koku/scripts' - './testing:/testing' - './testing/local_providers/azure_local:/tmp/local_container' - './testing/local_providers/oci_local:/tmp/oci_local' - './testing/local_providers/aws_local:/tmp/local_bucket' - './testing/local_providers/aws_local_0:/tmp/local_bucket_0' - './testing/local_providers/aws_local_1:/tmp/local_bucket_1' - './testing/local_providers/aws_local_2:/tmp/local_bucket_2' - './testing/local_providers/aws_local_3:/tmp/local_bucket_3' - './testing/local_providers/aws_local_4:/tmp/local_bucket_4' - './testing/local_providers/aws_local_5:/tmp/local_bucket_5' - './testing/local_providers/insights_local:/var/tmp/masu/insights_local' - './testing/local_providers/gcp_local/:/tmp/gcp_local_bucket' - './testing/local_providers/gcp_local_0/:/tmp/gcp_local_bucket_0' - './testing/local_providers/gcp_local_1/:/tmp/gcp_local_bucket_1' - './testing/local_providers/gcp_local_2/:/tmp/gcp_local_bucket_2' - './testing/local_providers/gcp_local_3/:/tmp/gcp_local_bucket_3' - './testing/auth_files/:/testing/auth_files/' depends_on: - koku-base koku-listener: container_name: koku_listener image: koku_base working_dir: /koku/ entrypoint: ['watchmedo', 'auto-restart', '--directory=./koku', '--pattern=*.py', '--ignore-patterns=*test*', '--recursive', '--', 'python', 'koku/manage.py', 'listener'] environment: - DJANGO_READ_DOT_ENV_FILE=True - DATABASE_SERVICE_NAME=POSTGRES_SQL - POSTGRES_SQL_SERVICE_HOST=db - POSTGRES_SQL_SERVICE_PORT=5432 - DATABASE_ENGINE=postgresql - DATABASE_NAME=${DATABASE_NAME-postgres} - DATABASE_HOST=db - DATABASE_PORT=5432 - DATABASE_USER=${DATABASE_USER-postgres} - DATABASE_PASSWORD=${DATABASE_PASSWORD-postgres} - REDIS_HOST=${REDIS_HOST-redis} - REDIS_PORT=${REDIS_PORT-6379} - INSIGHTS_KAFKA_HOST=kafka - INSIGHTS_KAFKA_PORT=29092 - KAFKA_CONNECT=True - PROMETHEUS_MULTIPROC_DIR=/tmp - MASU_DATE_OVERRIDE - KOKU_LOG_LEVEL=${KOKU_LOG_LEVEL-INFO} - DJANGO_LOG_LEVEL=${DJANGO_LOG_LEVEL-INFO} - UNLEASH_LOG_LEVEL=${UNLEASH_LOG_LEVEL-WARNING} - INITIAL_INGEST_NUM_MONTHS=1 - PYTHONPATH=/koku/koku - ENABLE_S3_ARCHIVING=${ENABLE_S3_ARCHIVING-False} - S3_BUCKET_NAME=${S3_BUCKET_NAME-koku-bucket} - S3_BUCKET_PATH=${S3_BUCKET_PATH-data_archive} - S3_ENDPOINT - S3_ACCESS_KEY - S3_SECRET - TRINO_HOST=${TRINO_HOST-trino} - TRINO_PORT=${TRINO_PORT-8080} - UNLEASH_HOST=${UNLEASH_HOST-unleash} - UNLEASH_PREFIX=${UNLEASH_PREFIX-http} - UNLEASH_PORT=${UNLEASH_PORT-4242} - UNLEASH_TOKEN=${UNLEASH_TOKEN:?} - ENHANCED_ORG_ADMIN=${ENHANCED_ORG_ADMIN-True} ports: - 7001-7020:9000 volumes: - './koku:/koku/koku' - './.unleash:/koku/.unleash' - './db_functions:/koku/db_functions' - './scripts:/koku/scripts' depends_on: - koku-base subs-worker: container_name: subs_worker image: koku_base working_dir: /koku/koku entrypoint: ['watchmedo', 'auto-restart', '--directory=./', '--pattern=*.py', '--ignore-patterns=*test*', '--recursive', '--', 'celery', '-A', 'koku', 'worker', '--without-gossip', '-l', 'info', '-Q', 'subs_extraction,subs_transmission'] environment: - DJANGO_READ_DOT_ENV_FILE=True - DATABASE_SERVICE_NAME=POSTGRES_SQL - DATABASE_ENGINE=postgresql - DATABASE_NAME=${DATABASE_NAME-postgres} - POSTGRES_SQL_SERVICE_HOST=db - POSTGRES_SQL_SERVICE_PORT=5432 - DATABASE_USER=${DATABASE_USER-postgres} - DATABASE_PASSWORD=${DATABASE_PASSWORD-postgres} - KOKU_API_HOST=${KOKU_API_HOST-koku-server} - KOKU_API_PORT=${KOKU_API_PORT-8000} - KOKU_API_PATH_PREFIX=${KOKU_API_PATH_PREFIX-/api/cost-management/v1} - ENABLE_SUBS_DEBUG=${ENABLE_SUBS_DEBUG-False} - S3_BUCKET_NAME=${S3_BUCKET_NAME-koku-bucket} - S3_BUCKET_PATH=${S3_BUCKET_PATH-data_archive} - S3_ENDPOINT - S3_ACCESS_KEY - S3_SECRET - AWS_ACCESS_KEY_ID - AWS_SECRET_ACCESS_KEY - REDIS_HOST=${REDIS_HOST-redis} - REDIS_PORT=${REDIS_PORT-6379} - INSIGHTS_KAFKA_HOST=${INSIGHTS_KAFKA_HOST-kafka} - INSIGHTS_KAFKA_PORT=${INSIGHTS_KAFKA_PORT-29092} - PROMETHEUS_MULTIPROC_DIR=/tmp - KOKU_LOG_LEVEL=${KOKU_LOG_LEVEL-DEBUG} - DJANGO_LOG_LEVEL=${DJANGO_LOG_LEVEL-INFO} - UNLEASH_LOG_LEVEL=${UNLEASH_LOG_LEVEL-WARNING} - AUTO_DATA_INGEST=${AUTO_DATA_INGEST-True} - RUN_GUNICORN=${RUN_GUNICORN-} - POD_CPU_LIMIT=${POD_CPU_LIMIT-1} - GUNICORN_WORKERS=${GUNICORN_WORKERS-3} - GUNICORN_THREADS=${GUNICORN_THREADS-False} - UNLEASH_HOST=${UNLEASH_HOST-unleash} - UNLEASH_PREFIX=${UNLEASH_PREFIX-http} - UNLEASH_PORT=${UNLEASH_PORT-4242} - UNLEASH_TOKEN=${UNLEASH_TOKEN:?} - ENHANCED_ORG_ADMIN=${ENHANCED_ORG_ADMIN-True} ports: - 7021:9000 volumes: - './koku:/koku/koku' - './.unleash:/koku/.unleash' - './db_functions:/koku/db_functions' - './scripts:/koku/scripts' depends_on: - koku-base sources-client: container_name: sources_client image: koku_base restart: on-failure:25 working_dir: /koku/ entrypoint: ['watchmedo', 'auto-restart', '--directory=./koku', '--pattern=*.py', '--ignore-patterns=*test*', '--recursive', '--', 'python', 'koku/manage.py', 'sources'] environment: - DJANGO_READ_DOT_ENV_FILE=True - SOURCES=True - DATABASE_SERVICE_NAME=POSTGRES_SQL - DATABASE_ENGINE=postgresql - DATABASE_NAME=${DATABASE_NAME-postgres} - POSTGRES_SQL_SERVICE_HOST=db - POSTGRES_SQL_SERVICE_PORT=5432 - DATABASE_USER=${DATABASE_USER-postgres} - DATABASE_PASSWORD=${DATABASE_PASSWORD-postgres} - KOKU_API_HOST=${KOKU_API_HOST-koku-server} - KOKU_API_PORT=${KOKU_API_PORT-8000} - KOKU_API_PATH_PREFIX=${KOKU_API_PATH_PREFIX-/api/cost-management/v1} - SOURCES_API_SVC_HOST=${SOURCES_API_HOST-sources-server} - SOURCES_API_SVC_PORT=${SOURCES_API_PORT-3000} - REDIS_HOST=${REDIS_HOST-redis} - REDIS_PORT=${REDIS_PORT-6379} - INSIGHTS_KAFKA_HOST=${INSIGHTS_KAFKA_HOST-kafka} - INSIGHTS_KAFKA_PORT=${INSIGHTS_KAFKA_PORT-29092} - KOKU_SOURCES_CLIENT_PORT=${KOKU_SOURCES_CLIENT_PORT-9000} - GOOGLE_APPLICATION_CREDENTIALS=${GOOGLE_APPLICATION_CREDENTIALS-} - DEMO_ACCOUNTS - PROMETHEUS_MULTIPROC_DIR=/tmp - KOKU_LOG_LEVEL=${KOKU_LOG_LEVEL-DEBUG} - DJANGO_LOG_LEVEL=${DJANGO_LOG_LEVEL-INFO} - UNLEASH_LOG_LEVEL=${UNLEASH_LOG_LEVEL-WARNING} - SOURCES_API_PREFIX=${SOURCES_API_PREFIX-/api/sources/v1.0} - SOURCES_PSK=${SOURCES_PSK-thisMustBeEphemeralOrMinikube} - AUTO_DATA_INGEST=${AUTO_DATA_INGEST-True} - RUN_GUNICORN=${RUN_GUNICORN-} - POD_CPU_LIMIT=${POD_CPU_LIMIT-1} - GUNICORN_WORKERS=${GUNICORN_WORKERS-3} - GUNICORN_THREADS=${GUNICORN_THREADS-False} - UNLEASH_HOST=${UNLEASH_HOST-unleash} - UNLEASH_PREFIX=${UNLEASH_PREFIX-http} - UNLEASH_PORT=${UNLEASH_PORT-4242} - UNLEASH_TOKEN=${UNLEASH_TOKEN:?} - OCI_CLI_USER=${OCI_CLI_USER} - OCI_CLI_FINGERPRINT=${OCI_CLI_FINGERPRINT} - OCI_CLI_TENANCY=${OCI_CLI_TENANCY} - OCI_CLI_KEY_FILE=${OCI_CLI_KEY_FILE} - OCI_PYTHON_SDK_NO_SERVICE_IMPORTS=True - ENHANCED_ORG_ADMIN=${ENHANCED_ORG_ADMIN-True} ports: - 4000:8000 - 4001:9000 volumes: - './koku:/koku/koku' - './.unleash:/koku/.unleash' - './db_functions:/koku/db_functions' - './scripts:/koku/scripts' - './testing/auth_files/:/testing/auth_files/' depends_on: - koku-base koku-beat: container_name: koku_beat hostname: koku_beat image: koku_base working_dir: /koku/koku entrypoint: ['celery', '-A', 'koku', 'beat', '-l', 'info'] environment: - AWS_ACCESS_KEY_ID - AWS_SECRET_ACCESS_KEY - DATABASE_SERVICE_NAME=POSTGRES_SQL - DATABASE_ENGINE=postgresql - DATABASE_NAME=${DATABASE_NAME-postgres} - POSTGRES_SQL_SERVICE_HOST=db - POSTGRES_SQL_SERVICE_PORT=5432 - DATABASE_USER=${DATABASE_USER-postgres} - DATABASE_PASSWORD=${DATABASE_PASSWORD-postgres} - REDIS_HOST=${REDIS_HOST-redis} - REDIS_PORT=${REDIS_PORT-6379} - LOG_LEVEL=INFO - DJANGO_SETTINGS_MODULE=koku.settings - PROMETHEUS_MULTIPROC_DIR=/tmp - SCHEDULE_REPORT_CHECKS=True - SOURCE_STATUS_FREQUENCY_MINUTES - UPLOAD_NORMALIZED_DATA_INTERVAL=10000000 - UNLEASH_HOST=${UNLEASH_HOST-unleash} - UNLEASH_PREFIX=${UNLEASH_PREFIX-http} - UNLEASH_PORT=${UNLEASH_PORT-4242} - UNLEASH_TOKEN=${UNLEASH_TOKEN:?} - ENHANCED_ORG_ADMIN=${ENHANCED_ORG_ADMIN-True} volumes: - './koku:/koku/koku' - './.unleash:/koku/.unleash' - './db_functions:/koku/db_functions' - './scripts:/koku/scripts' depends_on: - koku-base ### Utilities db: container_name: koku_db image: postgres:13 environment: - POSTGRES_DB=${DATABASE_NAME-postgres} - POSTGRES_USER=${DATABASE_USER-postgres} - POSTGRES_PASSWORD=${DATABASE_PASSWORD-postgres} - _PG_CREATE_DATABASES=${DATABASE_NAME-postgres}|${DATABASE_USER-postgres},unleash|${DATABASE_USER-postgres} ports: - 15432:5432 volumes: - ./pg_data/data:/var/lib/${DATABASE_DATA_DIR-postgresql}/data - ./pg_init:/docker-entrypoint-initdb.d command: # This command give more precise control over the parameter settings # Included are loading the pg_stat_statements lib # as well as autovacuum settings that are designed to make more efficient # use of the autovacuum processes # The pg_init mount is still needed to enable the necesary extensions. - postgres - -c - ssl=on - -c - ssl_cert_file=/etc/ssl/certs/ssl-cert-snakeoil.pem - -c - ssl_key_file=/etc/ssl/private/ssl-cert-snakeoil.key - -c - max_connections=1710 - -c - shared_preload_libraries=pg_stat_statements - -c - autovacuum_max_workers=8 - -c - autovacuum_vacuum_cost_limit=4800 - -c - autovacuum_vacuum_cost_delay=10 - -c - max_locks_per_transaction=72 - -c - pg_stat_statements.max=2000 - -c - pg_stat_statements.save=off - -c - pg_stat_statements.track_utility=off - -c - track_activity_query_size=2048 - -c - track_functions=pl - -c - log_min_error_statement=error - -c - log_min_duration_statement=2000 - -c - logging_collector=on # Log line prefix (this matches RDS): # %t = timestamp without milliseconds # %r = remote host name/ip and remote port # %u = user name # %d = database name # %p = process id (postgres backend) - -c - "log_line_prefix=%t:%r:%u@%d:[%p]:" # Change mod to all to log all statements including SELECT - -c - log_statement=mod # Uncomment this to get duration information on all statements, whether # the statement is logged or not. # - -c # - log_duration=true # Uncomment this parameter to get statistics details in the logs # This will really increase the log file size # - -c # - log_statement_stats=true redis: container_name: koku_redis image: redis:5.0.4 ports: - 6379:6379 pushgateway: container_name: koku-pushgateway image: prom/pushgateway ports: - 9091:9091 pgadmin: container_name: pgAdmin image: dpage/pgadmin4 environment: - PGADMIN_DEFAULT_EMAIL=${PGADMIN_EMAIL-postgres@local.dev} - PGADMIN_DEFAULT_PASSWORD=${PGADMIN_PASSWORD-postgres} ports: - "${PGADMIN_PORT-8432}:80" volumes: - ./pgadmin_servers.json:/pgadmin4/servers.json grafana: container_name: koku_grafana build: context: grafana dockerfile: Dockerfile-grafana ports: - 3001:3000 depends_on: - db unleash: container_name: koku-unleash image: unleashorg/unleash-server:4.21.0 environment: - INIT_ADMIN_API_TOKENS=${UNLEASH_TOKEN:?} - CHECK_VERSION=false - DATABASE_HOST=db - DATABASE_NAME=unleash - DATABASE_USERNAME=${DATABASE_USER-postgres} - DATABASE_PASSWORD=${DATABASE_PASSWORD-postgres} - DATABASE_SSL=false - IMPORT_DROP_BEFORE_IMPORT=false - IMPORT_FILE=/.unleash/flags.json - LOG_LEVEL=INFO ports: - 4242:4242 volumes: - './.unleash:/.unleash' depends_on: - db koku-minio: image: local-minio:latest build: context: . dockerfile: Dockerfile.minio args: USER_ID: ${USER_ID-1000} container_name: kokuminio environment: MINIO_ROOT_USER: kokuminioaccess MINIO_ROOT_PASSWORD: kokuminiosecret ports: - 9000:9000 - 9090:9090 command: server /data --console-address ":9090" volumes: - ./.trino/parquet_data:/data koku-create-parquet-bucket: image: minio/mc:latest depends_on: - koku-minio entrypoint: > /bin/sh -c " /usr/bin/mc config host rm local; /usr/bin/mc config host add --quiet --api s3v4 local http://kokuminio:9000 kokuminioaccess kokuminiosecret; /usr/bin/mc rb --force local/koku-bucket/; /usr/bin/mc mb --quiet local/koku-bucket/; /usr/bin/mc policy set public local/koku-bucket; /usr/bin/mc rb --force local/metastore/; /usr/bin/mc mb --quiet local/metastore/; /usr/bin/mc policy set public local/metastore; " hive-metastore: container_name: hive-metastore image: quay.io/cloudservices/ubi-hive:3.1.2-metastore-008 ports: - 9083:8000 environment: - HIVE_LOGLEVEL=INFO - S3_HOST=kokuminio - S3_PORT=9000 - S3_ACCESS_KEY=${S3_ACCESS_KEY-kokuminioaccess} - S3_SECRET=${S3_SECRET-kokuminiosecret} - S3_BUCKET_NAME=${S3_BUCKET_NAME-koku-bucket} - S3_BUCKET_PATH=${S3_BUCKET_PATH-data} - POSTGRES_SQL_SERVICE_HOST=db - POSTGRES_SQL_SERVICE_PORT=5432 - DATABASE_NAME=${DATABASE_NAME-hive} - DATABASE_USER=${DATABASE_USER-hive} - DATABASE_PASSWORD=${DATABASE_PASSWORD-hive} depends_on: - koku-create-parquet-bucket trino: container_name: trino image: quay.io/cloudservices/ubi-trino:411-002 user: root ports: - 8080:8080 - 10000:10000 environment: - MY_NODE_ID=${MY_NODE_ID-localhost} - LOCAL=TRUE - TRINO_LOG_LEVEL=${LOG_LEVEL-INFO} - AWS_ACCESS_KEY_ID=${S3_ACCESS_KEY-kokuminioaccess} - AWS_SECRET_ACCESS_KEY=${S3_SECRET-kokuminiosecret} - S3_ENDPOINT=${S3_ENDPOINT-http://kokuminio:9000} - S3_BUCKET_NAME=${S3_BUCKET_NAME-koku-bucket} - S3_ACCESS_KEY=${S3_ACCESS_KEY-kokuminioaccess} - S3_SECRET=${S3_SECRET-kokuminiosecret} - DATABASE_NAME=${DATABASE_NAME-postgres} - DATABASE_USER=${DATABASE_USER-postgres} - DATABASE_PASSWORD=${DATABASE_PASSWORD-postgres} volumes: - ./deploy/trino/etc/config.properties:/etc/trino/config.properties - ./deploy/trino/etc/jvm.config:/etc/trino/jvm.config - ./deploy/trino/etc/log.properties:/etc/trino/log.properties - ./deploy/trino/etc/catalog/hive.properties:/etc/trino/catalog/hive.properties - ./deploy/trino/etc/catalog/postgres.properties:/etc/trino/catalog/postgres.properties - ./deploy/hadoop/hadoop-config/core-site.xml:/etc/trino/hadoop-config/core-site.xml - ./.trino/trino/trino-data:/data/trino/data - ./.trino/trino/trino-logs:/data/trino/logs links: - hive-metastore depends_on: - hive-metastore ulimits: nofile: soft: 4096 hard: 4096