# docker-compose.yml # ------------------ # Spins up a minimal single-node "cluster" for the assignment: # - HDFS: namenode + datanode # - YARN: resourcemanager + nodemanager (used by Hadoop Streaming) # - Hive: metastore + hiveserver2 (SQL on top of HDFS) # - Spark: master + worker with PySpark # # Images are from the well-maintained `bde2020` Hadoop/Hive project and # the official Apache Spark `spark:python3` image on Docker Hub. (An earlier # version of this file used bitnami/spark, but Bitnami moved most of its # versioned tags behind a paid subscription in 2025, so it's no longer # pullable for free — the official Apache Spark image replaces it here.) # # Usage: # docker compose up -d # start # docker compose down # stop # docker compose down -v # stop + wipe HDFS data (fresh start) x-hadoop-env: &hadoop-env CORE_CONF_fs_defaultFS: "hdfs://namenode:9000" CORE_CONF_hadoop_http_staticuser_user: "root" HDFS_CONF_dfs_replication: "1" HDFS_CONF_dfs_permissions_enabled: "false" YARN_CONF_yarn_resourcemanager_hostname: "resourcemanager" YARN_CONF_yarn_nodemanager_aux___services: "mapreduce_shuffle" YARN_CONF_yarn_nodemanager_resource_memory___mb: "3072" YARN_CONF_yarn_scheduler_minimum___allocation___mb: "256" YARN_CONF_yarn_scheduler_maximum___allocation___mb: "3072" # Without this, Hadoop silently falls back to LocalJobRunner and runs the # ENTIRE job inside whichever container issued `hadoop jar` (resourcemanager # here), never touching YARN or nodemanager at all -- which is exactly why # installing Python3 into nodemanager earlier had no effect on the error. MAPRED_CONF_mapreduce_framework_name: "yarn" MAPRED_CONF_mapreduce_map_env: "HADOOP_MAPRED_HOME=/opt/hadoop-3.2.1/" MAPRED_CONF_mapreduce_reduce_env: "HADOOP_MAPRED_HOME=/opt/hadoop-3.2.1/" MAPRED_CONF_yarn_app_mapreduce_am_env: "HADOOP_MAPRED_HOME=/opt/hadoop-3.2.1/" # Explicit, small memory requests -- with only one NodeManager at 3072MB # total, the AM's default request (1536MB) plus a map task's default # (1024MB) would only just barely fit with nothing left for a reducer, so # we shrink all three to comfortably coexist. MAPRED_CONF_yarn_app_mapreduce_am_resource_mb: "512" MAPRED_CONF_mapreduce_map_memory_mb: "512" MAPRED_CONF_mapreduce_reduce_memory_mb: "512" services: namenode: image: bde2020/hadoop-namenode:2.0.0-hadoop3.2.1-java8 container_name: namenode environment: <<: *hadoop-env CLUSTER_NAME: "bict3608" ports: - "9870:9870" # HDFS web UI - "9000:9000" volumes: - namenode_data:/hadoop/dfs/name datanode: image: bde2020/hadoop-datanode:2.0.0-hadoop3.2.1-java8 container_name: datanode depends_on: - namenode environment: <<: *hadoop-env SERVICE_PRECONDITION: "namenode:9870" volumes: - datanode_data:/hadoop/dfs/data resourcemanager: image: bde2020/hadoop-resourcemanager:2.0.0-hadoop3.2.1-java8 container_name: resourcemanager depends_on: - namenode - datanode environment: <<: *hadoop-env SERVICE_PRECONDITION: "namenode:9000 namenode:9870 datanode:9864" ports: - "8088:8088" # YARN web UI # Built from nodemanager.Dockerfile, which adds Python3 -- the stock image # has none, and this is the container where YARN actually runs mapper.py / # reducer.py for the Hadoop Streaming job (see that Dockerfile for why). nodemanager: build: context: . dockerfile: nodemanager.Dockerfile image: bict3608-nodemanager:python3 container_name: nodemanager depends_on: - resourcemanager environment: <<: *hadoop-env SERVICE_PRECONDITION: "resourcemanager:8088" hive-metastore: image: bde2020/hive:2.3.2-postgresql-metastore container_name: hive-metastore depends_on: - namenode - datanode environment: <<: *hadoop-env HIVE_SITE_CONF_javax_jdo_option_ConnectionURL: "jdbc:postgresql://hive-metastore-postgresql/metastore" HIVE_SITE_CONF_javax_jdo_option_ConnectionDriverName: "org.postgresql.Driver" HIVE_SITE_CONF_javax_jdo_option_ConnectionUserName: "hive" HIVE_SITE_CONF_javax_jdo_option_ConnectionPassword: "hive" SERVICE_PRECONDITION: "hive-metastore-postgresql:5432 namenode:9870 datanode:9864" ports: - "9083:9083" command: ["/opt/hive/bin/hive", "--service", "metastore"] hive-metastore-postgresql: image: bde2020/hive-metastore-postgresql:2.3.0 container_name: hive-metastore-postgresql hive-server: image: bde2020/hive:2.3.2-postgresql-metastore container_name: hive-server depends_on: - hive-metastore environment: <<: *hadoop-env HIVE_SITE_CONF_javax_jdo_option_ConnectionURL: "jdbc:postgresql://hive-metastore-postgresql/metastore" HIVE_SITE_CONF_javax_jdo_option_ConnectionDriverName: "org.postgresql.Driver" HIVE_SITE_CONF_javax_jdo_option_ConnectionUserName: "hive" HIVE_SITE_CONF_javax_jdo_option_ConnectionPassword: "hive" HIVE_SITE_CONF_hive_metastore_uris: "thrift://hive-metastore:9083" SERVICE_PRECONDITION: "hive-metastore:9083" ports: - "10000:10000" # HiveServer2 (JDBC/Beeline/PyHive) - "10002:10002" # Hive web UI # NOTE: bitnami/spark images were moved behind a paid "Bitnami Secure Images" # subscription in 2025 and are no longer pullable for free. We use the # official Docker Hub "spark" image (maintained by the Apache Spark project) # instead, which ships Python3/PySpark support via the "python3" tag. spark-master: image: spark:python3 container_name: spark-master user: root command: > /opt/spark/bin/spark-class org.apache.spark.deploy.master.Master --host spark-master --port 7077 --webui-port 8080 ports: - "8080:8080" # Spark master web UI - "7077:7077" - "4040:4040" # Spark job UI (driver runs here in client deploy mode) volumes: - ../spark:/opt/spark-apps # etl.py / analytics.py, ready for spark-submit - ../results:/opt/spark-apps/results # so timings.csv lands back on the host spark-worker: image: spark:python3 container_name: spark-worker user: root depends_on: - spark-master command: > /opt/spark/bin/spark-class org.apache.spark.deploy.worker.Worker spark://spark-master:7077 --webui-port 8081 --cores 2 --memory 2G ports: - "8081:8081" # Spark worker web UI volumes: - ../spark:/opt/spark-apps volumes: namenode_data: datanode_data: