version: "3"

services:
#
#BUILD HADOOP
#
  namenode:
    image: demomagic/hadoop-namenode:hadoop-3.1.2
    container_name: namenode
    ports:
      - 9870:9870
    volumes:
      - hadoop_namenode:/hadoop/dfs/name
    environment:
      - CLUSTER_NAME=master
    env_file:
      - ./hadoop.env

  datanode:
    image: demomagic/hadoop-datanode:hadoop-3.1.2
    container_name: datanode
    volumes:
      - hadoop_datanode:/hadoop/dfs/data
    environment:
      SERVICE_PRECONDITION: "namenode:9870"
    env_file:
      - ./hadoop.env
  
  resourcemanager:
    image: demomagic/hadoop-resourcemanager:hadoop-3.1.2
    container_name: resourcemanager
    environment:
      SERVICE_PRECONDITION: "namenode:9870 datanode:9864"
    env_file:
      - ./hadoop.env

  nodemanager:
    image: demomagic/hadoop-nodemanager:hadoop-3.1.2
    container_name: nodemanager
    environment:
      SERVICE_PRECONDITION: "namenode:9870 datanode:9864 resourcemanager:8088"
    env_file:
      - ./hadoop.env
  
  historyserver:
    image: demomagic/hadoop-historyserver:hadoop-3.1.2
    container_name: historyserver
    environment:
      SERVICE_PRECONDITION: "namenode:9870 datanode:9864 resourcemanager:8088"
    volumes:
      - hadoop_historyserver:/hadoop/yarn/timeline
    env_file:
      - ./hadoop.env
#
#BUILD HIVE
#
  hive-metastore:
    image: demomagic/hive:3.1.2-postgresql12-metastore
    container_name: hive-metastore
    env_file:
      - ./hive-metastore.env
    environment:
      SERVICE_PRECONDITION: "namenode:9870 datanode:9864 metastore-postgresql:5432"
      
#  hive-server:
#    image: demomagic/hive:3.1.1-postgresql12-server
#    container_name: hive-server
#    env_file:
#      - ./hive-server.env
#    environment:
#      SERVICE_PRECONDITION: "hive-metastore:9083"
#    ports:
#      - 10000:10000
#      - 10002:10002
  
#
#BUILD SPARK
#
  spark-master:
    image: demomagic/spark-master:2.4.4-scala1.12
    container_name: spark-master
    ports:
      - 8080:8080
      - 7077:7077
      
  spark-worker-1:
    image: demomagic/spark-worker:2.4.4-scala1.12
    container_name: spark-worker-1
    depends_on:
      - spark-master
    environment:
      - SPARK_MASTER=spark://spark-master:7077
    ports:
      - 8081:8081

  spark-livy:
    image: demomagic/spark-livy:2.4.4-0.6.0
    container_name: spark-livy
    depends_on:
      - spark-master
    environment:
      - SPARK_MASTER=spark://spark-master:7077
    ports:
      - 8998:8998
      
#
#BUILD HUE
#
  hue:
    image: demomagic/hue:4.5.0
    container_name: hue
    ports:
      - 8088:8088
    environment:
      - NAMENODE_HOST=namenode

  metastore-postgresql:
    image: demomagic/metastore-postgresql:12
    container_name: metastore-postgresql

volumes:
  hadoop_namenode:
  hadoop_datanode:
  hadoop_historyserver: