heat_template_version: 2015-04-30

description: Sets up a hadoop-spark cluster

parameters:
  num-slaves:
    type: number
    label: Number of slave nodes
    description: Number of slave nodes for your cluster
    default: 1
  master-image:
    type: string
    label: Master Node Image
    description: ID or name of the image to use for the master node
    constraints:
      - custom_constraint: glance.image
    default: ubuntu-server-14.04-amd64
  master-flavor:
    type: string
    label: Master Node Flavor
    description: Hardware flavor to be used for the master node
    constraints:
      - custom_constraint: nova.flavor
    default: c1-7.5gb-30
  slave-image:
    type: string
    label: Slave Node Image
    description: ID or name of the image to use for the slave nodes
    constraints:
      - custom_constraint: glance.image
    default: ubuntu-server-14.04-amd64
  slave-flavor:
    type: string
    label: Slave Node Flavor
    description: Hardware flavor to be used for the slave nodes
    constraints:
      - custom_constraint: nova.flavor
    default: c1-7.5gb-30
  key:
    type: string
    label: Key-pair name
    description: Name of key-pair to be used for master and slave nodes
    default: thekey
    constraints:
      - custom_constraint: nova.keypair
  private_network:
    type: string
    label: Private network name or ID
    description: Network to attach instance to.
    constraints:
      - custom_constraint: neutron.network
  public_network:
    type: string
    description: Network to use for obtaining public IP (Public-Network for Arbutus, net04_ext for east-cloud)
    label: Public Network
    default: Public-Network
    constraints:
      - allowed_values:
        - Public-Network
        - net04_ext
  CIDR_rule:
    type: string
    description: CIDR rule to use for restricting access based on IP addresses. If in doubt use the address given at http://ipv4.icanhazip.com/ with a "/32" appended e.g. enter something like "216.123.12.43/32".
    label: CIDR access rule
    
resources:
  hs-server-group:
    type: OS::Nova::ServerGroup
    properties:
      name: hs-different-nodes
      policies: ['anti-affinity']
  hs-master:
    type: OS::Nova::Server
    properties:
      flavor: {get_param: master-flavor}
      image: {get_param: master-image}
      key_name: {get_param: key}
      name: hs-master
      scheduler_hints: {'group': {get_resource: hs-server-group}}
      networks:
        - port: {get_resource: hs-master_port}
      user_data_format: RAW
      user_data:
        str_replace:
          template: |
            #cloud-config
            apt_sources:
              - source: "ppa:saltstack/salt"
            packages:
              - python-software-properties
              - git
              - salt-master
              - salt-minion
              - make
            write_files:
              - content: |
                    roles:
                       - hadoop_master
                       - spark_target
                    hdfs_data_disks:
                       - /mnt
                path: /etc/salt/grains
              - content: |
                    #!/bin/bash
                    sed -i 's/^#auto_accept.*$/auto_accept: True/g' /etc/salt/master
                    sudo service salt-master restart
                path: /tmp/bootstrap-master.sh
                permissions: "0755"
              - content: |
                    #!/bin/bash
                    sed -i 's/^#master:.*$/master: 127.0.0.1/g' /etc/salt/minion
                    sudo service salt-minion restart
                path: /tmp/bootstrap-minion.sh
                permissions: "0755"
              - content: |
                  file_roots:
                    base:
                      - /srv/salt
                      - /srv/salt/formulas/hadoop-formula
                      - /srv/salt/formulas/hostsfile-formula
                      - /srv/salt/formulas/sun-java-formula
                path: /etc/salt/master.d/file_roots.conf
                permissions: '0644'
              - content: |
                  pillar_roots:
                    base:
                      - /srv/pillar
                path: /etc/salt/master.d/pillar_roots.conf
              - content: |
                  base:
                    'G@roles:hadoop_slave or G@roles:hadoop_master':
                      - match: compound
                      - hostsfile
                      - hostsfile.hostname
                      - sun-java
                      - sun-java.env
                      - hadoop
                      - hadoop.hdfs
                      - hadoop.mapred
                      - hadoop.yarn
                      - hadoop.spark
                path: /srv/salt/top.sls
              - content: |
                  base:
                    'G@roles:hadoop_slave or G@roles:hadoop_master':
                       - hadoop
                path: /srv/pillar/top.sls
              - content: |
                  spark:
                    version: spark-2.4.1-bin-hadoop2.7
                    spark_target: "G@roles:hadoop_slave or G@roles:hadoop_master"
                    spark_master: "G@roles:hadoop_master"
                    spark_slave: "G@roles:hadoop_slave"
                    targeting_method: compound
                  hadoop:
                    version: apache-2.7.1 # ['apache-1.2.1', 'apache-2.2.0', 'hdp-1.3.0', 'hdp-2.2.0', 'cdh-4.5.0', 'cdh-4.5.0-mr1']
                    targeting_method: grain # [compound, glob] also supported
                    users:
                      hadoop: 6000
                      hdfs: 6001
                      mapred: 6002
                      yarn: 6003
                    config:
                      directory: /etc/hadoop/conf
                      core-site:
                        io.native.lib.available:
                          value: true
                        io.file.buffer.size:
                          value: 65536
                        fs.trash.interval:
                          value: 60
                  hdfs:
                    namenode_target: "roles:hadoop_master" # Specify compound matching string to match all your namenodes
                    datanode_target: "roles:hadoop_slave" # Specify compound matching string to match all your datanodes e.g. if you were to use pillar I@datanode:true
                    config:
                      namenode_port: 8020
                      namenode_http_port: 50070
                      secondarynamenode_http_port: 50090
                      # the number of hdfs replicas is normally auto-configured for you in hdfs.settings
                      # according to the number of available datanodes
                      # replication: 1
                      hdfs-site:
                        dfs.permission:
                          value: false
                        dfs.durable.sync:
                          value: true
                        dfs.datanode.synconclose:
                          value: true
                  mapred:
                    jobtracker_target: "roles:hadoop_master"
                    tasktracker_target: "roles:hadoop_slave"
                    config:
                      jobtracker_port: 9001
                      jobtracker_http_port: 50030
                      jobhistory_port: 10020
                      jobhistory_webapp_port: 19888
                      history_dir: /mr-history
                      mapred-site:
                        mapred.map.memory.mb:
                          value: 1536
                        mapred.map.java.opts:
                          value: -Xmx1024M
                        mapred.reduce.memory.mb:
                          value: 3072
                        mapred.reduce.java.opts:
                          value: -Xmx1024m
                        mapred.task.io.sort.mb:
                          value: 512
                        mapred.task.io.sort.factor:
                          value: 100
                        mapred.reduce.shuffle.parallelcopies:
                          value: 50
                        mapreduce.framework.name:
                          value: yarn
                  yarn:
                    resourcemanager_target: "roles:hadoop_master"
                    nodemanager_target: "roles:hadoop_slave"
                    config:
                      yarn-site:
                        yarn.nodemanager.aux-services:
                          value: mapreduce_shuffle
                        yarn.nodemanager.aux-services.mapreduce.shuffle.class:
                          value: org.apache.hadoop.mapred.ShuffleHandler
                        yarn.resourcemanager.scheduler.class:
                          value: org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacityScheduler
                        yarn.log-aggregation-enable:
                          value: True
                      capacity-scheduler:
                        yarn.scheduler.capacity.maximum-applications:
                          value: 10000
                        yarn.scheduler.capacity.resource-calculator:
                          value: org.apache.hadoop.yarn.util.resource.DefaultResourceCalculator
                        yarn.scheduler.capacity.root.queues:
                          value: default,customqueue
                        yarn.scheduler.capacity.root.capacity:
                          value: 100
                        yarn.scheduler.capacity.root.default.capacity:
                          value: 70
                        yarn.scheduler.capacity.root.default.user-limit-factor:
                          value: 1
                        yarn.scheduler.capacity.root.default.maximum-capacity:
                          value: 100
                        yarn.scheduler.capacity.root.default.state:
                          value: RUNNING
                        yarn.scheduler.capacity.root.default.acl_submit_applications:
                          value: '*'
                        yarn.scheduler.capacity.root.default.acl_administer_queue:
                          value: '*'
                        yarn.scheduler.capacity.root.customqueue.capacity:
                          value: 30
                        yarn.scheduler.capacity.root.customqueue.maximum-am-resource-percent:
                          value: 1.0
                        yarn.scheduler.capacity.root.customqueue.user-limit-factor:
                          value: 1
                        yarn.scheduler.capacity.root.customqueue.maximum-capacity:
                          value: 100
                        yarn.scheduler.capacity.root.customqueue.state:
                          value: RUNNING
                        yarn.scheduler.capacity.root.customqueue.acl_submit_applications:
                          value: '*'
                        yarn.scheduler.capacity.root.customqueue.acl_administer_queue:
                          value: '*'
                        yarn.scheduler.capacity.node-locality-delay:
                          value: -1
                path: /srv/pillar/hadoop.sls
              - content: |
                    mine_functions:
                        network.interfaces: []
                        network.ip_addrs: []
                        grains.items: []
                path: /etc/salt/minion.d/mine_functions.conf
              - content: |
                  #!/bin/bash
                  wc_notify --data-binary '{"status": "SUCCESS", "reason": "master server CI done"}'
                path: /tmp/finished.sh
                permissions: "0755"
            runcmd:
              - bash sed -i '/force_color_prompt=yes/s/^#//g' /home/ubuntu/.bashrc
              - bash /tmp/bootstrap-master.sh
              - bash /tmp/bootstrap-minion.sh
              - git clone https://github.com/cgeroux/hadoop-formula.git /srv/salt/formulas/hadoop-formula
              - git clone https://github.com/cgeroux/hostsfile-formula.git /srv/salt/formulas/hostsfile-formula
              - git clone https://github.com/cgeroux/sun-java-formula.git /srv/salt/formulas/sun-java-formula
              - git clone -b heat-compatible https://github.com/cgeroux/init-salt.git /tmp/salt-init
              - sudo service salt-master restart
              - sudo service salt-minion restart
              - cd /srv/salt/formulas/hadoop-formula/hadoop/files;./generate_keypairs.sh
              - /tmp/salt-init/salt-init.py MASTERNAME SLAVENAMEBASE NUMSLAVES
              - sudo -H -u spark bash -c "/usr/lib/spark/sbin/start-all.sh"
              - sudo -H -u ubuntu bash -c 'git clone https://github.com/cgeroux/big-data-examples.git $HOME/big-data-examples'
              - bash /tmp/finished.sh
          params:
            MASTERNAME: hs-master
            SLAVENAMEBASE: hs-slave-
            NUMSLAVES: {get_param: num-slaves}
            wc_notify: {get_attr: ['master_wait_handle','curl_cli']}
  hs-master_port:
    type: OS::Neutron::Port
    properties:
      network: {get_param: private_network}
      security_groups: [{get_resource: hs-security},default]
  hs-security:
    type: OS::Neutron::SecurityGroup
    properties:
      name: hs-security
      rules:
        - remote_ip_prefix: {get_param: CIDR_rule}
          protocol: tcp
          port_range_min: 22
          port_range_max: 22
        - remote_ip_prefix: {get_param: CIDR_rule}
          protocol: tcp
          port_range_min: 50070
          port_range_max: 50070
        - remote_ip_prefix: {get_param: CIDR_rule}
          protocol: tcp
          port_range_min: 8088
          port_range_max: 8088
  hs-master_floating_ip:
    type: OS::Neutron::FloatingIP
    properties:
      port_id: {get_resource: hs-master_port}
      floating_network: {get_param: public_network}
  slave_group:
    type: OS::Heat::ResourceGroup
    properties:
      count: {get_param: num-slaves}
      resource_def:
        type: OS::Nova::Server
        properties:
          name: hs-slave-%index%
          flavor: {get_param: slave-flavor}
          image: {get_param: slave-image}
          key_name: {get_param: key}
          scheduler_hints: {'group': {get_resource: hs-server-group}}
          networks: 
            - network: {get_param: private_network}
          user_data_format: RAW
          user_data:
            str_replace:
              template: |
                #cloud-config
                apt_sources:
                  - source: "ppa:saltstack/salt"
                packages: 
                  - python-software-properties
                  - salt-minion
                write_files: 
                  - content: |
                         roles:
                            - hadoop_slave
                            - spark_target
                         hdfs_data_disks:
                            - /mnt
                    path: /etc/salt/grains
                  - content: |
                        #!/bin/bash
                        sed -i 's/^#master:.*$/master: IPTOREPLACE/g' /etc/salt/minion
                        sudo service salt-minion restart
                    path: /tmp/bootstrap-minion.sh
                    permissions: "0755"
                  - content: |
                        mine_functions:
                            network.interfaces: []
                            network.ip_addrs: []
                            grains.items: []
                    path: /etc/salt/minion.d/mine_functions.conf
                  - content: |
                      #!/bin/bash
                      wc_notify --data-binary '{"status": "SUCCESS", "reason": "Slave CI done"}'
                    path: /tmp/finished.sh
                    permissions: "0755"
                runcmd: 
                 - bash sed -i '/force_color_prompt=yes/s/^#//g' /home/ubuntu/.bashrc
                 - bash /tmp/bootstrap-minion.sh
                 - bash /tmp/finished.sh
              params:
                IPTOREPLACE: { get_attr: [hs-master, first_address] }
                wc_notify: {get_attr: ['slave_group_wait_handle','curl_cli']}
  master_wait_handle:
    type: OS::Heat::WaitConditionHandle
  slave_group_wait_handle:
    type: OS::Heat::WaitConditionHandle
  master_wait_condition:
    type: OS::Heat::WaitCondition
    properties:
      handle: {get_resource: master_wait_handle}
      count: 1
      timeout: 1200
  slave_group_wait_condition:
    type: OS::Heat::WaitCondition
    properties:
      handle: {get_resource: slave_group_wait_handle}
      count: {get_param: num-slaves}
      timeout: 1200
outputs:
  public_ip:
    description: Floating IP address of hs-master in public network, can be used to ssh into master node
    value: { get_attr: [ hs-master_floating_ip, floating_ip_address ] }
  node_info_url:
    description: Link to hdfs node information page
    value:
      str_replace:
        template: http://<hostip>:50070
        params:
          <hostip>: { get_attr: [ hs-master_floating_ip, floating_ip_address ] }
  yarn_url:
    description: Link to yarn job page
    value:
      str_replace:
        template: http://<hostip>:8088
        params:
          <hostip>: { get_attr: [ hs-master_floating_ip, floating_ip_address ] }