#cloud-config apt_upgrade: false apt_update: false manage_etc_hosts: false package_update: false package_upgrade: false write_files: #USER DATA - Cloudsigma #- path: /var/lib/micado/occopus/temp_user_data.yaml # content: | # user_data: # auth_data: # type: cloudsigma # email: YOUR_EMAIL # password: YOUR_PASSWORD # # resource: # type: cloudsigma # endpoint: YOUR_ENDPOINT # libdrive_id: UBUNTU_16.04_IMAGE_ID # description: # cpu: 1000 # mem: 1073741824 # vnc_password: secret # pubkeys: # - # KEY_UUID # nics: # - # ip_v4_conf: # conf: dhcp # # scaling: # min: 1 # max: 10 #USER DATA - EC2 #- path: /var/lib/micado/occopus/temp_user_data.yaml # content: | # user_data: # auth_data: # type: ec2 # accesskey: YOUR_ACCESS_KEY # secretkey: YOUR_SECRET_KEY # resource: # type: ec2 # endpoint: YOUR_ENDPOINT # regionname: YOUR_REGION # image_id: UBUNTU_16.04_IMAGE_ID # instance_type: YOUR_INSTANCE_TYPE # key_name: YOUR_KEY_NAME # security_group_ids: # - YOUR_SECURITY_GROUP_ID # scaling: # min: 1 # max: 10 #USER DATA - Cloud Broker #- path: /var/lib/micado/occopus/temp_user_data.yaml # content: | # user_data: # auth_data: # type: cloudbroker # email: YOUR_CLOUDBROKER_EMAIL # password: YOUR_CLOUDBROKER_PASSWORD # resource: # type: cloudbroker # endpoint: YOUR_ENDPOINT # description: # deployment_id: YOUR_DEPLOYMENT_ID # instance_type_id: YOUR_INSTANCE_TYPE_ID # key_pair_id: YOUR_KEY_PAIR_ID # opened_port: PORTS_TO_BE_OPENED # 80, 443, 22, 53, 2375, 2377, 7946, 8080, 8300, 8301, 8302, 8400, 8500, 8600, 9090, 9093, 9095, 9100, 9200 # # scaling: # min: 1 # max: 10 #USER DATA - NOVA # - path: /var/lib/micado/occopus/temp_user_data.yaml # content: | # user_data: # auth_data: # type: nova # username: YOUR_USERNAME # password: YOUR_PASSWORD # resource: # type: nova # endpoint: YOUR_ENDPOINT # network_id: YOUR_NETWORK_ID # image_id: UBUNTU_16.04_IMAGE_ID # flavor_name: YOUR_FLAVOUR_TYPE # server_name: WORKER_NODE # key_name: YOUR_KEYNAME # security_groups: [YOUR_SECURITY_GROUP_ID] # scaling: # min: 1 # max: 10 #Scaling policy #- path: /var/lib/micado/alert-generator/scaling_policy.yaml # content: | # services: # service_name1: # scaledown: 20 # scaleup: 60 # service_name2: # scaledown: 20 # scaleup: 80 # consul config - content: | { "server": true, "datacenter": "application", "encrypt": "uohStneoKEoVYZIASGp6Nw==", "log_level": "INFO", "enable_syslog": false, "services": [{"name":"prometheus"}, {"name":"alertmanager"}, {"name":"prometheus_executor"}, {"name":"consul"}] } path: /etc/consul/config.json # Prometheus config - path: /etc/prometheus/prometheus.yml content: | rule_files: - '*.rules' scrape_configs: - job_name: cluster_monitoring scrape_interval: 10s consul_sd_configs: - server: '172.31.0.5:8500' datacenter: application services: ['lb_cluster', 'worker_cluster', 'app_docker_cluster'] relabel_configs: - source_labels: ['__meta_consul_service'] regex: '(.*)' target_label: 'job' replacement: '$1' - source_labels: ['__meta_consul_service'] regex: '(.*)' target_label: 'group' replacement: '$1' alerting: alertmanagers: - scheme: http static_configs: - targets: - "172.31.0.3:9093" # Prometheus rules (expressions and alerts) - path: /etc/prometheus/prometheus.rules content: | worker_cpu_utilization = 100 - (avg (rate(node_cpu{group="worker_cluster",mode="idle"}[60s])) * 100) worker_ram_utilization = (sum(node_memory_MemFree{job="worker_cluster"}) / sum(node_memory_MemTotal{job="worker_cluster"})) * 100 worker_hdd_utilization = sum(node_filesystem_free{job="worker_cluster",mountpoint="/", device="rootfs"}) / sum(node_filesystem_size{job="worker_cluster",mountpoint="/", device="rootfs"}) *100 ALERT worker_overloaded IF worker_cpu_utilization > 60 FOR 1m LABELS {alert="overloaded", cluster="worker_cluster", node="worker", infra_id="micado_worker_infra", type="VM"} ANNOTATIONS { summary = "Application cluster overloaded"} ALERT worker_underloaded IF worker_cpu_utilization < 20 FOR 2m LABELS {alert="underloaded", cluster="worker_cluster", node="worker", infra_id="micado_worker_infra", type="VM"} ANNOTATIONS { summary = "Application cluster underloaded"} # alertmanager - path: /etc/alertmanager/config.yml content: | global: # The root route on which each incoming alert enters. # The root route with all parameters, which are inherited by the child # routes if they are not overwritten. route: receiver: 'default' group_wait: 10s group_interval: 20s repeat_interval: 5m group_by: [alertname] routes: - receiver: 'default' match: type: docker group_wait: 10s group_interval: 20s repeat_interval: 1m group_by: [alertname] receivers: - name: 'default' webhook_configs: - url: http://172.31.0.4:9095 - content: | #!/bin/bash echo "Setup NETWORK starts." myhost=`hostname` ipaddress=`ifconfig | awk '/inet addr/{print substr($2,6)}' | grep -v 127.0.0.1 | head -n 1` cp /etc/hosts /etc/hosts.old grep -v "$myhost" /etc/hosts.old > /etc/hosts echo "IPADDRESS: $ipaddress" echo "$ipaddress $myhost" >> /etc/hosts rm -rf /etc/resolvconf/* echo "Setup NETWORK finished." path: /bin/consul-set-network.sh permissions: '755' - path: /etc/resolvconf/resolv.conf.d/base content: | nameserver 8.8.8.8 - path: /etc/micado/docker-compose.yml content: | version: '3.3' services: submitter: image: micado/micado-submitter:1.1 container_name: micado-submitter volumes: - "/var/run/docker.sock:/var/run/docker.sock" - "/etc/micado/occopus/:/etc/micado/occopus/" - "/var/lib/micado/occopus/:/var/lib/micado/occopus/" environment: #Input template files TEMP_AUTH_DATA_FILE: /var/lib/micado/occopus/temp_auth_data.yaml TEMP_NODE_DEF_FILE: /var/lib/micado/occopus/temp_node_definitions.yaml TEMP_INFRA_DEF_FILE: /var/lib/micado/occopus/temp_infrastructure_descriptor.yaml #Input user data files USER_DATA_FILE: /var/lib/micado/occopus/temp_user_data.yaml #Output occopus descriptors AUTH_DATA_FILE: /etc/micado/occopus/auth_data.yaml NODE_DEF_FILE: /etc/micado/occopus/nodes/node_definitions.yaml INFRA_DEF_FILE: /etc/micado/occopus/infrastructure_descriptor.yaml MASTER_IP: $IP WORKER_INFRA_NAME: micado_worker_infra command: python /app/submitter/submitter.py redis: image: redis container_name: occopus_redis volumes: - "/var/lib/micado/occopus/redis:/data/" command: redis-server --appendonly yes occopus: depends_on: - redis - submitter image: micado/occopus:1.5 container_name: occopus links: - redis ports: - 5000:5000 volumes: - "/etc/micado/occopus/:/etc/micado/occopus/" environment: - REDIS_NAME=redis command: occopus-rest-service --auth_data_path /etc/micado/occopus/auth_data.yaml --host "$HOST_IP" runcmd: - adduser --disabled-password --gecos "" prometheus - /bin/consul-set-network.sh - sudo dhclient - oldhostname=$(hostname -s) - new_host_name=master-$(date +%s | sha256sum | base64 | head -c 32 ; echo) - echo $new_host_name > /etc/hostname - hostname -F /etc/hostname - line=127.0.1.1'\t'$new_host_name - sed -i "s/$oldhostname/$new_host_name/g" /etc/hosts - echo $line >> /etc/hosts - export DEBIAN_FRONTEND=noninteractive - dpkg-reconfigure openssh-server - resolvconf -u - echo nameserver 8.8.8.8 >> /etc/resolv.conf # Download config files - export GITHUB_URL=https://raw.githubusercontent.com/micado-scale/micado/0.3.1 - curl -L $GITHUB_URL/configs/executor_config.sh --create-dirs -o /etc/prometheus_executor/conf.sh - curl -L $GITHUB_URL/configs/consul_checks.json --create-dirs -o /etc/consul/checks.json # Change health check ip address for host ip - sed -i 's/healthcheck_ip_change/'$(hostname -I | cut -d\ -f1)'/g' /etc/consul/* # Docker install - apt-get update - apt-get install -y --no-install-recommends apt-transport-https ca-certificates curl software-properties-common wget unzip jq dnsmasq - curl -fsSL https://download.docker.com/linux/ubuntu/gpg | apt-key add - - add-apt-repository "deb [arch=amd64] https://download.docker.com/linux/ubuntu $(lsb_release -cs) stable" - apt-get update - apt-get install -y docker-ce=17.09.1~ce-0~ubuntu - export IP=$(hostname -I | cut -d\ -f1) - sed -i -e "s/-H fd:\/\//-H fd:\/\/ -H tcp:\/\/$IP:2375/g" /lib/systemd/system/docker.service - systemctl daemon-reload - service docker restart # Install Docker Compose - curl -L https://github.com/docker/compose/releases/download/1.16.1/docker-compose-`uname -s`-`uname -m` -o /usr/local/bin/docker-compose - chmod +x /usr/local/bin/docker-compose # Start Swarm - docker swarm init --advertise-addr=$IP - docker node update --availability drain $(hostname) # Update executor IP - export IP=$(hostname -I | cut -d\ -f1) - sed -i -e 's/hostIP/'$IP'/g' /etc/prometheus_executor/conf.sh # Start infra. services - curl -L $GITHUB_URL/worker_node/templates/temp_auth_data.yaml --create-dirs -o /var/lib/micado/occopus/temp_auth_data.yaml - curl -L $GITHUB_URL/worker_node/templates/temp_node_definitions.yaml --create-dirs -o /var/lib/micado/occopus/temp_node_definitions.yaml - curl -L $GITHUB_URL/worker_node/templates/temp_infrastructure_descriptor.yaml --create-dirs -o /var/lib/micado/occopus/temp_infrastructure_descriptor.yaml - curl -L $GITHUB_URL/worker_node/cloud_init_worker.yaml --create-dirs -o /etc/micado/occopus/nodes/cloud_init_worker.yaml - docker-compose -f /etc/micado/docker-compose.yml up -d - chmod 777 /etc/prometheus_executor/conf.sh - docker network create -d bridge my-net --subnet 172.31.0.0/24 - docker run -d --network=my-net --ip="172.31.0.2" -p 9090:9090 -v /etc/:/etc prom/prometheus:v1.8.2 - docker run -d --network=my-net --ip="172.31.0.3" -v /etc/alertmanager/:/etc/alertmanager/ -p 9093:9093 prom/alertmanager:v0.12.0 - docker run -d --network=my-net --ip="172.31.0.4" -p 9095:9095 -v /etc/prometheus_executor/:/etc/prometheus_executor micado/prometheus_executor:3.0 - export IP=$(hostname -I | cut -d\ -f1) - docker run -d --network=my-net --ip="172.31.0.5" -p 8301:8301 -p 8301:8301/udp -p 8300:8300 -p 8302:8302 -p 8302:8302/udp -p 8400:8400 -p 8500:8500 -p 8600:8600/udp -v /etc/consul/:/etc/consul -e 'CONSUL_LOCAL_CONFIG={"skip_leave_on_interrupt":true}' consul:1.0.0 agent -server -client=0.0.0.0 -advertise=$IP -bootstrap=true -ui -config-dir=/etc/consul - docker run -d -v /var/run/docker.sock:/var/run/docker.sock -v /etc/prometheus/:/etc/prometheus -v /var/lib/micado/alert-generator/:/var/lib/micado/alert-generator/ -e CONTAINER_SCALING_FILE=/var/lib/micado/alert-generator/scaling_policy.yaml -e ALERTS_FILE_PATH=/etc/prometheus/ -e AUTO_GENERATE_ALERT=False -e DEFAULT_SCALEUP=90 -e DEFAULT_SCALEDOWN=10 -e PROMETHEUS="$IP" micado/alert-generator:1.2