{ "AWSTemplateFormatVersion": "2010-09-09", "Description": "InfraKit to start air-tasks in AWS", "Parameters": { "InfrakitConfigRoot" : { "Type" : "String", "Description" : "Root URL of the bootscript", "Default" : "https://raw.githubusercontent.com/wongwill86/examples/air-tasks/latest/swarm/", "MinLength": 1 }, "TagFieldUser": { "Description": "IAM username, also used as web auth Username", "Type": "String", "MinLength": 1 }, "TagFieldProject": { "Description": "i.e. what project and which run", "Type": "String", "MinLength": 1 }, "WebAuthPassword": { "Description": "Leave blank for no web authentication to portals", "Type": "String", "NoEcho": "true" }, "PreinstalledDockerAMI": { "Description": "Pick which AMI to use; yes for preinstalled Docker; no to install the latest from get.docker.com", "Type": "String", "AllowedValues" : [ "yes","no" ], "Default": "no" }, "ClusterSizeManager": { "Description": "Size of the cluster (1, 3, 5)", "Type": "Number", "AllowedValues" : [ 1, 3, 5], "Default": 1 }, "ClusterSizeWorker": { "Description": "Size of the cluster", "Type": "Number", "MinValue": "1", "MaxValue": "10" }, "KeyName": { "Description": "Name of an existing EC2 KeyPair to enable SSH access to the instances", "Type": "AWS::EC2::KeyPair::KeyName", "MinLength": "1", "ConstraintDescription": "must be the name of an existing EC2 KeyPair.", "Default": "willmvw" }, "InstanceTypeManager" : { "Type" : "String", "Description" : "Manager EC2 HVM instance type (t2.large, t2.xlarge, t2.2xlarge ...).", "AllowedValues" : [ "t2.large","t2.xlarge","t2.2xlarge" ], "ConstraintDescription" : "Must be a valid EC2 HVM instance type.", "Default" : "t2.large" }, "InstanceTypeWorker" : { "Type" : "String", "Description" : "Worker EC2 HVM instance type (t2.micro, m3.medium, etc).", "AllowedValues" : [ "t2.micro","t2.small","t2.medium", "p2.xlarge" ], "ConstraintDescription" : "Must be a valid EC2 HVM instance type.", "Default" : "t2.micro" }, "MetadataExportTemplate" : { "Type" : "String", "Description" : "URL to configure the metadata plugin", "Default" : "https://raw.githubusercontent.com/wongwill86/examples/air-tasks/latest/metadata/aws/export.ikt", "MinLength": 1 }, "Infrakit" : { "Type" : "String", "Description" : "Docker image of Infrakit", "Default" : "infrakit/devbundle:master-2662", "MinLength": 1 }, "DockerComposeLocation" : { "Type" : "String", "Description" : "Location of the docker compose file to deploy to swarm", "Default" : "https://raw.githubusercontent.com/wongwill86/air-tasks/master/deploy/docker-compose-CeleryExecutor.yml", "MinLength": 1 }, "ManagerIP0" : { "Description": "IP for manager node 0", "Type" : "String", "Default" : "172.31.255.250", "MinLength": 1 }, "ManagerIP1" : { "Description": "IP for manager node 1 (if available)", "Type" : "String", "Default" : "172.31.255.251", "MinLength": 1 }, "ManagerIP2" : { "Description": "IP for manager node 2 (if available)", "Type" : "String", "Default" : "172.31.255.252", "MinLength": 1 }, "ManagerIP3" : { "Description": "IP for manager node 3 (if available)", "Type" : "String", "Default" : "172.31.255.253", "MinLength": 1 }, "ManagerIP4" : { "Description": "IP for manager node 4 (if available)", "Type" : "String", "Default" : "172.31.255.254", "MinLength": 1 } }, "Metadata" : { "AWS::CloudFormation::Interface" : { "ParameterGroups" : [ { "Label" : { "default":"Cluster Properties" }, "Parameters" : [ "TagFieldUser", "WebAuthPassword", "TagFieldProject", "ClusterSizeWorker", "InstanceTypeWorker", "ClusterSizeManager", "InstanceTypeManager", "DockerComposeLocation", "KeyName" ] } ], "ParameterLabels" : { "InstanceTypeManager" : { "default" : "Manager instance type?" }, "InstanceTypeWorker" : { "default" : "Worker instance type?" }, "KeyName" : { "default" : "Which SSH key to use?" }, "ClusterSizeManager" : { "default" : "How many managers in the cluster?" }, "ClusterSizeWorker" : { "default" : "How many initial worker instances?" }, "TagFieldUser" : { "default" : "Who are you?" }, "TagFieldProject" : { "default" : "What project is this?" } } } }, "Mappings": { "VpcCidrs" : { "vpc" : { "cidr": "172.31.0.0/16" }, "subnet1" : { "cidr": "172.31.0.0/16" } }, "AWSInstanceType2Arch": { "t2.large": { "Arch": "HVM64" }, "t2.xlarge": { "Arch": "HVM64" }, "t2.2xlarge": { "Arch": "HVM64" } }, "AWSUbuntu16NoDocker": { "us-east-1": { "HVM64": "ami-7765a20d", "HVMG2": "NOT_SUPPORTED" }, "us-east-2": { "HVM64": "ami-10547475", "HVMG2": "NOT_SUPPORTED" }, "us-west-1": { "HVM64": "ami-16efb076", "HVMG2": "NOT_SUPPORTED" }, "us-west-2": { "HVM64": "ami-a58d0dc5", "HVMG2": "NOT_SUPPORTED" } }, "AWSRegionArch2AMI": { "eu-central-1": { "HVM64": "ami-aab96cc5", "HVMG2": "NOT_SUPPORTED" }, "us-east-1": { "HVM64": "ami-d8b569ce", "HVMG2": "NOT_SUPPORTED" }, "us-west-1": { "HVM64": "ami-71411f11", "HVMG2": "NOT_SUPPORTED" }, "us-west-2": { "HVM64": "ami-c877f4a8", "HVMG2": "NOT_SUPPORTED" } }, "AMISelector" : { "no" : { "hasDocker" : "no", "ami" : "AWSUbuntu16NoDocker", "userdata" : "curl -fsSL https://download.docker.com/linux/ubuntu/gpg | apt-key add - &&\napt-key fingerprint 0EBFCD88 &&\nadd-apt-repository \"deb [arch=amd64] https://download.docker.com/linux/ubuntu $(lsb_release -cs) stable\" &&\napt-get update &&\napt-get install docker-ce -y\n" }, "yes" : { "hasDocker" : "yes", "ami" : "AWSRegionArch2AMI", "userdata" : "# Docker pre-installed \n" } } }, "Conditions" : { "Over1Managers" : { "Fn::Or" : [ { "Fn::Equals": [ "2", {"Ref":"ClusterSizeManager"} ] }, { "Fn::Equals": [ "3", {"Ref":"ClusterSizeManager"} ] }, { "Fn::Equals": [ "4", {"Ref":"ClusterSizeManager"} ] }, { "Fn::Equals": [ "5", {"Ref":"ClusterSizeManager"} ] } ] }, "Over2Managers" : { "Fn::Or" : [ { "Fn::Equals": [ "3", {"Ref":"ClusterSizeManager"} ] }, { "Fn::Equals": [ "4", {"Ref":"ClusterSizeManager"} ] }, { "Fn::Equals": [ "5", {"Ref":"ClusterSizeManager"} ] } ] }, "Over3Managers" : { "Fn::Or" : [ { "Fn::Equals": [ "4", {"Ref":"ClusterSizeManager"} ] }, { "Fn::Equals": [ "5", {"Ref":"ClusterSizeManager"} ] } ] }, "Over4Managers" : { "Fn::Equals": [ "5", {"Ref":"ClusterSizeManager"} ] } }, "Resources": { "Vpc" : { "Type" : "AWS::EC2::VPC", "Properties" : { "CidrBlock" : { "Fn::FindInMap" : [ "VpcCidrs", "vpc", "cidr" ] }, "EnableDnsSupport" : "true", "EnableDnsHostnames" : "true", "Tags": [ { "Key" : "Name", "Value" : { "Fn::Join": [ "-", [ { "Ref": "AWS::StackName"}, "VPC"] ] } } ] } }, "Subnet1" : { "DependsOn" : "Vpc", "Type" : "AWS::EC2::Subnet", "Properties" : { "VpcId" : { "Ref" : "Vpc" }, "CidrBlock" : { "Fn::FindInMap" : [ "VpcCidrs", "subnet1", "cidr" ] }, "AvailabilityZone" : { "Fn::Select" : [ "0", { "Fn::GetAZs" : { "Ref" : "AWS::Region" } } ] }, "Tags": [ { "Key" : "Name", "Value" : { "Fn::Join": [ "-", [ { "Ref": "AWS::StackName"}, "Subnet1"] ] } } ] } }, "InternetGateway" : { "DependsOn" : "Vpc", "Type" : "AWS::EC2::InternetGateway", "Properties" : { "Tags": [ { "Key" : "Name", "Value" : { "Fn::Join": [ "-", [ { "Ref": "AWS::StackName"}, "IGW"] ] } } ] } }, "AttachGateway" : { "DependsOn" : ["Vpc", "InternetGateway"], "Type" : "AWS::EC2::VPCGatewayAttachment", "Properties" : { "VpcId" : { "Ref" : "Vpc" }, "InternetGatewayId" : { "Ref" : "InternetGateway" } } }, "RouteViaIgw" : { "DependsOn" : "Vpc", "Type" : "AWS::EC2::RouteTable", "Properties" : { "VpcId" : { "Ref" : "Vpc"}, "Tags": [ { "Key" : "Name", "Value" : { "Fn::Join": [ "-", [ { "Ref": "AWS::StackName"}, "RT"] ] } } ] } }, "PublicRouteViaIgw" : { "DependsOn": ["AttachGateway", "RouteViaIgw"], "Type" : "AWS::EC2::Route", "Properties" : { "RouteTableId" : { "Ref" : "RouteViaIgw" }, "DestinationCidrBlock" : "0.0.0.0/0", "GatewayId" : { "Ref" : "InternetGateway" } } }, "Subnet1RouteTableAssociation" : { "DependsOn": ["Subnet1", "RouteViaIgw"], "Type" : "AWS::EC2::SubnetRouteTableAssociation", "Properties" : { "SubnetId" : { "Ref" : "Subnet1" }, "RouteTableId" : { "Ref" : "RouteViaIgw" } } }, "SecurityGroup": { "DependsOn": "InternetGateway", "Type": "AWS::EC2::SecurityGroup", "Properties": { "GroupDescription": "VPC-wide security group", "VpcId": { "Ref": "Vpc" }, "SecurityGroupIngress": [ { "IpProtocol": "-1", "FromPort": "0", "ToPort": "65535", "CidrIp": { "Fn::FindInMap" : [ "VpcCidrs", "vpc", "cidr" ] } }, { "IpProtocol": "tcp", "FromPort": "22", "ToPort": "22", "CidrIp": "0.0.0.0/0" }, { "IpProtocol": "tcp", "FromPort": "443", "ToPort": "443", "CidrIp": "0.0.0.0/0" }, { "IpProtocol": "tcp", "FromPort": "80", "ToPort": "80", "CidrIp": "0.0.0.0/0" }, { "IpProtocol": "tcp", "FromPort": "24864", "ToPort": "24864", "CidrIp": "0.0.0.0/0" } ] } }, "ProvisionerRole": { "Type": "AWS::IAM::Role", "Properties": { "AssumeRolePolicyDocument": { "Version" : "2012-10-17", "Statement": [ { "Effect": "Allow", "Principal": { "Service": [ "ec2.amazonaws.com" ] }, "Action": [ "sts:AssumeRole" ] } ] }, "Path": "/" } }, "ProvisionerPolicies": { "Type": "AWS::IAM::Policy", "Properties": { "PolicyName": "managers-policy", "PolicyDocument": { "Version" : "2012-10-17", "Statement": [ { "Effect": "Allow", "Action": [ "cloudformation:DescribeStacks", "cloudformation:DescribeStackEvents", "cloudformation:DescribeStackResource", "cloudformation:DescribeStackResources", "cloudformation:GetTemplate", "cloudformation:List*", "ec2:*", "elasticloadbalancing:*", "cloudwatch:*", "autoscaling:*", "sqs:*", "s3:*", "ecr:GetAuthorizationToken", "ecr:BatchCheckLayerAvailability", "ecr:GetDownloadUrlForLayer", "ecr:GetRepositoryPolicy", "ecr:DescribeRepositories", "ecr:ListImages", "ecr:DescribeImages", "ecr:BatchGetImage", "iam:GetInstanceProfile", "iam:GetRole", "iam:GetRolePolicy", "iam:ListAttachedRolePolicies", "iam:ListRolePolicies", "iam:ListInstanceProfiles", "iam:PassRole", "dynamodb:BatchGetItem", "dynamodb:DescribeTable", "dynamodb:GetItem", "dynamodb:Query", "dynamodb:Scan", "dynamodb:UpdateTable" ], "Resource": "*" } ] }, "Roles": [ { "Ref": "ProvisionerRole" } ] } }, "InstanceProfile": { "Type": "AWS::IAM::InstanceProfile", "Properties": { "Path": "/", "Roles": [ { "Ref": "ProvisionerRole" } ] } }, "Volume0" : { "Type":"AWS::EC2::Volume", "Properties" : { "AvailabilityZone" : { "Fn::GetAtt" : [ "Subnet1", "AvailabilityZone" ] }, "Size" : 4, "Tags" : [ { "Key" : "Name", "Value" : { "Fn::Join": [ "-", [ { "Ref": "AWS::StackName"}, "EBS" ] ] } }, { "Key" : "infrakit.scope", "Value" : {"Ref": "AWS::StackName"} }, { "Key" : "docker-infrakit-volume", "Value" : { "Ref" : "ManagerIP0" } } ] } }, "Volume1" : { "Type":"AWS::EC2::Volume", "Condition": "Over1Managers", "Properties" : { "AvailabilityZone" : { "Fn::GetAtt" : [ "Subnet1", "AvailabilityZone" ] }, "Size" : 4, "Tags" : [ { "Key" : "Name", "Value" : { "Fn::Join": [ "-", [ { "Ref": "AWS::StackName"}, "EBS" ] ] } }, { "Key" : "infrakit.scope", "Value" : {"Ref": "AWS::StackName"} }, { "Key" : "docker-infrakit-volume", "Value" : { "Ref" : "ManagerIP1" } } ] } }, "Volume2" : { "Type":"AWS::EC2::Volume", "Condition": "Over2Managers", "Properties" : { "AvailabilityZone" : { "Fn::GetAtt" : [ "Subnet1", "AvailabilityZone" ] }, "Size" : 4, "Tags" : [ { "Key" : "Name", "Value" : { "Fn::Join": [ "-", [ { "Ref": "AWS::StackName"}, "EBS" ] ] } }, { "Key" : "infrakit.scope", "Value" : {"Ref": "AWS::StackName"} }, { "Key" : "docker-infrakit-volume", "Value" : { "Ref" : "ManagerIP2" } } ] } }, "Volume3" : { "Type":"AWS::EC2::Volume", "Condition": "Over3Managers", "Properties" : { "AvailabilityZone" : { "Fn::GetAtt" : [ "Subnet1", "AvailabilityZone" ] }, "Size" : 4, "Tags" : [ { "Key" : "Name", "Value" : { "Fn::Join": [ "-", [ { "Ref": "AWS::StackName"}, "EBS" ] ] } }, { "Key" : "infrakit.scope", "Value" : {"Ref": "AWS::StackName"} }, { "Key" : "docker-infrakit-volume", "Value" : { "Ref" : "ManagerIP3" } } ] } }, "Volume4" : { "Type":"AWS::EC2::Volume", "Condition": "Over4Managers", "Properties" : { "AvailabilityZone" : { "Fn::GetAtt" : [ "Subnet1", "AvailabilityZone" ] }, "Size" : 4, "Tags" : [ { "Key" : "Name", "Value" : { "Fn::Join": [ "-", [ { "Ref": "AWS::StackName"}, "EBS" ] ] } }, { "Key" : "infrakit.scope", "Value" : {"Ref": "AWS::StackName"} }, { "Key" : "docker-infrakit-volume", "Value" : { "Ref" : "ManagerIP4" } } ] } }, "BootstrapInstanceVolumeAttachment" : { "Type":"AWS::EC2::VolumeAttachment", "Properties" : { "Device" : "/dev/xvdf", "InstanceId" : {"Ref": "BootstrapInstance"}, "VolumeId" : {"Ref": "Volume0" } } }, "BootstrapInstance" : { "DependsOn": "Subnet1", "Type" : "AWS::EC2::Instance", "Properties" : { "KeyName" : { "Ref" : "KeyName" }, "InstanceType" : { "Ref" : "InstanceTypeManager" }, "IamInstanceProfile" : { "Ref" : "InstanceProfile" }, "NetworkInterfaces" : [ { "DeviceIndex" : "0", "Description" : "main interface", "DeleteOnTermination" : true, "AssociatePublicIpAddress" : true, "PrivateIpAddress" : { "Ref" : "ManagerIP0" }, "SubnetId" : { "Ref" : "Subnet1" }, "GroupSet" : [ {"Ref" : "SecurityGroup"} ] } ], "AvailabilityZone" : { "Fn::GetAtt" : [ "Subnet1", "AvailabilityZone" ] }, "ImageId" : { "Fn::FindInMap": [ { "Fn::FindInMap" : [ "AMISelector", { "Ref" : "PreinstalledDockerAMI" }, "ami" ] }, { "Ref": "AWS::Region" }, { "Fn::FindInMap": ["AWSInstanceType2Arch", {"Ref" : "InstanceTypeManager"}, "Arch"]} ] }, "UserData" : { "Fn::Base64" : { "Fn::Join" : [ "", [ "#!/bin/sh \nset -e\n", { "Fn::FindInMap": [ "AMISelector", { "Ref" : "PreinstalledDockerAMI" }, "userdata" ] }, "export INFRAKIT_GROUPS_URL=", { "Ref": "InfrakitConfigRoot" }, "\n", "export INFRAKIT_IMAGE=", {"Ref":"Infrakit"}, "\n", "echo INFRAKIT_GROUPS_URL=", { "Ref": "InfrakitConfigRoot" }, " >> /etc/environment \n", "echo INFRAKIT_IMAGE=", {"Ref":"Infrakit"}, " >> /etc/environment \n", "mkdir -p /infrakit\n", "docker swarm init\n", "docker run --rm -v /var/run/docker.sock:/var/run/docker.sock -v /infrakit:/infrakit ", {"Ref":"Infrakit"}, " infrakit util init --group-id managers --start combo --start swarm ", " --var /cluster/provider=aws", " --var /cluster/tag/user=", {"Ref": "TagFieldUser"}, " --var /cluster/tag/project=", {"Ref": "TagFieldProject"}, " --var /cluster/name=", {"Ref":"AWS::StackName"}, " --var /cluster/swarm/manager/ips/0=", { "Ref" : "ManagerIP0" }, " --var /cluster/swarm/manager/ips/1=", { "Ref" : "ManagerIP1" }, " --var /cluster/swarm/manager/ips/2=", { "Ref" : "ManagerIP2" }, " --var /cluster/swarm/manager/ips/3=", { "Ref" : "ManagerIP3" }, " --var /cluster/swarm/manager/ips/4=", { "Ref" : "ManagerIP4" }, " --var /cluster/swarm/deploy/compose=", {"Ref":"DockerComposeLocation"}, " --var /cluster/size/manager=", {"Ref":"ClusterSizeManager"}, " --var /cluster/size/worker=", {"Ref":"ClusterSizeWorker"}, " --var /cluster/instanceType/manager=", {"Ref":"InstanceTypeManager"}, " --var /cluster/instanceType/worker=", {"Ref":"InstanceTypeWorker"}, " --var /infrakit/config/root=", {"Ref":"InfrakitConfigRoot"}, " --var /infrakit/metadata/configURL=", {"Ref":"MetadataExportTemplate"}, " --var /provider/image/hasDocker=", { "Fn::FindInMap": ["AMISelector", { "Ref":"PreinstalledDockerAMI" }, "hasDocker"] }, " --var /infrakit/docker/image=", {"Ref":"Infrakit"}, " ", {"Ref":"InfrakitConfigRoot"}, "/groups.json", " | tee /var/lib/infrakit.boot | sh \n", "echo '", { "Ref" : "TagFieldUser" }, "' | docker secret create basic_auth_username - \n", "echo '", { "Ref" : "WebAuthPassword" }, "' | docker secret create basic_auth_password - \n", "echo '' | docker secret create ssl_certificate - \n", "echo '' | docker secret create ssl_certificate_key - \n", "while [ $(docker node ls -f 'role=manager' -q | wc -l ) -lt ", { "Ref": "ClusterSizeManager" }, " ]; do\n", "echo 'Waiting for managers to be online before deploy'\n", "sleep 10;\n", "done\n", "wget -O compose.yml ", { "Ref": "DockerComposeLocation" }, "\n", "docker stack deploy -c compose.yml ", { "Ref": "AWS::StackName" } ]] } }, "Tags" : [ { "Key" : "Name", "Value" : { "Fn::Join": [ "-", [ { "Ref": "AWS::StackName"}, "Bootstrap" ] ] } }, { "Key" : "infrakit.scope", "Value" : {"Ref":"AWS::StackName"} }, { "Key" : "infrakit.group", "Value" : "managers" }, { "Key" : "infrakit.config_sha", "Value" : "bootstrap" }, { "Key" : "infrakit.role", "Value" : "managers" }, { "Key" : "User", "Value" : { "Ref": "TagFieldUser" } }, { "Key" : "Project", "Value" : { "Ref": "TagFieldProject" } } ] } } }, "Outputs" : { "BootNodePublicIP" : { "Description" : "The public IP of the boot node", "Value" : { "Fn::GetAtt" : [ "BootstrapInstance", "PublicIp" ] } } } }