{ "cells": [ { "cell_type": "markdown", "metadata": {}, "source": [ "### End to End demo of starting spot instance in AWS" ] }, { "cell_type": "code", "execution_count": 1, "metadata": { "collapsed": true }, "outputs": [], "source": [ "%reload_ext autoreload\n", "%autoreload 2\n", "%matplotlib inline" ] }, { "cell_type": "code", "execution_count": 3, "metadata": {}, "outputs": [], "source": [ "from aws_setup import *" ] }, { "cell_type": "code", "execution_count": 4, "metadata": { "collapsed": true }, "outputs": [], "source": [ "name='fast-ai'" ] }, { "cell_type": "code", "execution_count": 5, "metadata": { "collapsed": true }, "outputs": [], "source": [ "vpc = create_vpc(name)" ] }, { "cell_type": "code", "execution_count": 9, "metadata": {}, "outputs": [ { "data": { "text/plain": [ "[ec2.Vpc(id='vpc-ee793297')]" ] }, "execution_count": 9, "metadata": {}, "output_type": "execute_result" } ], "source": [ "list(ec2.vpcs.filter(Filters=[{'Name':'tag:Name', 'Values':[name]}]))" ] }, { "cell_type": "code", "execution_count": 11, "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "Created keypair\n" ] } ], "source": [ "create_ec2_keypair(name)" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "#### Request Spot instance" ] }, { "cell_type": "code", "execution_count": 25, "metadata": { "collapsed": true }, "outputs": [], "source": [ "instance_name = f'{name}-instance'\n", "instance_type = 'c5.xlarge'" ] }, { "cell_type": "code", "execution_count": 26, "metadata": {}, "outputs": [ { "data": { "text/plain": [ "'0.062500'" ] }, "execution_count": 26, "metadata": {}, "output_type": "execute_result" } ], "source": [ "spot_prices = get_spot_prices(); spot_prices[instance_type]" ] }, { "cell_type": "code", "execution_count": 21, "metadata": { "collapsed": true }, "outputs": [ { "data": { "text/plain": [ "[('m3.medium', '0.006700'),\n", " ('m1.medium', '0.008700'),\n", " ('t2.medium', '0.013900'),\n", " ('c3.large', '0.027300'),\n", " ('c4.large', '0.028700'),\n", " ('r3.large', '0.030800'),\n", " ('m2.2xlarge', '0.049000'),\n", " ('c1.xlarge', '0.052000'),\n", " ('t2.xlarge', '0.055700'),\n", " ('t2.large', '0.055800'),\n", " ('m3.xlarge', '0.057400'),\n", " ('c4.xlarge', '0.059300'),\n", " ('c3.xlarge', '0.059800'),\n", " ('m4.xlarge', '0.060200'),\n", " ('c5.xlarge', '0.062500'),\n", " ('r3.xlarge', '0.069400'),\n", " ('r4.xlarge', '0.085600'),\n", " ('m2.4xlarge', '0.098000'),\n", " ('i3.xlarge', '0.102600'),\n", " ('c3.2xlarge', '0.113000'),\n", " ('c5.large', '0.122100'),\n", " ('r4.large', '0.123500'),\n", " ('m5.large', '0.123600'),\n", " ('m4.2xlarge', '0.123600'),\n", " ('r3.2xlarge', '0.133300'),\n", " ('c5.2xlarge', '0.140600'),\n", " ('m5.2xlarge', '0.147600'),\n", " ('r4.2xlarge', '0.154700'),\n", " ('c4.2xlarge', '0.160700'),\n", " ('h1.2xlarge', '0.165000'),\n", " ('m1.large', '0.175000'),\n", " ('i3.2xlarge', '0.191800'),\n", " ('m1.xlarge', '0.219000'),\n", " ('c3.4xlarge', '0.235000'),\n", " ('c4.4xlarge', '0.241100'),\n", " ('g2.2xlarge', '0.250800'),\n", " ('m4.4xlarge', '0.288400'),\n", " ('r3.4xlarge', '0.299000'),\n", " ('c5.4xlarge', '0.304000'),\n", " ('h1.4xlarge', '0.330000'),\n", " ('r4.4xlarge', '0.354000'),\n", " ('i3.4xlarge', '0.377100'),\n", " ('p2.xlarge', '0.454000'),\n", " ('m3.2xlarge', '0.482600'),\n", " ('c3.8xlarge', '0.491400'),\n", " ('x1e.2xlarge', '0.500400'),\n", " ('c4.8xlarge', '0.506300'),\n", " ('i2.2xlarge', '0.511500'),\n", " ('r3.8xlarge', '0.521300'),\n", " ('m4.10xlarge', '0.601800'),\n", " ('f1.2xlarge', '0.628000'),\n", " ('d2.2xlarge', '0.635000'),\n", " ('g3.8xlarge', '0.699500'),\n", " ('r4.8xlarge', '0.700400'),\n", " ('i3.8xlarge', '0.797700'),\n", " ('p3.2xlarge', '0.933500'),\n", " ('x1e.4xlarge', '1.000800'),\n", " ('g3.4xlarge', '1.078000'),\n", " ('r4.16xlarge', '1.095900'),\n", " ('c5.18xlarge', '1.150100'),\n", " ('i2.4xlarge', '1.231300'),\n", " ('cc2.8xlarge', '1.261500'),\n", " ('h1.16xlarge', '1.320000'),\n", " ('i3.16xlarge', '1.497600'),\n", " ('m5.24xlarge', '1.516500'),\n", " ('g2.8xlarge', '1.671100'),\n", " ('x1e.8xlarge', '2.001600'),\n", " ('i2.8xlarge', '2.046000'),\n", " ('h1.8xlarge', '2.132000'),\n", " ('p2.8xlarge', '2.160000'),\n", " ('c5.9xlarge', '2.197600'),\n", " ('d2.8xlarge', '2.334000'),\n", " ('cr1.8xlarge', '3.500000'),\n", " ('m4.16xlarge', '3.906800'),\n", " ('x1.32xlarge', '4.001400'),\n", " ('p3.16xlarge', '10.288000'),\n", " ('p3.8xlarge', '12.488000'),\n", " ('f1.16xlarge', '13.200000'),\n", " ('x1e.32xlarge', '26.688000')]" ] }, "execution_count": 21, "metadata": {}, "output_type": "execute_result" } ], "source": [ "sorted(spot_prices.items(), key=lambda x: float(x[1]))" ] }, { "cell_type": "code", "execution_count": 24, "metadata": {}, "outputs": [ { "data": { "text/plain": [ "[('c5.xlarge', '0.062500'),\n", " ('c5.large', '0.122100'),\n", " ('c5.2xlarge', '0.140600'),\n", " ('c5.4xlarge', '0.304000'),\n", " ('c5.18xlarge', '1.150100'),\n", " ('c5.9xlarge', '2.197600')]" ] }, "execution_count": 24, "metadata": {}, "output_type": "execute_result" } ], "source": [ "sorted([o for o in spot_prices.items() if o[0].startswith('c5')], key=lambda x: float(x[1]))" ] }, { "cell_type": "code", "execution_count": 27, "metadata": { "collapsed": true }, "outputs": [], "source": [ "launch_specs = LaunchSpecs(vpc, instance_type=instance_type).build()" ] }, { "cell_type": "code", "execution_count": 28, "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "Waiting on spot fullfillment...\n", "Fullfillment completed. InstanceId: i-00e902bf70c696999\n", "Rebooting...\n", "Completed. SSH: ssh -i ~/.ssh/aws-key-fast-ai.pem ubuntu@35.162.88.124\n" ] }, { "data": { "text/plain": [ "ec2.Instance(id='i-00e902bf70c696999')" ] }, "execution_count": 28, "metadata": {}, "output_type": "execute_result" } ], "source": [ "instance = create_spot_instance(instance_name, launch_specs); instance" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "Request on demand instance (if spot error)" ] }, { "cell_type": "code", "execution_count": 68, "metadata": { "collapsed": true }, "outputs": [], "source": [ "# instance = create_instance(instance_name, vpc, instance_type='t2.micro'); instance" ] }, { "cell_type": "code", "execution_count": 29, "metadata": {}, "outputs": [ { "data": { "text/plain": [ "ec2.Instance(id='i-00e902bf70c696999')" ] }, "execution_count": 29, "metadata": {}, "output_type": "execute_result" } ], "source": [ "instance = get_instance(f'{instance_name}'); instance" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "#### Attach EBS volume (Optional)" ] }, { "cell_type": "code", "execution_count": 30, "metadata": { "collapsed": true }, "outputs": [], "source": [ "volume_tag = f'{name}-ebs-volume'" ] }, { "cell_type": "code", "execution_count": 40, "metadata": {}, "outputs": [ { "data": { "text/plain": [ "'us-west-2b'" ] }, "execution_count": 40, "metadata": {}, "output_type": "execute_result" } ], "source": [ "az = list(vpc.network_interfaces.all())[0].availability_zone\n", "az" ] }, { "cell_type": "code", "execution_count": 42, "metadata": {}, "outputs": [ { "data": { "text/plain": [ "'us-west-2b'" ] }, "execution_count": 42, "metadata": {}, "output_type": "execute_result" } ], "source": [ "az" ] }, { "cell_type": "code", "execution_count": 43, "metadata": { "collapsed": true }, "outputs": [ { "ename": "ClientError", "evalue": "An error occurred (InvalidParameterValue) when calling the CreateVolume operation: 'null' is not a valid taggable resource type for this operation.", "output_type": "error", "traceback": [ "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m", "\u001b[0;31mClientError\u001b[0m Traceback (most recent call last)", "\u001b[0;32m\u001b[0m in \u001b[0;36m\u001b[0;34m()\u001b[0m\n\u001b[0;32m----> 1\u001b[0;31m \u001b[0mvolume\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mcreate_volume\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mvolume_tag\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0maz\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0maz\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0msize\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;36m100\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m", "\u001b[0;32m/mnt/c/Users/j/git/imagenet-fast/aws/aws_setup.py\u001b[0m in \u001b[0;36mcreate_volume\u001b[0;34m(name, az, size, volume_type)\u001b[0m\n\u001b[1;32m 238\u001b[0m }]\n\u001b[1;32m 239\u001b[0m volume = ec2.create_volume(Size=size, VolumeType=volume_type, TagSpecifications=tag_specs,\n\u001b[0;32m--> 240\u001b[0;31m AvailabilityZone=az)\n\u001b[0m\u001b[1;32m 241\u001b[0m \u001b[0;32mreturn\u001b[0m \u001b[0mvolume\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 242\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n", "\u001b[0;32m~/anaconda3/lib/python3.6/site-packages/boto3/resources/factory.py\u001b[0m in \u001b[0;36mdo_action\u001b[0;34m(self, *args, **kwargs)\u001b[0m\n\u001b[1;32m 518\u001b[0m \u001b[0;31m# instance via ``self``.\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 519\u001b[0m \u001b[0;32mdef\u001b[0m \u001b[0mdo_action\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m*\u001b[0m\u001b[0margs\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m**\u001b[0m\u001b[0mkwargs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 520\u001b[0;31m \u001b[0mresponse\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0maction\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m*\u001b[0m\u001b[0margs\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m**\u001b[0m\u001b[0mkwargs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 521\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 522\u001b[0m \u001b[0;32mif\u001b[0m \u001b[0mhasattr\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m'load'\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n", "\u001b[0;32m~/anaconda3/lib/python3.6/site-packages/boto3/resources/action.py\u001b[0m in \u001b[0;36m__call__\u001b[0;34m(self, parent, *args, **kwargs)\u001b[0m\n\u001b[1;32m 81\u001b[0m operation_name, params)\n\u001b[1;32m 82\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 83\u001b[0;31m \u001b[0mresponse\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mgetattr\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mparent\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mmeta\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mclient\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0moperation_name\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m**\u001b[0m\u001b[0mparams\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 84\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 85\u001b[0m \u001b[0mlogger\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mdebug\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m'Response: %r'\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mresponse\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n", "\u001b[0;32m~/anaconda3/lib/python3.6/site-packages/botocore/client.py\u001b[0m in \u001b[0;36m_api_call\u001b[0;34m(self, *args, **kwargs)\u001b[0m\n\u001b[1;32m 312\u001b[0m \"%s() only accepts keyword arguments.\" % py_operation_name)\n\u001b[1;32m 313\u001b[0m \u001b[0;31m# The \"self\" in this scope is referring to the BaseClient.\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 314\u001b[0;31m \u001b[0;32mreturn\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_make_api_call\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0moperation_name\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mkwargs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 315\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 316\u001b[0m \u001b[0m_api_call\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m__name__\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mstr\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mpy_operation_name\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n", "\u001b[0;32m~/anaconda3/lib/python3.6/site-packages/botocore/client.py\u001b[0m in \u001b[0;36m_make_api_call\u001b[0;34m(self, operation_name, api_params)\u001b[0m\n\u001b[1;32m 610\u001b[0m \u001b[0merror_code\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mparsed_response\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mget\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m\"Error\"\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m{\u001b[0m\u001b[0;34m}\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mget\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m\"Code\"\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 611\u001b[0m \u001b[0merror_class\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mexceptions\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mfrom_code\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0merror_code\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 612\u001b[0;31m \u001b[0;32mraise\u001b[0m \u001b[0merror_class\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mparsed_response\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0moperation_name\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 613\u001b[0m \u001b[0;32melse\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 614\u001b[0m \u001b[0;32mreturn\u001b[0m \u001b[0mparsed_response\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n", "\u001b[0;31mClientError\u001b[0m: An error occurred (InvalidParameterValue) when calling the CreateVolume operation: 'null' is not a valid taggable resource type for this operation." ] } ], "source": [ "volume = create_volume(volume_tag, az=az, size=100)" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "collapsed": true }, "outputs": [], "source": [ "_ = attach_volume(instance, volume_tag, device='/dev/xvdf')" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "#### Create EFS" ] }, { "cell_type": "code", "execution_count": 45, "metadata": { "collapsed": true }, "outputs": [], "source": [ "efs_tag = f'{name}-efs'" ] }, { "cell_type": "code", "execution_count": 47, "metadata": {}, "outputs": [], "source": [ "efs = create_efs(efs_tag, vpc)" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "### SSH into instance" ] }, { "cell_type": "code", "execution_count": 48, "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "Connecting to SSH...\n", "Connected!\n" ] } ], "source": [ "client = connect_to_instance(instance)" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "#### Mount EBS" ] }, { "cell_type": "code", "execution_count": 49, "metadata": {}, "outputs": [ { "data": { "text/plain": [ "('', '')" ] }, "execution_count": 49, "metadata": {}, "output_type": "execute_result" } ], "source": [ "upload_path = Path.cwd()/'upload_scripts/mount_ebs.sh'\n", "upload_file(client, str(upload_path), 'mount_ebs.sh')" ] }, { "cell_type": "code", "execution_count": 50, "metadata": {}, "outputs": [], "source": [ "out, _ = run_command(client, 'chmod 755 mount_ebs.sh')" ] }, { "cell_type": "code", "execution_count": 51, "metadata": {}, "outputs": [], "source": [ "out, _ = run_command(client, './mount_ebs.sh --device /dev/xvdf') # no reformatting\n", "# out, _ = run_command(client, './mount_ebs.sh --device /dev/xvdf --reformat true')" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "#### Mount EFS" ] }, { "cell_type": "code", "execution_count": 52, "metadata": {}, "outputs": [ { "data": { "text/plain": [ "'fs-080292a1.efs.us-west-2.amazonaws.com'" ] }, "execution_count": 52, "metadata": {}, "output_type": "execute_result" } ], "source": [ "efs_addr = get_efs_address('fast-ai-efs'); efs_addr" ] }, { "cell_type": "code", "execution_count": 53, "metadata": {}, "outputs": [], "source": [ "out, _ = run_command(client, 'mkdir ~/efs_mount')" ] }, { "cell_type": "code", "execution_count": 54, "metadata": { "collapsed": true }, "outputs": [], "source": [ "mount_opts = 'nfsvers=4.1,rsize=1048576,wsize=1048576,hard,timeo=600,retrans=2'\n", "efs_mount_cmd = f'sudo mount -t nfs -o {mount_opts} {efs_addr}:/ ~/efs_mount'" ] }, { "cell_type": "code", "execution_count": 59, "metadata": {}, "outputs": [], "source": [ "out, _ = run_command(client, efs_mount_cmd)" ] }, { "cell_type": "code", "execution_count": 60, "metadata": {}, "outputs": [ { "data": { "text/plain": [ "''" ] }, "execution_count": 60, "metadata": {}, "output_type": "execute_result" } ], "source": [ "out" ] }, { "cell_type": "code", "execution_count": 61, "metadata": { "collapsed": true }, "outputs": [], "source": [ "client.close()" ] }, { "cell_type": "code", "execution_count": 63, "metadata": {}, "outputs": [ { "data": { "text/plain": [ "'ssh -i /home/jhoward/.ssh/aws-key-fast-ai.pem ubuntu@35.162.88.124'" ] }, "execution_count": 63, "metadata": {}, "output_type": "execute_result" } ], "source": [ "keypath=f'{Path.home()}/.ssh/aws-key-fast-ai.pem'\n", "username='ubuntu'\n", "f'ssh -i {keypath} {username}@{instance.public_ip_address}'" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "collapsed": true }, "outputs": [], "source": [] } ], "metadata": { "kernelspec": { "display_name": "Python 3", "language": "python", "name": "python3" }, "language_info": { "codemirror_mode": { "name": "ipython", "version": 3 }, "file_extension": ".py", "mimetype": "text/x-python", "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", "version": "3.6.2" } }, "nbformat": 4, "nbformat_minor": 2 }