{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Make AWS Layer!"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "This notebook automates the (re)build, install, zip, upload to S3, and aws cli command to publish new version to an AWS layer."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "from my_utils.os_utils import subprocess_execute\n",
    "import shutil"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "# list of packages to build\n",
    "packages_todo = [\n",
    "    # list of tuples: ('package_name','from_pip')\n",
    "    ('pandas',True),\n",
    "    ('my_utils',False),\n",
    "    ('my_dev',False),    \n",
    "]\n",
    "package_base_dir = '/home/jovyan/packages'\n",
    "work_dir = '/home/jovyan/work'\n",
    "s3_path = 's3://andrewm4894-python-packages/latest'\n",
    "s3_region = 'us-west-2'\n",
    "s3_bucket = s3_path.split('s3://')[1].split('/')[0]\n",
    "s3_url = f\"https://s3.console.aws.amazon.com/s3/buckets/{s3_path.replace('s3://','')}?region={s3_region}&tab=overview\"\n",
    "aws_layer_name = 'my-aws-python-packages'"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Build & Install Packages"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "... cmd to execute:\n",
      "pip install --upgrade --force-reinstall \"pandas\" -t /home/jovyan/work/python/python/.\n",
      "... result:\n",
      "Collecting pandas\n",
      "  Downloading https://files.pythonhosted.org/packages/73/9b/52e228545d14f14bb2a1622e225f38463c8726645165e1cb7dde95bfe6d4/pandas-0.25.1-cp36-cp36m-manylinux1_x86_64.whl (10.5MB)\n",
      "Collecting pytz>=2017.2 (from pandas)\n",
      "  Downloading https://files.pythonhosted.org/packages/87/76/46d697698a143e05f77bec5a526bf4e56a0be61d63425b68f4ba553b51f2/pytz-2019.2-py2.py3-none-any.whl (508kB)\n",
      "Collecting numpy>=1.13.3 (from pandas)\n",
      "  Downloading https://files.pythonhosted.org/packages/75/92/57179ed45307ec6179e344231c47da7f3f3da9e2eee5c8ab506bd279ce4e/numpy-1.17.1-cp36-cp36m-manylinux1_x86_64.whl (20.4MB)\n",
      "Collecting python-dateutil>=2.6.1 (from pandas)\n",
      "  Downloading https://files.pythonhosted.org/packages/41/17/c62faccbfbd163c7f57f3844689e3a78bae1f403648a6afb1d0866d87fbb/python_dateutil-2.8.0-py2.py3-none-any.whl (226kB)\n",
      "Collecting six>=1.5 (from python-dateutil>=2.6.1->pandas)\n",
      "  Downloading https://files.pythonhosted.org/packages/73/fb/00a976f728d0d1fecfe898238ce23f502a721c0ac0ecfedb80e0d88c64e9/six-1.12.0-py2.py3-none-any.whl\n",
      "Installing collected packages: pytz, numpy, six, python-dateutil, pandas\n",
      "Successfully installed numpy-1.17.1 pandas-0.25.1 python-dateutil-2.8.0 pytz-2019.2 six-1.12.0\n",
      "\n",
      "... cmd to execute:\n",
      "cd /home/jovyan/packages/my_utils && python setup.py bdist_egg && pip install --upgrade \"/home/jovyan/packages/my_utils\" -t /home/jovyan/work/python/python/.\n",
      "... result:\n",
      "running bdist_egg\n",
      "running egg_info\n",
      "writing my_utils.egg-info/PKG-INFO\n",
      "writing dependency_links to my_utils.egg-info/dependency_links.txt\n",
      "writing top-level names to my_utils.egg-info/top_level.txt\n",
      "reading manifest file 'my_utils.egg-info/SOURCES.txt'\n",
      "writing manifest file 'my_utils.egg-info/SOURCES.txt'\n",
      "installing library code to build/bdist.linux-x86_64/egg\n",
      "running install_lib\n",
      "running build_py\n",
      "creating build/bdist.linux-x86_64/egg\n",
      "creating build/bdist.linux-x86_64/egg/my_utils\n",
      "copying build/lib/my_utils/os_utils.py -> build/bdist.linux-x86_64/egg/my_utils\n",
      "copying build/lib/my_utils/__init__.py -> build/bdist.linux-x86_64/egg/my_utils\n",
      "byte-compiling build/bdist.linux-x86_64/egg/my_utils/os_utils.py to os_utils.cpython-36.pyc\n",
      "byte-compiling build/bdist.linux-x86_64/egg/my_utils/__init__.py to __init__.cpython-36.pyc\n",
      "creating build/bdist.linux-x86_64/egg/EGG-INFO\n",
      "copying my_utils.egg-info/PKG-INFO -> build/bdist.linux-x86_64/egg/EGG-INFO\n",
      "copying my_utils.egg-info/SOURCES.txt -> build/bdist.linux-x86_64/egg/EGG-INFO\n",
      "copying my_utils.egg-info/dependency_links.txt -> build/bdist.linux-x86_64/egg/EGG-INFO\n",
      "copying my_utils.egg-info/not-zip-safe -> build/bdist.linux-x86_64/egg/EGG-INFO\n",
      "copying my_utils.egg-info/top_level.txt -> build/bdist.linux-x86_64/egg/EGG-INFO\n",
      "creating 'dist/my_utils-0.1-py3.6.egg' and adding 'build/bdist.linux-x86_64/egg' to it\n",
      "removing 'build/bdist.linux-x86_64/egg' (and everything under it)\n",
      "Processing /home/jovyan/packages/my_utils\n",
      "Installing collected packages: my-utils\n",
      "  Running setup.py install for my-utils: started\n",
      "    Running setup.py install for my-utils: finished with status 'done'\n",
      "Successfully installed my-utils-0.1\n",
      "\n",
      "... cmd to execute:\n",
      "cd /home/jovyan/packages/my_dev && python setup.py bdist_egg && pip install --upgrade \"/home/jovyan/packages/my_dev\" -t /home/jovyan/work/python/python/.\n",
      "... result:\n",
      "running bdist_egg\n",
      "running egg_info\n",
      "writing my_dev.egg-info/PKG-INFO\n",
      "writing dependency_links to my_dev.egg-info/dependency_links.txt\n",
      "writing top-level names to my_dev.egg-info/top_level.txt\n",
      "reading manifest file 'my_dev.egg-info/SOURCES.txt'\n",
      "writing manifest file 'my_dev.egg-info/SOURCES.txt'\n",
      "installing library code to build/bdist.linux-x86_64/egg\n",
      "running install_lib\n",
      "running build_py\n",
      "creating build/bdist.linux-x86_64/egg\n",
      "creating build/bdist.linux-x86_64/egg/my_dev\n",
      "copying build/lib/my_dev/dev.py -> build/bdist.linux-x86_64/egg/my_dev\n",
      "copying build/lib/my_dev/__init__.py -> build/bdist.linux-x86_64/egg/my_dev\n",
      "byte-compiling build/bdist.linux-x86_64/egg/my_dev/dev.py to dev.cpython-36.pyc\n",
      "byte-compiling build/bdist.linux-x86_64/egg/my_dev/__init__.py to __init__.cpython-36.pyc\n",
      "creating build/bdist.linux-x86_64/egg/EGG-INFO\n",
      "copying my_dev.egg-info/PKG-INFO -> build/bdist.linux-x86_64/egg/EGG-INFO\n",
      "copying my_dev.egg-info/SOURCES.txt -> build/bdist.linux-x86_64/egg/EGG-INFO\n",
      "copying my_dev.egg-info/dependency_links.txt -> build/bdist.linux-x86_64/egg/EGG-INFO\n",
      "copying my_dev.egg-info/not-zip-safe -> build/bdist.linux-x86_64/egg/EGG-INFO\n",
      "copying my_dev.egg-info/top_level.txt -> build/bdist.linux-x86_64/egg/EGG-INFO\n",
      "creating 'dist/my_dev-0.1-py3.6.egg' and adding 'build/bdist.linux-x86_64/egg' to it\n",
      "removing 'build/bdist.linux-x86_64/egg' (and everything under it)\n",
      "Processing /home/jovyan/packages/my_dev\n",
      "Installing collected packages: my-dev\n",
      "  Running setup.py install for my-dev: started\n",
      "    Running setup.py install for my-dev: finished with status 'done'\n",
      "Successfully installed my-dev-0.1\n",
      "\n"
     ]
    }
   ],
   "source": [
    "# build and install locally each package\n",
    "for package_name, from_pip in packages_todo:\n",
    "    cmd = ''\n",
    "    # run pip install within the python dir\n",
    "    if from_pip:\n",
    "        # if from pip then just use name\n",
    "        cmd = cmd + f'pip install --upgrade --force-reinstall \"{package_name}\" -t {work_dir}/python/python/.'\n",
    "    else:\n",
    "        # cd into the package dir\n",
    "        cmd = f'cd {package_base_dir}/{package_name}'\n",
    "        # run the build commands to create local egg file to be zipped up into an AWS layer\n",
    "        cmd = cmd + f' && python setup.py bdist_egg'\n",
    "        # then do a local install\n",
    "        cmd = cmd + f' && pip install --upgrade \"{package_base_dir}/{package_name}\" -t {work_dir}/python/python/.'\n",
    "    # kick off big command to do the stuff\n",
    "    result = subprocess_execute(cmd)\n",
    "    # look at results\n",
    "    print(result)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Create python.zip"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'/home/jovyan/work/python.zip'"
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# make zip\n",
    "shutil.make_archive(f'{work_dir}/python', 'zip', f'{work_dir}/python')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Upload to S3"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "... cmd to execute:\n",
      "aws s3 cp --region us-west-2 /home/jovyan/work/python.zip s3://andrewm4894-python-packages/latest/\n",
      "... result:\n",
      "... zip should be updated at: https://s3.console.aws.amazon.com/s3/buckets/andrewm4894-python-packages/latest?region=us-west-2&tab=overview ...\n"
     ]
    }
   ],
   "source": [
    "# upload zip to s3 using aws cli\n",
    "cmd = f'aws s3 cp --region {s3_region} {work_dir}/python.zip {s3_path}/'\n",
    "subprocess_execute(cmd)\n",
    "print(f'... zip should be updated at: {s3_url} ...')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Publish to AWS Layer"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "... cmd to execute:\n",
      "aws lambda publish-layer-version --layer-name my-aws-python-packages --content S3Bucket=andrewm4894-python-packages,S3Key=latest/python.zip --region us-west-2\n",
      "... result:\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "'{\\n    \"Content\": {\\n        \"Location\": \"https://awslambda-us-west-2-layers.s3.us-west-2.amazonaws.com/snapshots/908566381001/my-aws-python-packages-a2132736-f0ed-4bf2-8ca8-a3237a60254f?versionId=NyejNbE2dp8dUKR3Ln05R_x1QgJ5KBNB&X-Amz-Security-Token=12345&X-Amz-Algorithm=AWS4-HMAC-SHA256&X-Amz-Date=20190902T221416Z&X-Amz-SignedHeaders=host&X-Amz-Expires=600&X-Amz-Credential=1234&X-Amz-Signature=1234\",\\n        \"CodeSha256\": \"1234=\",\\n        \"CodeSize\": 38572755\\n    },\\n    \"LayerArn\": \"arn:aws:lambda:us-west-2:1234:layer:my-aws-python-packages\",\\n    \"LayerVersionArn\": \"arn:aws:lambda:us-west-2:1234:layer:my-aws-python-packages:1\",\\n    \"Description\": \"\",\\n    \"CreatedDate\": \"2019-09-02T22:14:32.712+0000\",\\n    \"Version\": 1\\n}\\n'"
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# publish zip file from s3 location to aws layer\n",
    "cmd = f\"aws lambda publish-layer-version\"\n",
    "cmd += f\" --layer-name {aws_layer_name}\"\n",
    "cmd += f\" --content S3Bucket={s3_bucket},S3Key={s3_path.split('/')[-1]}/python.zip\"\n",
    "cmd += f\" --region {s3_region}\"\n",
    "subprocess_execute(cmd)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.8"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}