{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Requirement already satisfied: coiled in c:\\users\\argenisleon\\anaconda3\\lib\\site-packages (0.0.23)\n",
      "Requirement already satisfied: pandas>=1.1.0 in c:\\users\\argenisleon\\anaconda3\\lib\\site-packages (from coiled) (1.1.2)\n",
      "Requirement already satisfied: aiohttp in c:\\users\\argenisleon\\anaconda3\\lib\\site-packages (from coiled) (3.6.2)\n",
      "Requirement already satisfied: click in c:\\users\\argenisleon\\anaconda3\\lib\\site-packages (from coiled) (7.1.1)\n",
      "Requirement already satisfied: distributed>=2.23.0 in c:\\users\\argenisleon\\anaconda3\\lib\\site-packages (from coiled) (2.25.0)\n",
      "Requirement already satisfied: s3fs in c:\\users\\argenisleon\\anaconda3\\lib\\site-packages (from coiled) (0.5.1)\n",
      "Requirement already satisfied: dask[complete]>=2.23.0 in c:\\users\\argenisleon\\anaconda3\\lib\\site-packages (from coiled) (2.25.0)\n",
      "Requirement already satisfied: ipython in c:\\users\\argenisleon\\appdata\\roaming\\python\\python37\\site-packages (from coiled) (7.5.0)\n",
      "Requirement already satisfied: aiobotocore>=1.0.7 in c:\\users\\argenisleon\\anaconda3\\lib\\site-packages (from coiled) (1.1.1)\n",
      "Requirement already satisfied: pytz>=2017.2 in c:\\users\\argenisleon\\appdata\\roaming\\python\\python37\\site-packages (from pandas>=1.1.0->coiled) (2019.3)\n",
      "Requirement already satisfied: numpy>=1.15.4 in c:\\users\\argenisleon\\anaconda3\\lib\\site-packages (from pandas>=1.1.0->coiled) (1.19.1)\n",
      "Requirement already satisfied: python-dateutil>=2.7.3 in c:\\users\\argenisleon\\appdata\\roaming\\python\\python37\\site-packages (from pandas>=1.1.0->coiled) (2.8.1)\n",
      "Requirement already satisfied: yarl<2.0,>=1.0 in c:\\users\\argenisleon\\anaconda3\\lib\\site-packages (from aiohttp->coiled) (1.5.1)\n",
      "Requirement already satisfied: multidict<5.0,>=4.5 in c:\\users\\argenisleon\\anaconda3\\lib\\site-packages (from aiohttp->coiled) (4.7.6)\n",
      "Requirement already satisfied: async-timeout<4.0,>=3.0 in c:\\users\\argenisleon\\anaconda3\\lib\\site-packages (from aiohttp->coiled) (3.0.1)\n",
      "Requirement already satisfied: attrs>=17.3.0 in c:\\users\\argenisleon\\anaconda3\\lib\\site-packages (from aiohttp->coiled) (19.3.0)\n",
      "Requirement already satisfied: chardet<4.0,>=2.0 in c:\\users\\argenisleon\\appdata\\roaming\\python\\python37\\site-packages (from aiohttp->coiled) (3.0.4)\n",
      "Requirement already satisfied: psutil>=5.0 in c:\\users\\argenisleon\\appdata\\roaming\\python\\python37\\site-packages (from distributed>=2.23.0->coiled) (5.7.0)\n",
      "Requirement already satisfied: sortedcontainers!=2.0.0,!=2.0.1 in c:\\users\\argenisleon\\anaconda3\\lib\\site-packages (from distributed>=2.23.0->coiled) (2.1.0)\n",
      "Requirement already satisfied: cloudpickle>=1.5.0 in c:\\users\\argenisleon\\anaconda3\\lib\\site-packages (from distributed>=2.23.0->coiled) (1.5.0)\n",
      "Requirement already satisfied: setuptools in c:\\users\\argenisleon\\anaconda3\\lib\\site-packages (from distributed>=2.23.0->coiled) (41.6.0)\n",
      "Requirement already satisfied: zict>=0.1.3 in c:\\users\\argenisleon\\anaconda3\\lib\\site-packages (from distributed>=2.23.0->coiled) (2.0.0)\n",
      "Requirement already satisfied: toolz>=0.8.2 in c:\\users\\argenisleon\\appdata\\roaming\\python\\python37\\site-packages (from distributed>=2.23.0->coiled) (0.10.0)\n",
      "Requirement already satisfied: pyyaml in c:\\users\\argenisleon\\appdata\\roaming\\python\\python37\\site-packages (from distributed>=2.23.0->coiled) (5.3.1)\n",
      "Requirement already satisfied: tornado>=5; python_version < \"3.8\" in c:\\users\\argenisleon\\anaconda3\\lib\\site-packages (from distributed>=2.23.0->coiled) (6.0.4)\n",
      "Requirement already satisfied: msgpack>=0.6.0 in c:\\users\\argenisleon\\anaconda3\\lib\\site-packages (from distributed>=2.23.0->coiled) (1.0.0)\n",
      "Requirement already satisfied: tblib>=1.6.0 in c:\\users\\argenisleon\\anaconda3\\lib\\site-packages (from distributed>=2.23.0->coiled) (1.6.0)\n",
      "Requirement already satisfied: fsspec>=0.8.0 in c:\\users\\argenisleon\\anaconda3\\lib\\site-packages (from s3fs->coiled) (0.8.1)\n",
      "Requirement already satisfied: partd>=0.3.10; extra == \"complete\" in c:\\users\\argenisleon\\anaconda3\\lib\\site-packages (from dask[complete]>=2.23.0->coiled) (1.1.0)\n",
      "Requirement already satisfied: bokeh>=1.0.0; extra == \"complete\" in c:\\users\\argenisleon\\anaconda3\\lib\\site-packages (from dask[complete]>=2.23.0->coiled) (2.0.1)\n",
      "Requirement already satisfied: pygments in c:\\users\\argenisleon\\anaconda3\\lib\\site-packages (from ipython->coiled) (2.6.1)\n",
      "Requirement already satisfied: prompt-toolkit<2.1.0,>=2.0.0 in c:\\users\\argenisleon\\anaconda3\\lib\\site-packages (from ipython->coiled) (2.0.10)\n",
      "Requirement already satisfied: decorator in c:\\users\\argenisleon\\anaconda3\\lib\\site-packages (from ipython->coiled) (4.4.2)\n",
      "Requirement already satisfied: traitlets>=4.2 in c:\\users\\argenisleon\\anaconda3\\lib\\site-packages (from ipython->coiled) (4.3.3)\n",
      "Requirement already satisfied: pickleshare in c:\\users\\argenisleon\\anaconda3\\lib\\site-packages (from ipython->coiled) (0.7.5)\n",
      "Requirement already satisfied: backcall in c:\\users\\argenisleon\\anaconda3\\lib\\site-packages (from ipython->coiled) (0.1.0)\n",
      "Requirement already satisfied: jedi>=0.10 in c:\\users\\argenisleon\\anaconda3\\lib\\site-packages (from ipython->coiled) (0.14.1)\n",
      "Requirement already satisfied: colorama; sys_platform == \"win32\" in c:\\users\\argenisleon\\anaconda3\\lib\\site-packages (from ipython->coiled) (0.4.3)\n",
      "Requirement already satisfied: wrapt>=1.10.10 in c:\\users\\argenisleon\\anaconda3\\lib\\site-packages (from aiobotocore>=1.0.7->coiled) (1.12.1)\n",
      "Requirement already satisfied: botocore<1.17.45,>=1.17.44 in c:\\users\\argenisleon\\anaconda3\\lib\\site-packages (from aiobotocore>=1.0.7->coiled) (1.17.44)\n",
      "Requirement already satisfied: aioitertools>=0.5.1 in c:\\users\\argenisleon\\anaconda3\\lib\\site-packages (from aiobotocore>=1.0.7->coiled) (0.7.0)\n",
      "Requirement already satisfied: six>=1.5 in c:\\users\\argenisleon\\appdata\\roaming\\python\\python37\\site-packages (from python-dateutil>=2.7.3->pandas>=1.1.0->coiled) (1.14.0)\n",
      "Requirement already satisfied: typing-extensions>=3.7.4; python_version < \"3.8\" in c:\\users\\argenisleon\\anaconda3\\lib\\site-packages (from yarl<2.0,>=1.0->aiohttp->coiled) (3.7.4.1)\n",
      "Requirement already satisfied: idna>=2.0 in c:\\users\\argenisleon\\appdata\\roaming\\python\\python37\\site-packages (from yarl<2.0,>=1.0->aiohttp->coiled) (2.9)\n",
      "Requirement already satisfied: heapdict in c:\\users\\argenisleon\\anaconda3\\lib\\site-packages (from zict>=0.1.3->distributed>=2.23.0->coiled) (1.0.1)\n",
      "Requirement already satisfied: locket in c:\\users\\argenisleon\\anaconda3\\lib\\site-packages (from partd>=0.3.10; extra == \"complete\"->dask[complete]>=2.23.0->coiled) (0.2.0)\n",
      "Requirement already satisfied: Jinja2>=2.7 in c:\\users\\argenisleon\\anaconda3\\lib\\site-packages (from bokeh>=1.0.0; extra == \"complete\"->dask[complete]>=2.23.0->coiled) (2.11.1)\n",
      "Requirement already satisfied: pillow>=4.0 in c:\\users\\argenisleon\\anaconda3\\lib\\site-packages (from bokeh>=1.0.0; extra == \"complete\"->dask[complete]>=2.23.0->coiled) (7.1.1)\n",
      "Requirement already satisfied: packaging>=16.8 in c:\\users\\argenisleon\\anaconda3\\lib\\site-packages (from bokeh>=1.0.0; extra == \"complete\"->dask[complete]>=2.23.0->coiled) (20.3)\n",
      "Requirement already satisfied: wcwidth in c:\\users\\argenisleon\\anaconda3\\lib\\site-packages (from prompt-toolkit<2.1.0,>=2.0.0->ipython->coiled) (0.1.9)\n",
      "Requirement already satisfied: ipython-genutils in c:\\users\\argenisleon\\anaconda3\\lib\\site-packages (from traitlets>=4.2->ipython->coiled) (0.2.0)\n",
      "Requirement already satisfied: parso>=0.5.0 in c:\\users\\argenisleon\\anaconda3\\lib\\site-packages (from jedi>=0.10->ipython->coiled) (0.5.2)\n",
      "Requirement already satisfied: urllib3<1.26,>=1.20; python_version != \"3.4\" in c:\\users\\argenisleon\\anaconda3\\lib\\site-packages (from botocore<1.17.45,>=1.17.44->aiobotocore>=1.0.7->coiled) (1.24.3)\n",
      "Requirement already satisfied: jmespath<1.0.0,>=0.7.1 in c:\\users\\argenisleon\\anaconda3\\lib\\site-packages (from botocore<1.17.45,>=1.17.44->aiobotocore>=1.0.7->coiled) (0.9.4)\n",
      "Requirement already satisfied: docutils<0.16,>=0.10 in c:\\users\\argenisleon\\appdata\\roaming\\python\\python37\\site-packages (from botocore<1.17.45,>=1.17.44->aiobotocore>=1.0.7->coiled) (0.15.2)\n",
      "Requirement already satisfied: MarkupSafe>=0.23 in c:\\users\\argenisleon\\anaconda3\\lib\\site-packages (from Jinja2>=2.7->bokeh>=1.0.0; extra == \"complete\"->dask[complete]>=2.23.0->coiled) (1.1.1)\n",
      "Requirement already satisfied: pyparsing>=2.0.2 in c:\\users\\argenisleon\\anaconda3\\lib\\site-packages (from packaging>=16.8->bokeh>=1.0.0; extra == \"complete\"->dask[complete]>=2.23.0->coiled) (2.4.6)\n"
     ]
    }
   ],
   "source": [
    "# !pip install coiled"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Traceback (most recent call last):\n",
      "  File \"c:\\users\\argenisleon\\anaconda3\\lib\\runpy.py\", line 193, in _run_module_as_main\n",
      "    \"__main__\", mod_spec)\n",
      "  File \"c:\\users\\argenisleon\\anaconda3\\lib\\runpy.py\", line 85, in _run_code\n",
      "    exec(code, run_globals)\n",
      "  File \"C:\\Users\\argenisleon\\Anaconda3\\Scripts\\coiled.exe\\__main__.py\", line 7, in <module>\n",
      "  File \"c:\\users\\argenisleon\\anaconda3\\lib\\site-packages\\click\\core.py\", line 829, in __call__\n",
      "    return self.main(*args, **kwargs)\n",
      "  File \"c:\\users\\argenisleon\\anaconda3\\lib\\site-packages\\click\\core.py\", line 782, in main\n",
      "    rv = self.invoke(ctx)\n",
      "  File \"c:\\users\\argenisleon\\anaconda3\\lib\\site-packages\\click\\core.py\", line 1259, in invoke\n",
      "    return _process_result(sub_ctx.command.invoke(sub_ctx))\n",
      "  File \"c:\\users\\argenisleon\\anaconda3\\lib\\site-packages\\click\\core.py\", line 1066, in invoke\n",
      "    return ctx.invoke(self.callback, **ctx.params)\n",
      "  File \"c:\\users\\argenisleon\\anaconda3\\lib\\site-packages\\click\\core.py\", line 610, in invoke\n",
      "    return callback(*args, **kwargs)\n",
      "  File \"c:\\users\\argenisleon\\anaconda3\\lib\\site-packages\\coiled\\cli\\login.py\", line 17, in login\n",
      "    handle_credentials(server=server, token=token, save=True)\n",
      "  File \"c:\\users\\argenisleon\\anaconda3\\lib\\asyncio\\base_events.py\", line 583, in run_until_complete\n",
      "    return future.result()\n",
      "  File \"c:\\users\\argenisleon\\anaconda3\\lib\\site-packages\\coiled\\utils.py\", line 137, in handle_credentials\n",
      "    await handle_api_exception(response)\n",
      "  File \"c:\\users\\argenisleon\\anaconda3\\lib\\site-packages\\coiled\\utils.py\", line 64, in handle_api_exception\n",
      "    raise exception_cls(error_body)\n",
      "Exception: {'detail': 'Authentication credentials were not provided.'}\n"
     ]
    }
   ],
   "source": [
    "# !coiled login --token $coiled_token"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "import coiled"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "coiled.create_software_environment(name='zarr', conda=['xarray', 'zarr','dask=2.2.23','distributed'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "coiled.create_cluster_configuration(\n",
    "    name=\"bumblebee\",\n",
    "    worker_memory=\"16 GiB\",\n",
    "    worker_cpu=4,\n",
    "    scheduler_memory=\"4 GiB\",\n",
    "    scheduler_cpu=1,\n",
    "    software=\"coiled/default\"\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "cluster.name #salvar en bumblebee"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "coiled.list_clusters()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "cluster = coiled.Cluster(configuration='bumblebee')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "cluster.name # argenisleon-xxxxxxxx-x"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "coiled.list_clusters()  # {\"argenisleon-xxxxxxxx-x\":{...}}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "from dask.distributed import Client\n",
    "client = Client(name=cluster.name)\n",
    "print('Dashboard:', client.dashboard_link)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "dir(coiled)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import dask.dataframe as dd\n",
    "\n",
    "df = dd.read_csv(\n",
    "    \"s3://nyc-tlc/trip data/yellow_tripdata_2019-*.csv\",\n",
    "    dtype={\n",
    "        \"payment_type\": \"UInt8\",\n",
    "        \"VendorID\": \"UInt8\",\n",
    "        \"passenger_count\": \"UInt8\",\n",
    "        \"RatecodeID\": \"UInt8\",\n",
    "    },\n",
    "    storage_options={\"anon\": True},\n",
    "    blocksize=\"16 MiB\",\n",
    ").persist()\n",
    "\n",
    "df.groupby(\"passenger_count\").tip_amount.mean().compute()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [
    {
     "ename": "NameError",
     "evalue": "name 'op' is not defined",
     "output_type": "error",
     "traceback": [
      "\u001b[1;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[1;31mNameError\u001b[0m                                 Traceback (most recent call last)",
      "\u001b[1;32m<ipython-input-1-99393fff965a>\u001b[0m in \u001b[0;36m<module>\u001b[1;34m\u001b[0m\n\u001b[1;32m----> 1\u001b[1;33m \u001b[0mop\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m",
      "\u001b[1;31mNameError\u001b[0m: name 'op' is not defined"
     ]
    }
   ],
   "source": [
    "op"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.6"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}