{ "cells": [ { "cell_type": "code", "execution_count": 2, "metadata": { "collapsed": true }, "outputs": [], "source": [ "from dask.distributed import Client\n", "c = Client()\n", "import dask.dataframe as dd" ] }, { "cell_type": "code", "execution_count": 3, "metadata": { "collapsed": true }, "outputs": [], "source": [ "s = dd.read_parquet('/bigdata/subway.parquet')" ] }, { "cell_type": "code", "execution_count": 6, "metadata": { "collapsed": false }, "outputs": [], "source": [ "s = c.persist(s.set_index('ca', npartitions=200))" ] }, { "cell_type": "code", "execution_count": 7, "metadata": { "collapsed": true }, "outputs": [], "source": [ "s.to_parquet('/bigdata/subway2.parquet')" ] }, { "cell_type": "code", "execution_count": 8, "metadata": { "collapsed": false }, "outputs": [ { "data": { "text/html": [ "
\n", "\n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", "
unitscpstationlinenamedivisiondescriptioncumul_entriescumul_exits
ca
55R50800-00-01NULLNULLNULLREGULAR241
55R50800-00-01NULLNULLNULLREGULAR241
55R50800-00-01NULLNULLNULLREGULAR241
55R50800-00-01NULLNULLNULLREGULAR241
55R50800-00-01NULLNULLNULLREGULAR241
\n", "
" ], "text/plain": [ " unit scp station linename division description cumul_entries \\\n", "ca \n", "55 R508 00-00-01 NULL NULL NULL REGULAR 24 \n", "55 R508 00-00-01 NULL NULL NULL REGULAR 24 \n", "55 R508 00-00-01 NULL NULL NULL REGULAR 24 \n", "55 R508 00-00-01 NULL NULL NULL REGULAR 24 \n", "55 R508 00-00-01 NULL NULL NULL REGULAR 24 \n", "\n", " cumul_exits \n", "ca \n", "55 1 \n", "55 1 \n", "55 1 \n", "55 1 \n", "55 1 " ] }, "execution_count": 8, "metadata": {}, "output_type": "execute_result" } ], "source": [ "s.head()" ] }, { "cell_type": "code", "execution_count": 19, "metadata": { "collapsed": false }, "outputs": [ { "ename": "TypeError", "evalue": "Don't know how to create metadata from [False False]", "output_type": "error", "traceback": [ "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m", "\u001b[0;31mTypeError\u001b[0m Traceback (most recent call last)", "\u001b[0;32m\u001b[0m in \u001b[0;36m\u001b[0;34m()\u001b[0m\n\u001b[0;32m----> 1\u001b[0;31m \u001b[0ms\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0ms\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mindex\u001b[0m \u001b[0;34m==\u001b[0m \u001b[0;34m'55'\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m", "\u001b[0;32m/home/shekhar/anaconda3/lib/python3.5/site-packages/dask/dataframe/core.py\u001b[0m in \u001b[0;36m\u001b[0;34m(self, other)\u001b[0m\n\u001b[1;32m 970\u001b[0m \u001b[0;32mreturn\u001b[0m \u001b[0;32mlambda\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mother\u001b[0m\u001b[0;34m:\u001b[0m \u001b[0melemwise\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mop\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mother\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 971\u001b[0m \u001b[0;32melse\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 972\u001b[0;31m \u001b[0;32mreturn\u001b[0m \u001b[0;32mlambda\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mother\u001b[0m\u001b[0;34m:\u001b[0m \u001b[0melemwise\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mop\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mother\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 973\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 974\u001b[0m def rolling(self, window, min_periods=None, freq=None, center=False,\n", "\u001b[0;32m/home/shekhar/anaconda3/lib/python3.5/site-packages/dask/dataframe/core.py\u001b[0m in \u001b[0;36melemwise\u001b[0;34m(op, *args, **kwargs)\u001b[0m\n\u001b[1;32m 2817\u001b[0m \u001b[0mmeta\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0m_emulate\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mop\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m*\u001b[0m\u001b[0margs\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m**\u001b[0m\u001b[0mkwargs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 2818\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m-> 2819\u001b[0;31m \u001b[0;32mreturn\u001b[0m \u001b[0mnew_dd_object\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mdsk\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0m_name\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mmeta\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mdivisions\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 2820\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 2821\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n", "\u001b[0;32m/home/shekhar/anaconda3/lib/python3.5/site-packages/dask/dataframe/core.py\u001b[0m in \u001b[0;36mnew_dd_object\u001b[0;34m(dsk, _name, meta, divisions)\u001b[0m\n\u001b[1;32m 77\u001b[0m \u001b[0mDecides\u001b[0m \u001b[0mthe\u001b[0m \u001b[0mappropriate\u001b[0m \u001b[0moutput\u001b[0m \u001b[0;32mclass\u001b[0m \u001b[0mbased\u001b[0m \u001b[0mon\u001b[0m \u001b[0mthe\u001b[0m \u001b[0mtype\u001b[0m \u001b[0mof\u001b[0m\u001b[0;31m \u001b[0m\u001b[0;31m`\u001b[0m\u001b[0mmeta\u001b[0m\u001b[0;31m`\u001b[0m \u001b[0mprovided\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 78\u001b[0m \"\"\"\n\u001b[0;32m---> 79\u001b[0;31m \u001b[0;32mreturn\u001b[0m \u001b[0m_get_return_type\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mmeta\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mdsk\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0m_name\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mmeta\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mdivisions\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 80\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 81\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n", "\u001b[0;32m/home/shekhar/anaconda3/lib/python3.5/site-packages/dask/dataframe/core.py\u001b[0m in \u001b[0;36m__init__\u001b[0;34m(self, dsk, name, meta, divisions)\u001b[0m\n\u001b[1;32m 101\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mdask\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mdsk\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 102\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_name\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mname\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 103\u001b[0;31m \u001b[0mmeta\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mmake_meta\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mmeta\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 104\u001b[0m \u001b[0;32mif\u001b[0m \u001b[0misinstance\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mmeta\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m(\u001b[0m\u001b[0mpd\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mDataFrame\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mpd\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mSeries\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mpd\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mIndex\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 105\u001b[0m raise TypeError(\"Expected meta to specify scalar, got \"\n", "\u001b[0;32m/home/shekhar/anaconda3/lib/python3.5/site-packages/dask/dataframe/utils.py\u001b[0m in \u001b[0;36mmake_meta\u001b[0;34m(x, index)\u001b[0m\n\u001b[1;32m 304\u001b[0m \u001b[0;32mreturn\u001b[0m \u001b[0m_nonempty_scalar\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mx\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 305\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 306\u001b[0;31m \u001b[0;32mraise\u001b[0m \u001b[0mTypeError\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m\"Don't know how to create metadata from {0}\"\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mformat\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mx\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 307\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 308\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n", "\u001b[0;31mTypeError\u001b[0m: Don't know how to create metadata from [False False]" ] } ], "source": [ "s[s.index == '55']" ] }, { "cell_type": "code", "execution_count": 13, "metadata": { "collapsed": false }, "outputs": [ { "data": { "text/plain": [ "0 55\n", "1 A002\n", "2 A006\n", "3 A007\n", "4 A010\n", "5 A011\n", "6 A013\n", "7 A014\n", "8 A015\n", "9 A016\n", "10 A021\n", "11 A022\n", "12 A025\n", "13 A027\n", "14 A029\n", "15 A030\n", "16 A031\n", "17 A033\n", "18 A034\n", "19 A035\n", "20 A037\n", "21 A038\n", "22 A039\n", "23 A041\n", "24 A042\n", "25 A043\n", "26 A046\n", "27 A047\n", "28 A049\n", "29 A050\n", " ... \n", "724 R624\n", "725 R625\n", "726 R626\n", "727 R627\n", "728 R628\n", "729 R629\n", "730 R630\n", "731 R632\n", "732 R633\n", "733 R634\n", "734 R635\n", "735 R636\n", "736 R637\n", "737 R639\n", "738 R641\n", "739 R643\n", "740 R644\n", "741 R645\n", "742 R646\n", "743 R647\n", "744 R726\n", "745 R727\n", "746 R728\n", "747 R729\n", "748 R730\n", "749 S101\n", "750 S101A\n", "751 S102\n", "752 TRAM1\n", "753 TRAM2\n", "Name: ca, dtype: object" ] }, "execution_count": 13, "metadata": {}, "output_type": "execute_result" } ], "source": [ "s.index.unique().compute()" ] }, { "cell_type": "code", "execution_count": 4, "metadata": { "collapsed": false }, "outputs": [], "source": [ "zz = s['ca unit scp station'.split()].groupby(['ca', 'unit', 'scp']).count().compute()" ] }, { "cell_type": "code", "execution_count": 9, "metadata": { "collapsed": false }, "outputs": [ { "name": "stderr", "output_type": "stream", "text": [ "distributed.batched - INFO - Batched Comm Closed: ConnectionResetError: [Errno 104] Connection reset by peer\n", "distributed.utils - ERROR - (\"('getitem-dd827194738d781422cddd0e8dc97ff0', 38)\", 'tcp://127.0.0.1:44991')\n", "Traceback (most recent call last):\n", " File \"/home/shekhar/anaconda3/lib/python3.5/site-packages/distributed/utils.py\", line 193, in f\n", " result[0] = yield gen.maybe_future(func(*args, **kwargs))\n", " File \"/home/shekhar/anaconda3/lib/python3.5/site-packages/tornado/gen.py\", line 1015, in run\n", " value = future.result()\n", " File \"/home/shekhar/anaconda3/lib/python3.5/site-packages/tornado/concurrent.py\", line 237, in result\n", " raise_exc_info(self._exc_info)\n", " File \"\", line 3, in raise_exc_info\n", " File \"/home/shekhar/anaconda3/lib/python3.5/site-packages/tornado/gen.py\", line 1021, in run\n", " yielded = self.gen.throw(*exc_info)\n", " File \"/home/shekhar/anaconda3/lib/python3.5/site-packages/distributed/client.py\", line 1486, in _get\n", " result = yield self._gather(packed)\n", " File \"/home/shekhar/anaconda3/lib/python3.5/site-packages/tornado/gen.py\", line 1015, in run\n", " value = future.result()\n", " File \"/home/shekhar/anaconda3/lib/python3.5/site-packages/tornado/concurrent.py\", line 237, in result\n", " raise_exc_info(self._exc_info)\n", " File \"\", line 3, in raise_exc_info\n", " File \"/home/shekhar/anaconda3/lib/python3.5/site-packages/tornado/gen.py\", line 1021, in run\n", " yielded = self.gen.throw(*exc_info)\n", " File \"/home/shekhar/anaconda3/lib/python3.5/site-packages/distributed/client.py\", line 939, in _gather\n", " traceback)\n", " File \"/home/shekhar/anaconda3/lib/python3.5/site-packages/six.py\", line 686, in reraise\n", " raise value\n", "distributed.scheduler.KilledWorker: (\"('getitem-dd827194738d781422cddd0e8dc97ff0', 38)\", 'tcp://127.0.0.1:44991')\n" ] }, { "ename": "KilledWorker", "evalue": "(\"('getitem-dd827194738d781422cddd0e8dc97ff0', 38)\", 'tcp://127.0.0.1:44991')", "output_type": "error", "traceback": [ "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m", "\u001b[0;31mKilledWorker\u001b[0m Traceback (most recent call last)", "\u001b[0;32m\u001b[0m in \u001b[0;36m\u001b[0;34m()\u001b[0m\n\u001b[0;32m----> 1\u001b[0;31m \u001b[0mk0\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0ms\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0ms\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mca\u001b[0m\u001b[0;34m==\u001b[0m\u001b[0;34m'A002'\u001b[0m\u001b[0;34m)\u001b[0m \u001b[0;34m&\u001b[0m \u001b[0;34m(\u001b[0m\u001b[0ms\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0munit\u001b[0m\u001b[0;34m==\u001b[0m\u001b[0;34m'R051'\u001b[0m\u001b[0;34m)\u001b[0m \u001b[0;34m&\u001b[0m \u001b[0;34m(\u001b[0m\u001b[0ms\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mscp\u001b[0m\u001b[0;34m==\u001b[0m\u001b[0;34m'02-00-00'\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mcompute\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m", "\u001b[0;32m/home/shekhar/anaconda3/lib/python3.5/site-packages/dask/base.py\u001b[0m in \u001b[0;36mcompute\u001b[0;34m(self, **kwargs)\u001b[0m\n\u001b[1;32m 93\u001b[0m \u001b[0mExtra\u001b[0m \u001b[0mkeywords\u001b[0m \u001b[0mto\u001b[0m \u001b[0mforward\u001b[0m \u001b[0mto\u001b[0m \u001b[0mthe\u001b[0m \u001b[0mscheduler\u001b[0m\u001b[0;31m \u001b[0m\u001b[0;31m`\u001b[0m\u001b[0;31m`\u001b[0m\u001b[0mget\u001b[0m\u001b[0;31m`\u001b[0m\u001b[0;31m`\u001b[0m \u001b[0mfunction\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 94\u001b[0m \"\"\"\n\u001b[0;32m---> 95\u001b[0;31m \u001b[0;34m(\u001b[0m\u001b[0mresult\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m)\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mcompute\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mtraverse\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;32mFalse\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m**\u001b[0m\u001b[0mkwargs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 96\u001b[0m \u001b[0;32mreturn\u001b[0m \u001b[0mresult\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 97\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n", "\u001b[0;32m/home/shekhar/anaconda3/lib/python3.5/site-packages/dask/base.py\u001b[0m in \u001b[0;36mcompute\u001b[0;34m(*args, **kwargs)\u001b[0m\n\u001b[1;32m 200\u001b[0m \u001b[0mdsk\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mcollections_to_dsk\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mvariables\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0moptimize_graph\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m**\u001b[0m\u001b[0mkwargs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 201\u001b[0m \u001b[0mkeys\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;34m[\u001b[0m\u001b[0mvar\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_keys\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m \u001b[0;32mfor\u001b[0m \u001b[0mvar\u001b[0m \u001b[0;32min\u001b[0m \u001b[0mvariables\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 202\u001b[0;31m \u001b[0mresults\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mget\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mdsk\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mkeys\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m**\u001b[0m\u001b[0mkwargs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 203\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 204\u001b[0m \u001b[0mresults_iter\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0miter\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mresults\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n", "\u001b[0;32m/home/shekhar/anaconda3/lib/python3.5/site-packages/distributed/client.py\u001b[0m in \u001b[0;36mget\u001b[0;34m(self, dsk, keys, restrictions, loose_restrictions, resources, **kwargs)\u001b[0m\n\u001b[1;32m 1523\u001b[0m return sync(self.loop, self._get, dsk, keys, restrictions=restrictions,\n\u001b[1;32m 1524\u001b[0m \u001b[0mloose_restrictions\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mloose_restrictions\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m-> 1525\u001b[0;31m resources=resources)\n\u001b[0m\u001b[1;32m 1526\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 1527\u001b[0m \u001b[0;32mdef\u001b[0m \u001b[0m_optimize_insert_futures\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mdsk\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mkeys\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n", "\u001b[0;32m/home/shekhar/anaconda3/lib/python3.5/site-packages/distributed/utils.py\u001b[0m in \u001b[0;36msync\u001b[0;34m(loop, func, *args, **kwargs)\u001b[0m\n\u001b[1;32m 202\u001b[0m \u001b[0me\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mwait\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;36m1000000\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 203\u001b[0m \u001b[0;32mif\u001b[0m \u001b[0merror\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;36m0\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 204\u001b[0;31m \u001b[0msix\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mreraise\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m*\u001b[0m\u001b[0merror\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;36m0\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 205\u001b[0m \u001b[0;32melse\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 206\u001b[0m \u001b[0;32mreturn\u001b[0m \u001b[0mresult\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;36m0\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n", "\u001b[0;32m/home/shekhar/anaconda3/lib/python3.5/site-packages/six.py\u001b[0m in \u001b[0;36mreraise\u001b[0;34m(tp, value, tb)\u001b[0m\n\u001b[1;32m 684\u001b[0m \u001b[0;32mif\u001b[0m \u001b[0mvalue\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m__traceback__\u001b[0m \u001b[0;32mis\u001b[0m \u001b[0;32mnot\u001b[0m \u001b[0mtb\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 685\u001b[0m \u001b[0;32mraise\u001b[0m \u001b[0mvalue\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mwith_traceback\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mtb\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 686\u001b[0;31m \u001b[0;32mraise\u001b[0m \u001b[0mvalue\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 687\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 688\u001b[0m \u001b[0;32melse\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n", "\u001b[0;32m/home/shekhar/anaconda3/lib/python3.5/site-packages/distributed/utils.py\u001b[0m in \u001b[0;36mf\u001b[0;34m()\u001b[0m\n\u001b[1;32m 191\u001b[0m \u001b[0;32mraise\u001b[0m \u001b[0mRuntimeError\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m\"sync() called from thread of running loop\"\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 192\u001b[0m \u001b[0;32myield\u001b[0m \u001b[0mgen\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mmoment\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 193\u001b[0;31m \u001b[0mresult\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;36m0\u001b[0m\u001b[0;34m]\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;32myield\u001b[0m \u001b[0mgen\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mmaybe_future\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mfunc\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m*\u001b[0m\u001b[0margs\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m**\u001b[0m\u001b[0mkwargs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 194\u001b[0m \u001b[0;32mexcept\u001b[0m \u001b[0mException\u001b[0m \u001b[0;32mas\u001b[0m \u001b[0mexc\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 195\u001b[0m \u001b[0mlogger\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mexception\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mexc\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n", "\u001b[0;32m/home/shekhar/anaconda3/lib/python3.5/site-packages/tornado/gen.py\u001b[0m in \u001b[0;36mrun\u001b[0;34m(self)\u001b[0m\n\u001b[1;32m 1013\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 1014\u001b[0m \u001b[0;32mtry\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m-> 1015\u001b[0;31m \u001b[0mvalue\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mfuture\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mresult\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 1016\u001b[0m \u001b[0;32mexcept\u001b[0m \u001b[0mException\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 1017\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mhad_exception\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;32mTrue\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n", "\u001b[0;32m/home/shekhar/anaconda3/lib/python3.5/site-packages/tornado/concurrent.py\u001b[0m in \u001b[0;36mresult\u001b[0;34m(self, timeout)\u001b[0m\n\u001b[1;32m 235\u001b[0m \u001b[0;32mreturn\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_result\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 236\u001b[0m \u001b[0;32mif\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_exc_info\u001b[0m \u001b[0;32mis\u001b[0m \u001b[0;32mnot\u001b[0m \u001b[0;32mNone\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 237\u001b[0;31m \u001b[0mraise_exc_info\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_exc_info\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 238\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_check_done\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 239\u001b[0m \u001b[0;32mreturn\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_result\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n", "\u001b[0;32m/home/shekhar/anaconda3/lib/python3.5/site-packages/tornado/util.py\u001b[0m in \u001b[0;36mraise_exc_info\u001b[0;34m(exc_info)\u001b[0m\n", "\u001b[0;32m/home/shekhar/anaconda3/lib/python3.5/site-packages/tornado/gen.py\u001b[0m in \u001b[0;36mrun\u001b[0;34m(self)\u001b[0m\n\u001b[1;32m 1019\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 1020\u001b[0m \u001b[0;32mif\u001b[0m \u001b[0mexc_info\u001b[0m \u001b[0;32mis\u001b[0m \u001b[0;32mnot\u001b[0m \u001b[0;32mNone\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m-> 1021\u001b[0;31m \u001b[0myielded\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mgen\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mthrow\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m*\u001b[0m\u001b[0mexc_info\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 1022\u001b[0m \u001b[0mexc_info\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;32mNone\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 1023\u001b[0m \u001b[0;32melse\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n", "\u001b[0;32m/home/shekhar/anaconda3/lib/python3.5/site-packages/distributed/client.py\u001b[0m in \u001b[0;36m_get\u001b[0;34m(self, dsk, keys, restrictions, loose_restrictions, resources, raise_on_error)\u001b[0m\n\u001b[1;32m 1484\u001b[0m \u001b[0mpacked\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mpack_data\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mkeys\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mfutures\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 1485\u001b[0m \u001b[0;32mtry\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m-> 1486\u001b[0;31m \u001b[0mresult\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;32myield\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_gather\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mpacked\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 1487\u001b[0m \u001b[0;32mexcept\u001b[0m \u001b[0mException\u001b[0m \u001b[0;32mas\u001b[0m \u001b[0me\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 1488\u001b[0m \u001b[0;32mif\u001b[0m \u001b[0mraise_on_error\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n", "\u001b[0;32m/home/shekhar/anaconda3/lib/python3.5/site-packages/tornado/gen.py\u001b[0m in \u001b[0;36mrun\u001b[0;34m(self)\u001b[0m\n\u001b[1;32m 1013\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 1014\u001b[0m \u001b[0;32mtry\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m-> 1015\u001b[0;31m \u001b[0mvalue\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mfuture\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mresult\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 1016\u001b[0m \u001b[0;32mexcept\u001b[0m \u001b[0mException\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 1017\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mhad_exception\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;32mTrue\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n", "\u001b[0;32m/home/shekhar/anaconda3/lib/python3.5/site-packages/tornado/concurrent.py\u001b[0m in \u001b[0;36mresult\u001b[0;34m(self, timeout)\u001b[0m\n\u001b[1;32m 235\u001b[0m \u001b[0;32mreturn\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_result\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 236\u001b[0m \u001b[0;32mif\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_exc_info\u001b[0m \u001b[0;32mis\u001b[0m \u001b[0;32mnot\u001b[0m \u001b[0;32mNone\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 237\u001b[0;31m \u001b[0mraise_exc_info\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_exc_info\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 238\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_check_done\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 239\u001b[0m \u001b[0;32mreturn\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_result\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n", "\u001b[0;32m/home/shekhar/anaconda3/lib/python3.5/site-packages/tornado/util.py\u001b[0m in \u001b[0;36mraise_exc_info\u001b[0;34m(exc_info)\u001b[0m\n", "\u001b[0;32m/home/shekhar/anaconda3/lib/python3.5/site-packages/tornado/gen.py\u001b[0m in \u001b[0;36mrun\u001b[0;34m(self)\u001b[0m\n\u001b[1;32m 1019\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 1020\u001b[0m \u001b[0;32mif\u001b[0m \u001b[0mexc_info\u001b[0m \u001b[0;32mis\u001b[0m \u001b[0;32mnot\u001b[0m \u001b[0;32mNone\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m-> 1021\u001b[0;31m \u001b[0myielded\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mgen\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mthrow\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m*\u001b[0m\u001b[0mexc_info\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 1022\u001b[0m \u001b[0mexc_info\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;32mNone\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 1023\u001b[0m \u001b[0;32melse\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n", "\u001b[0;32m/home/shekhar/anaconda3/lib/python3.5/site-packages/distributed/client.py\u001b[0m in \u001b[0;36m_gather\u001b[0;34m(self, futures, errors)\u001b[0m\n\u001b[1;32m 937\u001b[0m six.reraise(type(exception),\n\u001b[1;32m 938\u001b[0m \u001b[0mexception\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 939\u001b[0;31m traceback)\n\u001b[0m\u001b[1;32m 940\u001b[0m \u001b[0;32mif\u001b[0m \u001b[0merrors\u001b[0m \u001b[0;34m==\u001b[0m \u001b[0;34m'skip'\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 941\u001b[0m \u001b[0mbad_keys\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0madd\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mkey\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n", "\u001b[0;32m/home/shekhar/anaconda3/lib/python3.5/site-packages/six.py\u001b[0m in \u001b[0;36mreraise\u001b[0;34m(tp, value, tb)\u001b[0m\n\u001b[1;32m 684\u001b[0m \u001b[0;32mif\u001b[0m \u001b[0mvalue\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m__traceback__\u001b[0m \u001b[0;32mis\u001b[0m \u001b[0;32mnot\u001b[0m \u001b[0mtb\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 685\u001b[0m \u001b[0;32mraise\u001b[0m \u001b[0mvalue\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mwith_traceback\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mtb\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 686\u001b[0;31m \u001b[0;32mraise\u001b[0m \u001b[0mvalue\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 687\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 688\u001b[0m \u001b[0;32melse\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n", "\u001b[0;31mKilledWorker\u001b[0m: (\"('getitem-dd827194738d781422cddd0e8dc97ff0', 38)\", 'tcp://127.0.0.1:44991')" ] }, { "name": "stderr", "output_type": "stream", "text": [ "tornado.application - ERROR - Exception in callback functools.partial(.null_wrapper at 0x7f2bf807e730>, )\n", "Traceback (most recent call last):\n", " File \"/home/shekhar/anaconda3/lib/python3.5/site-packages/tornado/ioloop.py\", line 604, in _run_callback\n", " ret = callback()\n", " File \"/home/shekhar/anaconda3/lib/python3.5/site-packages/tornado/stack_context.py\", line 275, in null_wrapper\n", " return fn(*args, **kwargs)\n", " File \"/home/shekhar/anaconda3/lib/python3.5/site-packages/tornado/ioloop.py\", line 619, in \n", " self.add_future(ret, lambda f: f.result())\n", " File \"/home/shekhar/anaconda3/lib/python3.5/site-packages/tornado/concurrent.py\", line 237, in result\n", " raise_exc_info(self._exc_info)\n", " File \"\", line 3, in raise_exc_info\n", " File \"/home/shekhar/anaconda3/lib/python3.5/site-packages/tornado/gen.py\", line 1021, in run\n", " yielded = self.gen.throw(*exc_info)\n", " File \"/home/shekhar/anaconda3/lib/python3.5/site-packages/distributed/nanny.py\", line 286, in _watch\n", " yield self.instantiate()\n", " File \"/home/shekhar/anaconda3/lib/python3.5/site-packages/tornado/gen.py\", line 1015, in run\n", " value = future.result()\n", " File \"/home/shekhar/anaconda3/lib/python3.5/site-packages/tornado/concurrent.py\", line 237, in result\n", " raise_exc_info(self._exc_info)\n", " File \"\", line 3, in raise_exc_info\n", " File \"/home/shekhar/anaconda3/lib/python3.5/site-packages/tornado/gen.py\", line 285, in wrapper\n", " yielded = next(result)\n", " File \"/home/shekhar/anaconda3/lib/python3.5/site-packages/distributed/nanny.py\", line 214, in instantiate\n", " self.process.start()\n", " File \"/home/shekhar/anaconda3/lib/python3.5/multiprocessing/process.py\", line 105, in start\n", " self._popen = self._Popen(self)\n", " File \"/home/shekhar/anaconda3/lib/python3.5/multiprocessing/context.py\", line 281, in _Popen\n", " return Popen(process_obj)\n", " File \"/home/shekhar/anaconda3/lib/python3.5/multiprocessing/popen_forkserver.py\", line 36, in __init__\n", " super().__init__(process_obj)\n", " File \"/home/shekhar/anaconda3/lib/python3.5/multiprocessing/popen_fork.py\", line 20, in __init__\n", " self._launch(process_obj)\n", " File \"/home/shekhar/anaconda3/lib/python3.5/multiprocessing/popen_forkserver.py\", line 52, in _launch\n", " self.sentinel, w = forkserver.connect_to_new_process(self._fds)\n", " File \"/home/shekhar/anaconda3/lib/python3.5/multiprocessing/forkserver.py\", line 66, in connect_to_new_process\n", " client.connect(self._forkserver_address)\n", "ConnectionRefusedError: [Errno 111] Connection refused\n", "tornado.application - ERROR - Exception in callback functools.partial(.null_wrapper at 0x7f2bf8070950>, )\n", "Traceback (most recent call last):\n", " File \"/home/shekhar/anaconda3/lib/python3.5/site-packages/tornado/ioloop.py\", line 604, in _run_callback\n", " ret = callback()\n", " File \"/home/shekhar/anaconda3/lib/python3.5/site-packages/tornado/stack_context.py\", line 275, in null_wrapper\n", " return fn(*args, **kwargs)\n", " File \"/home/shekhar/anaconda3/lib/python3.5/site-packages/tornado/ioloop.py\", line 619, in \n", " self.add_future(ret, lambda f: f.result())\n", " File \"/home/shekhar/anaconda3/lib/python3.5/site-packages/tornado/concurrent.py\", line 237, in result\n", " raise_exc_info(self._exc_info)\n", " File \"\", line 3, in raise_exc_info\n", " File \"/home/shekhar/anaconda3/lib/python3.5/site-packages/tornado/gen.py\", line 1021, in run\n", " yielded = self.gen.throw(*exc_info)\n", " File \"/home/shekhar/anaconda3/lib/python3.5/site-packages/distributed/nanny.py\", line 286, in _watch\n", " yield self.instantiate()\n", " File \"/home/shekhar/anaconda3/lib/python3.5/site-packages/tornado/gen.py\", line 1015, in run\n", " value = future.result()\n", " File \"/home/shekhar/anaconda3/lib/python3.5/site-packages/tornado/concurrent.py\", line 237, in result\n", " raise_exc_info(self._exc_info)\n", " File \"\", line 3, in raise_exc_info\n", " File \"/home/shekhar/anaconda3/lib/python3.5/site-packages/tornado/gen.py\", line 285, in wrapper\n", " yielded = next(result)\n", " File \"/home/shekhar/anaconda3/lib/python3.5/site-packages/distributed/nanny.py\", line 214, in instantiate\n", " self.process.start()\n", " File \"/home/shekhar/anaconda3/lib/python3.5/multiprocessing/process.py\", line 105, in start\n", " self._popen = self._Popen(self)\n", " File \"/home/shekhar/anaconda3/lib/python3.5/multiprocessing/context.py\", line 281, in _Popen\n", " return Popen(process_obj)\n", " File \"/home/shekhar/anaconda3/lib/python3.5/multiprocessing/popen_forkserver.py\", line 36, in __init__\n", " super().__init__(process_obj)\n", " File \"/home/shekhar/anaconda3/lib/python3.5/multiprocessing/popen_fork.py\", line 20, in __init__\n", " self._launch(process_obj)\n", " File \"/home/shekhar/anaconda3/lib/python3.5/multiprocessing/popen_forkserver.py\", line 52, in _launch\n", " self.sentinel, w = forkserver.connect_to_new_process(self._fds)\n", " File \"/home/shekhar/anaconda3/lib/python3.5/multiprocessing/forkserver.py\", line 66, in connect_to_new_process\n", " client.connect(self._forkserver_address)\n", "ConnectionRefusedError: [Errno 111] Connection refused\n", "tornado.application - ERROR - Exception in callback functools.partial(.null_wrapper at 0x7f2bf8077158>, )\n", "Traceback (most recent call last):\n", " File \"/home/shekhar/anaconda3/lib/python3.5/site-packages/tornado/ioloop.py\", line 604, in _run_callback\n", " ret = callback()\n", " File \"/home/shekhar/anaconda3/lib/python3.5/site-packages/tornado/stack_context.py\", line 275, in null_wrapper\n", " return fn(*args, **kwargs)\n", " File \"/home/shekhar/anaconda3/lib/python3.5/site-packages/tornado/ioloop.py\", line 619, in \n", " self.add_future(ret, lambda f: f.result())\n", " File \"/home/shekhar/anaconda3/lib/python3.5/site-packages/tornado/concurrent.py\", line 237, in result\n", " raise_exc_info(self._exc_info)\n", " File \"\", line 3, in raise_exc_info\n", " File \"/home/shekhar/anaconda3/lib/python3.5/site-packages/tornado/gen.py\", line 1021, in run\n", " yielded = self.gen.throw(*exc_info)\n", " File \"/home/shekhar/anaconda3/lib/python3.5/site-packages/distributed/nanny.py\", line 286, in _watch\n", " yield self.instantiate()\n", " File \"/home/shekhar/anaconda3/lib/python3.5/site-packages/tornado/gen.py\", line 1015, in run\n", " value = future.result()\n", " File \"/home/shekhar/anaconda3/lib/python3.5/site-packages/tornado/concurrent.py\", line 237, in result\n", " raise_exc_info(self._exc_info)\n", " File \"\", line 3, in raise_exc_info\n", " File \"/home/shekhar/anaconda3/lib/python3.5/site-packages/tornado/gen.py\", line 285, in wrapper\n", " yielded = next(result)\n", " File \"/home/shekhar/anaconda3/lib/python3.5/site-packages/distributed/nanny.py\", line 214, in instantiate\n", " self.process.start()\n", " File \"/home/shekhar/anaconda3/lib/python3.5/multiprocessing/process.py\", line 105, in start\n", " self._popen = self._Popen(self)\n", " File \"/home/shekhar/anaconda3/lib/python3.5/multiprocessing/context.py\", line 281, in _Popen\n", " return Popen(process_obj)\n", " File \"/home/shekhar/anaconda3/lib/python3.5/multiprocessing/popen_forkserver.py\", line 36, in __init__\n", " super().__init__(process_obj)\n", " File \"/home/shekhar/anaconda3/lib/python3.5/multiprocessing/popen_fork.py\", line 20, in __init__\n", " self._launch(process_obj)\n", " File \"/home/shekhar/anaconda3/lib/python3.5/multiprocessing/popen_forkserver.py\", line 52, in _launch\n", " self.sentinel, w = forkserver.connect_to_new_process(self._fds)\n", " File \"/home/shekhar/anaconda3/lib/python3.5/multiprocessing/forkserver.py\", line 66, in connect_to_new_process\n", " client.connect(self._forkserver_address)\n", "ConnectionRefusedError: [Errno 111] Connection refused\n", "tornado.application - ERROR - Exception in callback functools.partial(.null_wrapper at 0x7f2bf8077bf8>, )\n", "Traceback (most recent call last):\n", " File \"/home/shekhar/anaconda3/lib/python3.5/site-packages/tornado/ioloop.py\", line 604, in _run_callback\n", " ret = callback()\n", " File \"/home/shekhar/anaconda3/lib/python3.5/site-packages/tornado/stack_context.py\", line 275, in null_wrapper\n", " return fn(*args, **kwargs)\n", " File \"/home/shekhar/anaconda3/lib/python3.5/site-packages/tornado/ioloop.py\", line 619, in \n", " self.add_future(ret, lambda f: f.result())\n", " File \"/home/shekhar/anaconda3/lib/python3.5/site-packages/tornado/concurrent.py\", line 237, in result\n", " raise_exc_info(self._exc_info)\n", " File \"\", line 3, in raise_exc_info\n", " File \"/home/shekhar/anaconda3/lib/python3.5/site-packages/tornado/gen.py\", line 1021, in run\n", " yielded = self.gen.throw(*exc_info)\n", " File \"/home/shekhar/anaconda3/lib/python3.5/site-packages/distributed/nanny.py\", line 286, in _watch\n", " yield self.instantiate()\n", " File \"/home/shekhar/anaconda3/lib/python3.5/site-packages/tornado/gen.py\", line 1015, in run\n", " value = future.result()\n", " File \"/home/shekhar/anaconda3/lib/python3.5/site-packages/tornado/concurrent.py\", line 237, in result\n", " raise_exc_info(self._exc_info)\n", " File \"\", line 3, in raise_exc_info\n", " File \"/home/shekhar/anaconda3/lib/python3.5/site-packages/tornado/gen.py\", line 285, in wrapper\n", " yielded = next(result)\n", " File \"/home/shekhar/anaconda3/lib/python3.5/site-packages/distributed/nanny.py\", line 214, in instantiate\n", " self.process.start()\n", " File \"/home/shekhar/anaconda3/lib/python3.5/multiprocessing/process.py\", line 105, in start\n", " self._popen = self._Popen(self)\n", " File \"/home/shekhar/anaconda3/lib/python3.5/multiprocessing/context.py\", line 281, in _Popen\n", " return Popen(process_obj)\n", " File \"/home/shekhar/anaconda3/lib/python3.5/multiprocessing/popen_forkserver.py\", line 36, in __init__\n", " super().__init__(process_obj)\n", " File \"/home/shekhar/anaconda3/lib/python3.5/multiprocessing/popen_fork.py\", line 20, in __init__\n", " self._launch(process_obj)\n", " File \"/home/shekhar/anaconda3/lib/python3.5/multiprocessing/popen_forkserver.py\", line 52, in _launch\n", " self.sentinel, w = forkserver.connect_to_new_process(self._fds)\n", " File \"/home/shekhar/anaconda3/lib/python3.5/multiprocessing/forkserver.py\", line 66, in connect_to_new_process\n", " client.connect(self._forkserver_address)\n", "ConnectionRefusedError: [Errno 111] Connection refused\n" ] } ], "source": [ "k0 = s[(s.ca=='A002') & (s.unit=='R051') & (s.scp=='02-00-00')].compute()" ] }, { "cell_type": "code", "execution_count": 5, "metadata": { "collapsed": false }, "outputs": [ { "data": { "text/html": [ "
\n", "\n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", "
station
caunitscp
A002R05102-00-0015997
02-00-0116200
02-03-0015921
02-03-0116112
02-03-0215861
02-03-0315884
02-03-0415874
02-03-0516027
02-03-0615908
02-05-0015815
02-05-0115713
02-06-0016140
A006R07900-00-0016300
00-00-0116078
00-00-0216223
00-00-0316044
00-00-0416033
00-03-0016058
00-03-0116089
00-03-0216069
A007R07901-05-0015752
01-05-0115725
01-06-0016152
01-06-0116126
01-06-0216121
01-06-0316363
A010R08000-00-0017654
00-00-0117187
00-00-0217117
00-00-0316938
............
PTH22R54002-02-01291
02-02-02288
02-02-03292
02-02-04291
02-02-05293
02-02-06292
02-02-07291
02-03-00291
02-03-01291
02-03-02284
02-03-03292
02-03-04291
02-03-05291
02-03-06290
02-03-07284
02-04-00291
02-04-02258
02-04-03257
02-04-04257
02-04-05256
02-04-06257
02-04-07257
02-05-00257
02-05-01257
02-05-02256
02-05-03258
02-05-05250
TRAM1R46800-03-003
00-03-013
00-05-013
\n", "

5217 rows × 1 columns

\n", "
" ], "text/plain": [ " station\n", "ca unit scp \n", "A002 R051 02-00-00 15997\n", " 02-00-01 16200\n", " 02-03-00 15921\n", " 02-03-01 16112\n", " 02-03-02 15861\n", " 02-03-03 15884\n", " 02-03-04 15874\n", " 02-03-05 16027\n", " 02-03-06 15908\n", " 02-05-00 15815\n", " 02-05-01 15713\n", " 02-06-00 16140\n", "A006 R079 00-00-00 16300\n", " 00-00-01 16078\n", " 00-00-02 16223\n", " 00-00-03 16044\n", " 00-00-04 16033\n", " 00-03-00 16058\n", " 00-03-01 16089\n", " 00-03-02 16069\n", "A007 R079 01-05-00 15752\n", " 01-05-01 15725\n", " 01-06-00 16152\n", " 01-06-01 16126\n", " 01-06-02 16121\n", " 01-06-03 16363\n", "A010 R080 00-00-00 17654\n", " 00-00-01 17187\n", " 00-00-02 17117\n", " 00-00-03 16938\n", "... ...\n", "PTH22 R540 02-02-01 291\n", " 02-02-02 288\n", " 02-02-03 292\n", " 02-02-04 291\n", " 02-02-05 293\n", " 02-02-06 292\n", " 02-02-07 291\n", " 02-03-00 291\n", " 02-03-01 291\n", " 02-03-02 284\n", " 02-03-03 292\n", " 02-03-04 291\n", " 02-03-05 291\n", " 02-03-06 290\n", " 02-03-07 284\n", " 02-04-00 291\n", " 02-04-02 258\n", " 02-04-03 257\n", " 02-04-04 257\n", " 02-04-05 256\n", " 02-04-06 257\n", " 02-04-07 257\n", " 02-05-00 257\n", " 02-05-01 257\n", " 02-05-02 256\n", " 02-05-03 258\n", " 02-05-05 250\n", "TRAM1 R468 00-03-00 3\n", " 00-03-01 3\n", " 00-05-01 3\n", "\n", "[5217 rows x 1 columns]" ] }, "execution_count": 5, "metadata": {}, "output_type": "execute_result" } ], "source": [ "zz" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "collapsed": true }, "outputs": [], "source": [] } ], "metadata": { "anaconda-cloud": {}, "kernelspec": { "display_name": "Python [default]", "language": "python", "name": "python3" }, "language_info": { "codemirror_mode": { "name": "ipython", "version": 3 }, "file_extension": ".py", "mimetype": "text/x-python", "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", "version": "3.5.2" } }, "nbformat": 4, "nbformat_minor": 1 }