{ "cells": [ { "cell_type": "code", "execution_count": 1, "metadata": { "ExecuteTime": { "end_time": "2018-10-11T12:55:30.952211Z", "start_time": "2018-10-11T12:55:28.742550Z" } }, "outputs": [ { "ename": "ValueError", "evalue": "Mismatched versions found\n\nblosc\n+-------------------------+---------+\n| | version |\n+-------------------------+---------+\n| client | None |\n| scheduler | 1.5.1 |\n| tcp://10.244.0.27:42537 | 1.5.1 |\n| tcp://10.244.1.25:46549 | 1.5.1 |\n| tcp://10.244.2.23:40885 | 1.5.1 |\n| tcp://10.244.3.8:33531 | 1.5.1 |\n| tcp://10.244.4.8:43383 | 1.5.1 |\n| tcp://10.244.5.7:41665 | 1.5.1 |\n| tcp://10.244.6.5:45879 | 1.5.1 |\n| tcp://10.244.7.7:37667 | 1.5.1 |\n+-------------------------+---------+", "output_type": "error", "traceback": [ "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m", "\u001b[0;31mValueError\u001b[0m Traceback (most recent call last)", "\u001b[0;32m\u001b[0m in \u001b[0;36m\u001b[0;34m\u001b[0m\n\u001b[1;32m 1\u001b[0m \u001b[0;32mfrom\u001b[0m \u001b[0mdask\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mdistributed\u001b[0m \u001b[0;32mimport\u001b[0m \u001b[0mClient\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mconfig\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 2\u001b[0m \u001b[0mclient\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mClient\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m\"dask.azure.farsetlabs.org.uk:8786\"\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m----> 3\u001b[0;31m \u001b[0mclient\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mget_versions\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mcheck\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;32mTrue\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m", "\u001b[0;32m~/anaconda3/envs/dask_demo/lib/python3.6/site-packages/distributed/client.py\u001b[0m in \u001b[0;36mget_versions\u001b[0;34m(self, check)\u001b[0m\n\u001b[1;32m 3064\u001b[0m raise ValueError(\"Mismatched versions found\\n\"\n\u001b[1;32m 3065\u001b[0m \u001b[0;34m\"\\n\"\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m-> 3066\u001b[0;31m \"%s\" % ('\\n\\n'.join(errs)))\n\u001b[0m\u001b[1;32m 3067\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 3068\u001b[0m \u001b[0;32mreturn\u001b[0m \u001b[0mresult\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n", "\u001b[0;31mValueError\u001b[0m: Mismatched versions found\n\nblosc\n+-------------------------+---------+\n| | version |\n+-------------------------+---------+\n| client | None |\n| scheduler | 1.5.1 |\n| tcp://10.244.0.27:42537 | 1.5.1 |\n| tcp://10.244.1.25:46549 | 1.5.1 |\n| tcp://10.244.2.23:40885 | 1.5.1 |\n| tcp://10.244.3.8:33531 | 1.5.1 |\n| tcp://10.244.4.8:43383 | 1.5.1 |\n| tcp://10.244.5.7:41665 | 1.5.1 |\n| tcp://10.244.6.5:45879 | 1.5.1 |\n| tcp://10.244.7.7:37667 | 1.5.1 |\n+-------------------------+---------+" ] } ], "source": [ "from dask.distributed import Client, config\n", "client = Client(\"dask.azure.farsetlabs.org.uk:8786\")\n", "client.get_versions(check=True)" ] }, { "cell_type": "code", "execution_count": 8, "metadata": { "ExecuteTime": { "end_time": "2018-10-11T12:59:13.220734Z", "start_time": "2018-10-11T12:59:13.164522Z" } }, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "tcp://10.244.0.27:42537 INFO:distributed.worker - INFO - Start worker at: tcp://10.244.0.27:42537\n", "tcp://10.244.0.27:42537 INFO:distributed.worker - INFO - Listening to: tcp://10.244.0.27:42537\n", "tcp://10.244.0.27:42537 INFO:distributed.worker - INFO - nanny at: 10.244.0.27:41271\n", "tcp://10.244.0.27:42537 INFO:distributed.worker - INFO - Waiting to connect to: tcp://donating-jaguar-dask-scheduler:8786\n", "tcp://10.244.0.27:42537 INFO:distributed.worker - INFO - -------------------------------------------------\n", "tcp://10.244.0.27:42537 INFO:distributed.worker - INFO - Threads: 2\n", "tcp://10.244.0.27:42537 INFO:distributed.worker - INFO - Memory: 8.37 GB\n", "tcp://10.244.0.27:42537 INFO:distributed.worker - INFO - Local Directory: /dask-worker-space/worker-v8i3uzlp\n", "tcp://10.244.0.27:42537 INFO:distributed.worker - INFO - -------------------------------------------------\n", "tcp://10.244.0.27:42537 INFO:distributed.worker - INFO - Waiting to connect to: tcp://donating-jaguar-dask-scheduler:8786\n", "tcp://10.244.0.27:42537 INFO:distributed.worker - INFO - Waiting to connect to: tcp://donating-jaguar-dask-scheduler:8786\n", "tcp://10.244.0.27:42537 INFO:distributed.worker - INFO - Waiting to connect to: tcp://donating-jaguar-dask-scheduler:8786\n", "tcp://10.244.0.27:42537 INFO:distributed.worker - INFO - Waiting to connect to: tcp://donating-jaguar-dask-scheduler:8786\n", "tcp://10.244.0.27:42537 INFO:distributed.worker - INFO - Waiting to connect to: tcp://donating-jaguar-dask-scheduler:8786\n", "tcp://10.244.0.27:42537 INFO:distributed.worker - INFO - Waiting to connect to: tcp://donating-jaguar-dask-scheduler:8786\n", "tcp://10.244.0.27:42537 INFO:distributed.worker - INFO - Waiting to connect to: tcp://donating-jaguar-dask-scheduler:8786\n", "tcp://10.244.0.27:42537 INFO:distributed.worker - INFO - Waiting to connect to: tcp://donating-jaguar-dask-scheduler:8786\n", "tcp://10.244.0.27:42537 INFO:distributed.worker - INFO - Waiting to connect to: tcp://donating-jaguar-dask-scheduler:8786\n", "tcp://10.244.0.27:42537 INFO:distributed.worker - INFO - Waiting to connect to: tcp://donating-jaguar-dask-scheduler:8786\n", "tcp://10.244.0.27:42537 INFO:distributed.worker - INFO - Waiting to connect to: tcp://donating-jaguar-dask-scheduler:8786\n", "tcp://10.244.0.27:42537 INFO:distributed.worker - INFO - Waiting to connect to: tcp://donating-jaguar-dask-scheduler:8786\n", "tcp://10.244.0.27:42537 INFO:distributed.worker - INFO - Waiting to connect to: tcp://donating-jaguar-dask-scheduler:8786\n", "tcp://10.244.0.27:42537 INFO:distributed.worker - INFO - Waiting to connect to: tcp://donating-jaguar-dask-scheduler:8786\n", "tcp://10.244.0.27:42537 INFO:distributed.worker - INFO - Waiting to connect to: tcp://donating-jaguar-dask-scheduler:8786\n", "tcp://10.244.0.27:42537 INFO:distributed.worker - INFO - Waiting to connect to: tcp://donating-jaguar-dask-scheduler:8786\n", "tcp://10.244.0.27:42537 INFO:distributed.worker - INFO - Waiting to connect to: tcp://donating-jaguar-dask-scheduler:8786\n", "tcp://10.244.0.27:42537 INFO:distributed.worker - INFO - Waiting to connect to: tcp://donating-jaguar-dask-scheduler:8786\n", "tcp://10.244.0.27:42537 INFO:distributed.worker - INFO - Registered to: tcp://donating-jaguar-dask-scheduler:8786\n", "tcp://10.244.0.27:42537 INFO:distributed.worker - INFO - -------------------------------------------------\n", "tcp://10.244.1.25:46549 INFO:distributed.worker - INFO - Start worker at: tcp://10.244.1.25:46549\n", "tcp://10.244.1.25:46549 INFO:distributed.worker - INFO - Listening to: tcp://10.244.1.25:46549\n", "tcp://10.244.1.25:46549 INFO:distributed.worker - INFO - nanny at: 10.244.1.25:36677\n", "tcp://10.244.1.25:46549 INFO:distributed.worker - INFO - Waiting to connect to: tcp://donating-jaguar-dask-scheduler:8786\n", "tcp://10.244.1.25:46549 INFO:distributed.worker - INFO - -------------------------------------------------\n", "tcp://10.244.1.25:46549 INFO:distributed.worker - INFO - Threads: 2\n", "tcp://10.244.1.25:46549 INFO:distributed.worker - INFO - Memory: 8.37 GB\n", "tcp://10.244.1.25:46549 INFO:distributed.worker - INFO - Local Directory: /dask-worker-space/worker-c_zxst6o\n", "tcp://10.244.1.25:46549 INFO:distributed.worker - INFO - -------------------------------------------------\n", "tcp://10.244.1.25:46549 INFO:distributed.worker - INFO - Waiting to connect to: tcp://donating-jaguar-dask-scheduler:8786\n", "tcp://10.244.1.25:46549 INFO:distributed.worker - INFO - Waiting to connect to: tcp://donating-jaguar-dask-scheduler:8786\n", "tcp://10.244.1.25:46549 INFO:distributed.worker - INFO - Waiting to connect to: tcp://donating-jaguar-dask-scheduler:8786\n", "tcp://10.244.1.25:46549 INFO:distributed.worker - INFO - Waiting to connect to: tcp://donating-jaguar-dask-scheduler:8786\n", "tcp://10.244.1.25:46549 INFO:distributed.worker - INFO - Waiting to connect to: tcp://donating-jaguar-dask-scheduler:8786\n", "tcp://10.244.1.25:46549 INFO:distributed.worker - INFO - Waiting to connect to: tcp://donating-jaguar-dask-scheduler:8786\n", "tcp://10.244.1.25:46549 INFO:distributed.worker - INFO - Waiting to connect to: tcp://donating-jaguar-dask-scheduler:8786\n", "tcp://10.244.1.25:46549 INFO:distributed.worker - INFO - Waiting to connect to: tcp://donating-jaguar-dask-scheduler:8786\n", "tcp://10.244.1.25:46549 INFO:distributed.worker - INFO - Waiting to connect to: tcp://donating-jaguar-dask-scheduler:8786\n", "tcp://10.244.1.25:46549 INFO:distributed.worker - INFO - Waiting to connect to: tcp://donating-jaguar-dask-scheduler:8786\n", "tcp://10.244.1.25:46549 INFO:distributed.worker - INFO - Waiting to connect to: tcp://donating-jaguar-dask-scheduler:8786\n", "tcp://10.244.1.25:46549 INFO:distributed.worker - INFO - Waiting to connect to: tcp://donating-jaguar-dask-scheduler:8786\n", "tcp://10.244.1.25:46549 INFO:distributed.worker - INFO - Waiting to connect to: tcp://donating-jaguar-dask-scheduler:8786\n", "tcp://10.244.1.25:46549 INFO:distributed.worker - INFO - Waiting to connect to: tcp://donating-jaguar-dask-scheduler:8786\n", "tcp://10.244.1.25:46549 INFO:distributed.worker - INFO - Waiting to connect to: tcp://donating-jaguar-dask-scheduler:8786\n", "tcp://10.244.1.25:46549 INFO:distributed.worker - INFO - Waiting to connect to: tcp://donating-jaguar-dask-scheduler:8786\n", "tcp://10.244.1.25:46549 INFO:distributed.worker - INFO - Waiting to connect to: tcp://donating-jaguar-dask-scheduler:8786\n", "tcp://10.244.1.25:46549 INFO:distributed.worker - INFO - Waiting to connect to: tcp://donating-jaguar-dask-scheduler:8786\n", "tcp://10.244.1.25:46549 INFO:distributed.worker - INFO - Waiting to connect to: tcp://donating-jaguar-dask-scheduler:8786\n", "tcp://10.244.1.25:46549 INFO:distributed.worker - INFO - Waiting to connect to: tcp://donating-jaguar-dask-scheduler:8786\n", "tcp://10.244.1.25:46549 INFO:distributed.worker - INFO - Registered to: tcp://donating-jaguar-dask-scheduler:8786\n", "tcp://10.244.1.25:46549 INFO:distributed.worker - INFO - -------------------------------------------------\n", "tcp://10.244.2.23:40885 INFO:distributed.worker - INFO - Start worker at: tcp://10.244.2.23:40885\n", "tcp://10.244.2.23:40885 INFO:distributed.worker - INFO - Listening to: tcp://10.244.2.23:40885\n", "tcp://10.244.2.23:40885 INFO:distributed.worker - INFO - nanny at: 10.244.2.23:35197\n", "tcp://10.244.2.23:40885 INFO:distributed.worker - INFO - Waiting to connect to: tcp://donating-jaguar-dask-scheduler:8786\n", "tcp://10.244.2.23:40885 INFO:distributed.worker - INFO - -------------------------------------------------\n", "tcp://10.244.2.23:40885 INFO:distributed.worker - INFO - Threads: 2\n", "tcp://10.244.2.23:40885 INFO:distributed.worker - INFO - Memory: 8.37 GB\n", "tcp://10.244.2.23:40885 INFO:distributed.worker - INFO - Local Directory: /dask-worker-space/worker-j_1utjqf\n", "tcp://10.244.2.23:40885 INFO:distributed.worker - INFO - -------------------------------------------------\n", "tcp://10.244.2.23:40885 INFO:distributed.worker - INFO - Waiting to connect to: tcp://donating-jaguar-dask-scheduler:8786\n", "tcp://10.244.2.23:40885 INFO:distributed.worker - INFO - Waiting to connect to: tcp://donating-jaguar-dask-scheduler:8786\n", "tcp://10.244.2.23:40885 INFO:distributed.worker - INFO - Waiting to connect to: tcp://donating-jaguar-dask-scheduler:8786\n", "tcp://10.244.2.23:40885 INFO:distributed.worker - INFO - Waiting to connect to: tcp://donating-jaguar-dask-scheduler:8786\n", "tcp://10.244.2.23:40885 INFO:distributed.worker - INFO - Waiting to connect to: tcp://donating-jaguar-dask-scheduler:8786\n", "tcp://10.244.2.23:40885 INFO:distributed.worker - INFO - Waiting to connect to: tcp://donating-jaguar-dask-scheduler:8786\n", "tcp://10.244.2.23:40885 INFO:distributed.worker - INFO - Waiting to connect to: tcp://donating-jaguar-dask-scheduler:8786\n", "tcp://10.244.2.23:40885 INFO:distributed.worker - INFO - Waiting to connect to: tcp://donating-jaguar-dask-scheduler:8786\n", "tcp://10.244.2.23:40885 INFO:distributed.worker - INFO - Waiting to connect to: tcp://donating-jaguar-dask-scheduler:8786\n", "tcp://10.244.2.23:40885 INFO:distributed.worker - INFO - Waiting to connect to: tcp://donating-jaguar-dask-scheduler:8786\n", "tcp://10.244.2.23:40885 INFO:distributed.worker - INFO - Waiting to connect to: tcp://donating-jaguar-dask-scheduler:8786\n", "tcp://10.244.2.23:40885 INFO:distributed.worker - INFO - Waiting to connect to: tcp://donating-jaguar-dask-scheduler:8786\n", "tcp://10.244.2.23:40885 INFO:distributed.worker - INFO - Waiting to connect to: tcp://donating-jaguar-dask-scheduler:8786\n", "tcp://10.244.2.23:40885 INFO:distributed.worker - INFO - Waiting to connect to: tcp://donating-jaguar-dask-scheduler:8786\n", "tcp://10.244.2.23:40885 INFO:distributed.worker - INFO - Waiting to connect to: tcp://donating-jaguar-dask-scheduler:8786\n", "tcp://10.244.2.23:40885 INFO:distributed.worker - INFO - Waiting to connect to: tcp://donating-jaguar-dask-scheduler:8786\n", "tcp://10.244.2.23:40885 INFO:distributed.worker - INFO - Waiting to connect to: tcp://donating-jaguar-dask-scheduler:8786\n", "tcp://10.244.2.23:40885 INFO:distributed.worker - INFO - Registered to: tcp://donating-jaguar-dask-scheduler:8786\n", "tcp://10.244.2.23:40885 INFO:distributed.worker - INFO - -------------------------------------------------\n", "tcp://10.244.3.8:33531 INFO:distributed.worker - INFO - Start worker at: tcp://10.244.3.8:33531\n", "tcp://10.244.3.8:33531 INFO:distributed.worker - INFO - Listening to: tcp://10.244.3.8:33531\n", "tcp://10.244.3.8:33531 INFO:distributed.worker - INFO - nanny at: 10.244.3.8:34607\n", "tcp://10.244.3.8:33531 INFO:distributed.worker - INFO - Waiting to connect to: tcp://donating-jaguar-dask-scheduler:8786\n", "tcp://10.244.3.8:33531 INFO:distributed.worker - INFO - -------------------------------------------------\n", "tcp://10.244.3.8:33531 INFO:distributed.worker - INFO - Threads: 2\n", "tcp://10.244.3.8:33531 INFO:distributed.worker - INFO - Memory: 8.37 GB\n", "tcp://10.244.3.8:33531 INFO:distributed.worker - INFO - Local Directory: /dask-worker-space/worker-cy1si9td\n", "tcp://10.244.3.8:33531 INFO:distributed.worker - INFO - -------------------------------------------------\n", "tcp://10.244.3.8:33531 INFO:distributed.worker - INFO - Waiting to connect to: tcp://donating-jaguar-dask-scheduler:8786\n", "tcp://10.244.3.8:33531 INFO:distributed.worker - INFO - Waiting to connect to: tcp://donating-jaguar-dask-scheduler:8786\n", "tcp://10.244.3.8:33531 INFO:distributed.worker - INFO - Waiting to connect to: tcp://donating-jaguar-dask-scheduler:8786\n", "tcp://10.244.3.8:33531 INFO:distributed.worker - INFO - Waiting to connect to: tcp://donating-jaguar-dask-scheduler:8786\n", "tcp://10.244.3.8:33531 INFO:distributed.worker - INFO - Waiting to connect to: tcp://donating-jaguar-dask-scheduler:8786\n", "tcp://10.244.3.8:33531 INFO:distributed.worker - INFO - Waiting to connect to: tcp://donating-jaguar-dask-scheduler:8786\n", "tcp://10.244.3.8:33531 INFO:distributed.worker - INFO - Waiting to connect to: tcp://donating-jaguar-dask-scheduler:8786\n", "tcp://10.244.3.8:33531 INFO:distributed.worker - INFO - Waiting to connect to: tcp://donating-jaguar-dask-scheduler:8786\n", "tcp://10.244.3.8:33531 INFO:distributed.worker - INFO - Waiting to connect to: tcp://donating-jaguar-dask-scheduler:8786\n", "tcp://10.244.3.8:33531 INFO:distributed.worker - INFO - Waiting to connect to: tcp://donating-jaguar-dask-scheduler:8786\n", "tcp://10.244.3.8:33531 INFO:distributed.worker - INFO - Waiting to connect to: tcp://donating-jaguar-dask-scheduler:8786\n", "tcp://10.244.3.8:33531 INFO:distributed.worker - INFO - Waiting to connect to: tcp://donating-jaguar-dask-scheduler:8786\n", "tcp://10.244.3.8:33531 INFO:distributed.worker - INFO - Waiting to connect to: tcp://donating-jaguar-dask-scheduler:8786\n", "tcp://10.244.3.8:33531 INFO:distributed.worker - INFO - Waiting to connect to: tcp://donating-jaguar-dask-scheduler:8786\n", "tcp://10.244.3.8:33531 INFO:distributed.worker - INFO - Waiting to connect to: tcp://donating-jaguar-dask-scheduler:8786\n", "tcp://10.244.3.8:33531 INFO:distributed.worker - INFO - Waiting to connect to: tcp://donating-jaguar-dask-scheduler:8786\n", "tcp://10.244.3.8:33531 INFO:distributed.worker - INFO - Waiting to connect to: tcp://donating-jaguar-dask-scheduler:8786\n", "tcp://10.244.3.8:33531 INFO:distributed.worker - INFO - Waiting to connect to: tcp://donating-jaguar-dask-scheduler:8786\n", "tcp://10.244.3.8:33531 INFO:distributed.worker - INFO - Waiting to connect to: tcp://donating-jaguar-dask-scheduler:8786\n", "tcp://10.244.3.8:33531 INFO:distributed.worker - INFO - Waiting to connect to: tcp://donating-jaguar-dask-scheduler:8786\n", "tcp://10.244.3.8:33531 INFO:distributed.worker - INFO - Registered to: tcp://donating-jaguar-dask-scheduler:8786\n", "tcp://10.244.3.8:33531 INFO:distributed.worker - INFO - -------------------------------------------------\n", "tcp://10.244.4.8:43383 INFO:distributed.worker - INFO - Start worker at: tcp://10.244.4.8:43383\n", "tcp://10.244.4.8:43383 INFO:distributed.worker - INFO - Listening to: tcp://10.244.4.8:43383\n", "tcp://10.244.4.8:43383 INFO:distributed.worker - INFO - nanny at: 10.244.4.8:41207\n", "tcp://10.244.4.8:43383 INFO:distributed.worker - INFO - Waiting to connect to: tcp://donating-jaguar-dask-scheduler:8786\n", "tcp://10.244.4.8:43383 INFO:distributed.worker - INFO - -------------------------------------------------\n", "tcp://10.244.4.8:43383 INFO:distributed.worker - INFO - Threads: 2\n", "tcp://10.244.4.8:43383 INFO:distributed.worker - INFO - Memory: 8.37 GB\n", "tcp://10.244.4.8:43383 INFO:distributed.worker - INFO - Local Directory: /dask-worker-space/worker-b5h7qjjv\n", "tcp://10.244.4.8:43383 INFO:distributed.worker - INFO - -------------------------------------------------\n", "tcp://10.244.4.8:43383 INFO:distributed.worker - INFO - Waiting to connect to: tcp://donating-jaguar-dask-scheduler:8786\n", "tcp://10.244.4.8:43383 INFO:distributed.worker - INFO - Waiting to connect to: tcp://donating-jaguar-dask-scheduler:8786\n", "tcp://10.244.4.8:43383 INFO:distributed.worker - INFO - Waiting to connect to: tcp://donating-jaguar-dask-scheduler:8786\n", "tcp://10.244.4.8:43383 INFO:distributed.worker - INFO - Waiting to connect to: tcp://donating-jaguar-dask-scheduler:8786\n", "tcp://10.244.4.8:43383 INFO:distributed.worker - INFO - Waiting to connect to: tcp://donating-jaguar-dask-scheduler:8786\n", "tcp://10.244.4.8:43383 INFO:distributed.worker - INFO - Waiting to connect to: tcp://donating-jaguar-dask-scheduler:8786\n", "tcp://10.244.4.8:43383 INFO:distributed.worker - INFO - Waiting to connect to: tcp://donating-jaguar-dask-scheduler:8786\n", "tcp://10.244.4.8:43383 INFO:distributed.worker - INFO - Waiting to connect to: tcp://donating-jaguar-dask-scheduler:8786\n", "tcp://10.244.4.8:43383 INFO:distributed.worker - INFO - Waiting to connect to: tcp://donating-jaguar-dask-scheduler:8786\n", "tcp://10.244.4.8:43383 INFO:distributed.worker - INFO - Waiting to connect to: tcp://donating-jaguar-dask-scheduler:8786\n", "tcp://10.244.4.8:43383 INFO:distributed.worker - INFO - Waiting to connect to: tcp://donating-jaguar-dask-scheduler:8786\n", "tcp://10.244.4.8:43383 INFO:distributed.worker - INFO - Waiting to connect to: tcp://donating-jaguar-dask-scheduler:8786\n", "tcp://10.244.4.8:43383 INFO:distributed.worker - INFO - Waiting to connect to: tcp://donating-jaguar-dask-scheduler:8786\n", "tcp://10.244.4.8:43383 INFO:distributed.worker - INFO - Waiting to connect to: tcp://donating-jaguar-dask-scheduler:8786\n", "tcp://10.244.4.8:43383 INFO:distributed.worker - INFO - Waiting to connect to: tcp://donating-jaguar-dask-scheduler:8786\n", "tcp://10.244.4.8:43383 INFO:distributed.worker - INFO - Waiting to connect to: tcp://donating-jaguar-dask-scheduler:8786\n", "tcp://10.244.4.8:43383 INFO:distributed.worker - INFO - Waiting to connect to: tcp://donating-jaguar-dask-scheduler:8786\n", "tcp://10.244.4.8:43383 INFO:distributed.worker - INFO - Waiting to connect to: tcp://donating-jaguar-dask-scheduler:8786\n", "tcp://10.244.4.8:43383 INFO:distributed.worker - INFO - Waiting to connect to: tcp://donating-jaguar-dask-scheduler:8786\n", "tcp://10.244.4.8:43383 INFO:distributed.worker - INFO - Waiting to connect to: tcp://donating-jaguar-dask-scheduler:8786\n", "tcp://10.244.4.8:43383 INFO:distributed.worker - INFO - Waiting to connect to: tcp://donating-jaguar-dask-scheduler:8786\n", "tcp://10.244.4.8:43383 INFO:distributed.worker - INFO - Waiting to connect to: tcp://donating-jaguar-dask-scheduler:8786\n", "tcp://10.244.4.8:43383 INFO:distributed.worker - INFO - Waiting to connect to: tcp://donating-jaguar-dask-scheduler:8786\n", "tcp://10.244.4.8:43383 INFO:distributed.worker - INFO - Waiting to connect to: tcp://donating-jaguar-dask-scheduler:8786\n", "tcp://10.244.4.8:43383 INFO:distributed.worker - INFO - Waiting to connect to: tcp://donating-jaguar-dask-scheduler:8786\n", "tcp://10.244.4.8:43383 INFO:distributed.worker - INFO - Registered to: tcp://donating-jaguar-dask-scheduler:8786\n", "tcp://10.244.4.8:43383 INFO:distributed.worker - INFO - -------------------------------------------------\n", "tcp://10.244.4.8:43383 ERROR:distributed.worker - ERROR - No module named 'matplotlib'\n", "Traceback (most recent call last):\n", " File \"/opt/conda/lib/python3.6/site-packages/distributed/worker.py\", line 346, in handle_scheduler\n", " self.ensure_computing])\n", " File \"/opt/conda/lib/python3.6/site-packages/tornado/gen.py\", line 1099, in run\n", " value = future.result()\n", " File \"/opt/conda/lib/python3.6/site-packages/tornado/gen.py\", line 1107, in run\n", " yielded = self.gen.throw(*exc_info)\n", " File \"/opt/conda/lib/python3.6/site-packages/distributed/core.py\", line 361, in handle_stream\n", " msgs = yield comm.read()\n", " File \"/opt/conda/lib/python3.6/site-packages/tornado/gen.py\", line 1099, in run\n", " value = future.result()\n", " File \"/opt/conda/lib/python3.6/site-packages/tornado/gen.py\", line 1107, in run\n", " yielded = self.gen.throw(*exc_info)\n", " File \"/opt/conda/lib/python3.6/site-packages/distributed/comm/tcp.py\", line 203, in read\n", " deserializers=deserializers)\n", " File \"/opt/conda/lib/python3.6/site-packages/tornado/gen.py\", line 1099, in run\n", " value = future.result()\n", " File \"/opt/conda/lib/python3.6/site-packages/tornado/gen.py\", line 315, in wrapper\n", " yielded = next(result)\n", " File \"/opt/conda/lib/python3.6/site-packages/distributed/comm/utils.py\", line 79, in from_frames\n", " res = _from_frames()\n", " File \"/opt/conda/lib/python3.6/site-packages/distributed/comm/utils.py\", line 65, in _from_frames\n", " deserializers=deserializers)\n", " File \"/opt/conda/lib/python3.6/site-packages/distributed/protocol/core.py\", line 122, in loads\n", " value = _deserialize(head, fs, deserializers=deserializers)\n", " File \"/opt/conda/lib/python3.6/site-packages/distributed/protocol/serialize.py\", line 236, in deserialize\n", " return loads(header, frames)\n", " File \"/opt/conda/lib/python3.6/site-packages/distributed/protocol/serialize.py\", line 58, in pickle_loads\n", " return pickle.loads(b''.join(frames))\n", " File \"/opt/conda/lib/python3.6/site-packages/distributed/protocol/pickle.py\", line 59, in loads\n", " return pickle.loads(x)\n", " File \"/opt/conda/lib/python3.6/site-packages/cloudpickle/cloudpickle.py\", line 908, in subimport\n", " __import__(name)\n", "ModuleNotFoundError: No module named 'matplotlib'\n", "tcp://10.244.4.8:43383 INFO:distributed.worker - INFO - Connection to scheduler broken. Reconnecting...\n", "tcp://10.244.4.8:43383 INFO:distributed.worker - INFO - -------------------------------------------------\n", "tcp://10.244.4.8:43383 INFO:distributed.worker - INFO - Registered to: tcp://donating-jaguar-dask-scheduler:8786\n", "tcp://10.244.4.8:43383 INFO:distributed.worker - INFO - -------------------------------------------------\n", "tcp://10.244.4.8:43383 ERROR:distributed.worker - ERROR - No module named 'matplotlib'\n", "Traceback (most recent call last):\n", " File \"/opt/conda/lib/python3.6/site-packages/distributed/worker.py\", line 346, in handle_scheduler\n", " self.ensure_computing])\n", " File \"/opt/conda/lib/python3.6/site-packages/tornado/gen.py\", line 1099, in run\n", " value = future.result()\n", " File \"/opt/conda/lib/python3.6/site-packages/tornado/gen.py\", line 1107, in run\n", " yielded = self.gen.throw(*exc_info)\n", " File \"/opt/conda/lib/python3.6/site-packages/distributed/core.py\", line 361, in handle_stream\n", " msgs = yield comm.read()\n", " File \"/opt/conda/lib/python3.6/site-packages/tornado/gen.py\", line 1099, in run\n", " value = future.result()\n", " File \"/opt/conda/lib/python3.6/site-packages/tornado/gen.py\", line 1107, in run\n", " yielded = self.gen.throw(*exc_info)\n", " File \"/opt/conda/lib/python3.6/site-packages/distributed/comm/tcp.py\", line 203, in read\n", " deserializers=deserializers)\n", " File \"/opt/conda/lib/python3.6/site-packages/tornado/gen.py\", line 1099, in run\n", " value = future.result()\n", " File \"/opt/conda/lib/python3.6/site-packages/tornado/gen.py\", line 315, in wrapper\n", " yielded = next(result)\n", " File \"/opt/conda/lib/python3.6/site-packages/distributed/comm/utils.py\", line 79, in from_frames\n", " res = _from_frames()\n", " File \"/opt/conda/lib/python3.6/site-packages/distributed/comm/utils.py\", line 65, in _from_frames\n", " deserializers=deserializers)\n", " File \"/opt/conda/lib/python3.6/site-packages/distributed/protocol/core.py\", line 122, in loads\n", " value = _deserialize(head, fs, deserializers=deserializers)\n", " File \"/opt/conda/lib/python3.6/site-packages/distributed/protocol/serialize.py\", line 236, in deserialize\n", " return loads(header, frames)\n", " File \"/opt/conda/lib/python3.6/site-packages/distributed/protocol/serialize.py\", line 58, in pickle_loads\n", " return pickle.loads(b''.join(frames))\n", " File \"/opt/conda/lib/python3.6/site-packages/distributed/protocol/pickle.py\", line 59, in loads\n", " return pickle.loads(x)\n", " File \"/opt/conda/lib/python3.6/site-packages/cloudpickle/cloudpickle.py\", line 908, in subimport\n", " __import__(name)\n", "ModuleNotFoundError: No module named 'matplotlib'\n", "tcp://10.244.4.8:43383 INFO:distributed.worker - INFO - Connection to scheduler broken. Reconnecting...\n", "tcp://10.244.4.8:43383 INFO:distributed.worker - INFO - -------------------------------------------------\n", "tcp://10.244.4.8:43383 INFO:distributed.worker - INFO - Registered to: tcp://donating-jaguar-dask-scheduler:8786\n", "tcp://10.244.4.8:43383 INFO:distributed.worker - INFO - -------------------------------------------------\n", "tcp://10.244.5.7:41665 INFO:distributed.worker - INFO - Start worker at: tcp://10.244.5.7:41665\n", "tcp://10.244.5.7:41665 INFO:distributed.worker - INFO - Listening to: tcp://10.244.5.7:41665\n", "tcp://10.244.5.7:41665 INFO:distributed.worker - INFO - nanny at: 10.244.5.7:36027\n", "tcp://10.244.5.7:41665 INFO:distributed.worker - INFO - Waiting to connect to: tcp://donating-jaguar-dask-scheduler:8786\n", "tcp://10.244.5.7:41665 INFO:distributed.worker - INFO - -------------------------------------------------\n", "tcp://10.244.5.7:41665 INFO:distributed.worker - INFO - Threads: 2\n", "tcp://10.244.5.7:41665 INFO:distributed.worker - INFO - Memory: 8.37 GB\n", "tcp://10.244.5.7:41665 INFO:distributed.worker - INFO - Local Directory: /dask-worker-space/worker-5do2tqdo\n", "tcp://10.244.5.7:41665 INFO:distributed.worker - INFO - -------------------------------------------------\n", "tcp://10.244.5.7:41665 INFO:distributed.worker - INFO - Waiting to connect to: tcp://donating-jaguar-dask-scheduler:8786\n", "tcp://10.244.5.7:41665 INFO:distributed.worker - INFO - Waiting to connect to: tcp://donating-jaguar-dask-scheduler:8786\n", "tcp://10.244.5.7:41665 INFO:distributed.worker - INFO - Waiting to connect to: tcp://donating-jaguar-dask-scheduler:8786\n", "tcp://10.244.5.7:41665 INFO:distributed.worker - INFO - Waiting to connect to: tcp://donating-jaguar-dask-scheduler:8786\n", "tcp://10.244.5.7:41665 INFO:distributed.worker - INFO - Waiting to connect to: tcp://donating-jaguar-dask-scheduler:8786\n", "tcp://10.244.5.7:41665 INFO:distributed.worker - INFO - Waiting to connect to: tcp://donating-jaguar-dask-scheduler:8786\n", "tcp://10.244.5.7:41665 INFO:distributed.worker - INFO - Waiting to connect to: tcp://donating-jaguar-dask-scheduler:8786\n", "tcp://10.244.5.7:41665 INFO:distributed.worker - INFO - Waiting to connect to: tcp://donating-jaguar-dask-scheduler:8786\n", "tcp://10.244.5.7:41665 INFO:distributed.worker - INFO - Waiting to connect to: tcp://donating-jaguar-dask-scheduler:8786\n", "tcp://10.244.5.7:41665 INFO:distributed.worker - INFO - Waiting to connect to: tcp://donating-jaguar-dask-scheduler:8786\n", "tcp://10.244.5.7:41665 INFO:distributed.worker - INFO - Waiting to connect to: tcp://donating-jaguar-dask-scheduler:8786\n", "tcp://10.244.5.7:41665 INFO:distributed.worker - INFO - Waiting to connect to: tcp://donating-jaguar-dask-scheduler:8786\n", "tcp://10.244.5.7:41665 INFO:distributed.worker - INFO - Waiting to connect to: tcp://donating-jaguar-dask-scheduler:8786\n", "tcp://10.244.5.7:41665 INFO:distributed.worker - INFO - Waiting to connect to: tcp://donating-jaguar-dask-scheduler:8786\n", "tcp://10.244.5.7:41665 INFO:distributed.worker - INFO - Waiting to connect to: tcp://donating-jaguar-dask-scheduler:8786\n", "tcp://10.244.5.7:41665 INFO:distributed.worker - INFO - Waiting to connect to: tcp://donating-jaguar-dask-scheduler:8786\n", "tcp://10.244.5.7:41665 INFO:distributed.worker - INFO - Waiting to connect to: tcp://donating-jaguar-dask-scheduler:8786\n", "tcp://10.244.5.7:41665 INFO:distributed.worker - INFO - Waiting to connect to: tcp://donating-jaguar-dask-scheduler:8786\n", "tcp://10.244.5.7:41665 INFO:distributed.worker - INFO - Waiting to connect to: tcp://donating-jaguar-dask-scheduler:8786\n", "tcp://10.244.5.7:41665 INFO:distributed.worker - INFO - Waiting to connect to: tcp://donating-jaguar-dask-scheduler:8786\n", "tcp://10.244.5.7:41665 INFO:distributed.worker - INFO - Waiting to connect to: tcp://donating-jaguar-dask-scheduler:8786\n", "tcp://10.244.5.7:41665 INFO:distributed.worker - INFO - Waiting to connect to: tcp://donating-jaguar-dask-scheduler:8786\n", "tcp://10.244.5.7:41665 INFO:distributed.worker - INFO - Registered to: tcp://donating-jaguar-dask-scheduler:8786\n", "tcp://10.244.5.7:41665 INFO:distributed.worker - INFO - -------------------------------------------------\n" ] } ], "source": [ "for w,logs in client.get_worker_logs().items():\n", " for log in logs:\n", " print(w, ':'.join(log))\n" ] }, { "cell_type": "code", "execution_count": 7, "metadata": { "ExecuteTime": { "end_time": "2018-10-10T14:23:48.337728Z", "start_time": "2018-10-10T14:23:48.041661Z" } }, "outputs": [], "source": [ "from dask import dataframe\n", "# ('head-1-5-read-parquet-0298763dc1b1d75d8743c178e56ae7cd', 0)\n", "df = dataframe.read_parquet(\"s3://datasets.farsetlabs.org.uk/parquet_dask_example/\",\n", " storage_options={'anon': True})\n", "client.publish_dataset(parquet_dask_example=client.persist(df))" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "ExecuteTime": { "end_time": "2018-10-10T13:36:12.025584Z", "start_time": "2018-10-10T13:36:09.296Z" } }, "outputs": [], "source": [ "client.list_datasets()" ] }, { "cell_type": "code", "execution_count": 17, "metadata": { "ExecuteTime": { "end_time": "2018-10-10T14:26:17.791009Z", "start_time": "2018-10-10T14:26:17.683459Z" } }, "outputs": [], "source": [ "df=client.get_dataset('parquet_dask_example')\n", "df['date'] = dataframe.to_datetime(df['date'])\n", "df=client.persist(df)" ] }, { "cell_type": "code", "execution_count": 18, "metadata": { "ExecuteTime": { "end_time": "2018-10-10T14:26:21.517966Z", "start_time": "2018-10-10T14:26:21.512552Z" } }, "outputs": [ { "data": { "text/plain": [ "cid int64\n", "dc object\n", "date datetime64[ns]\n", "count int64\n", "rule object\n", "sensor int64\n", "dtype: object" ] }, "execution_count": 18, "metadata": {}, "output_type": "execute_result" } ], "source": [ "df.dtypes" ] }, { "cell_type": "code", "execution_count": 11, "metadata": { "ExecuteTime": { "end_time": "2018-10-10T13:37:41.500857Z", "start_time": "2018-10-10T13:37:41.497038Z" } }, "outputs": [ { "data": { "text/plain": [ ">" ] }, "execution_count": 11, "metadata": {}, "output_type": "execute_result" } ], "source": [ "df.groupby(['date'])" ] }, { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [], "source": [] } ], "metadata": { "kernelspec": { "display_name": "Python [conda env:dask_demo]", "language": "python", "name": "conda-env-dask_demo-py" }, "language_info": { "codemirror_mode": { "name": "ipython", "version": 3 }, "file_extension": ".py", "mimetype": "text/x-python", "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", "version": "3.6.6" }, "toc": { "base_numbering": 1, "nav_menu": {}, "number_sections": true, "sideBar": true, "skip_h1_title": false, "title_cell": "Table of Contents", "title_sidebar": "Contents", "toc_cell": false, "toc_position": {}, "toc_section_display": true, "toc_window_display": false } }, "nbformat": 4, "nbformat_minor": 2 }