{ "cells": [ { "cell_type": "markdown", "metadata": {}, "source": [ "# Support for dask arrays\n", "\n", "It is possible to operate on dask arrays and spare the memory (or perhaps even time)." ] }, { "cell_type": "code", "execution_count": 1, "metadata": {}, "outputs": [], "source": [ "# Necessary imports\n", "import dask\n", "import dask.multiprocessing\n", "import physt\n", "import numpy as np\n", "\n", "import dask.array as da\n", "from physt import h1, h2\n", "%matplotlib inline" ] }, { "cell_type": "code", "execution_count": 2, "metadata": {}, "outputs": [], "source": [ "# Create two arrays\n", "np.random.seed(42)\n", "\n", "SIZE = 2 ** 21\n", "CHUNK = int(SIZE / 16)\n", "\n", "million = np.random.rand(SIZE)#.astype(int)\n", "million2 = (3 * million + np.random.normal(0., 0.3, SIZE))#.astype(int)\n", "\n", "# Chunk them for dask\n", "chunked = da.from_array(million, chunks=(CHUNK))\n", "chunked2 = da.from_array(million2, chunks=(CHUNK))" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "## Create histograms\n", "\n", "`h1`, `h2`, ... have their alternatives in `physt.dask_compat`. They should work similarly. Although, they are not complete and unexpected errors may occur." ] }, { "cell_type": "code", "execution_count": 3, "metadata": {}, "outputs": [], "source": [ "from physt.compat.dask import h1 as d1\n", "from physt.compat.dask import h2 as d2" ] }, { "cell_type": "code", "execution_count": 4, "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "Check: True\n" ] }, { "data": { "text/plain": [ "Histogram1D(bins=(28,), total=2097152, dtype=int64)" ] }, "execution_count": 4, "metadata": {}, "output_type": "execute_result" }, { "data": { "image/png": "iVBORw0KGgoAAAANSUhEUgAAAagAAAEYCAYAAAAJeGK1AAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjQuMiwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy8rg+JYAAAACXBIWXMAAAsTAAALEwEAmpwYAAAXtElEQVR4nO3df6zd9X3f8eerdkpIIwg/TERtp2bDSgtobcqV4zRaFdUpuCEKdALJ2VKs1ZJVxNp06pbZqzS0RJ5Am0LLNJhooBiaBSyaBa+IEQ8SsWnEcEnS8sNhWIXBDV7szJSSVdCYvPfH+dz2+ObiS+4xPp977/MhfXW+5/39fj73c45sv/z5ns/53lQVkiT15sfGPQBJkmZjQEmSumRASZK6ZEBJkrpkQEmSurR83AM43s4888xas2bNuIchSXqTHnvsse9W1YqZ9UUXUGvWrGFycnLcw5AkvUlJ/vdsdS/xSZK6ZEBJkrpkQEmSumRASZK6ZEBJkrpkQEmSumRASZK6ZEBJkrpkQEmSumRASZK6ZEBJkrpkQEmSumRASZK6ZEBJkrpkQEmSumRASZK6ZEBJkro0Z0AluTXJwSRPzHLsnyWpJGcO1bYn2Z/k6SQXD9UvTPJ4O3ZDkrT6SUnuavW9SdYMtdmc5Jm2bR751UqSFow3M4O6Ddg4s5hkNfDLwPNDtfOATcD5rc2NSZa1wzcBW4G1bZvucwvwUlWdC1wPXNf6Oh24Bng/sA64JslpP9rLkyQtVHMGVFU9BBye5dD1wKeAGqpdCtxZVa9V1bPAfmBdkrOBU6rq4aoq4HbgsqE2O9v+3cCGNru6GNhTVYer6iVgD7MEpSRpcVo+n0ZJPgZ8u6r+tF2pm7YS+NrQ86lW+37bn1mfbvMCQFUdSfIycMZwfZY20pKzZtu9J+TnPHftJSfk50hz+ZEDKsk7gN8FLprt8Cy1OkZ9vm1mjmkrg8uHvOc975ntFOktc6KCo2fzeQ8MQs1lPjOovwucA0zPnlYBX0+yjsEsZ/XQuauAF1t91Sx1htpMJVkOnMrgkuIU8KEZbb4624Cq6mbgZoCJiYlZQ0xLj8ExP75v6sWPHFBV9Thw1vTzJM8BE1X13SS7gf+U5LPATzJYDPFIVb2e5JUk64G9wJXAv29d7AY2Aw8DlwMPVlUluR/4N0MLIy4Cts/nRUrqj7MuzWXOgEryBQYzmTOTTAHXVNUts51bVU8m2QU8BRwBrq6q19vhqxisCDwZuK9tALcAdyTZz2DmtKn1dTjJZ4BH23mfrqrZFmtIkhahOQOqqj4+x/E1M57vAHbMct4kcMEs9VeBK96g71uBW+caoyRp8fFOEpKkLhlQkqQuGVCSpC4ZUJKkLhlQkqQuzetWR9KJ5pdHpaXHgJK0YPjl3qXFS3ySpC4ZUJKkLhlQkqQuGVCSpC4ZUJKkLhlQkqQuGVCSpC4ZUJKkLhlQkqQuGVCSpC4ZUJKkLhlQkqQuGVCSpC4ZUJKkLhlQkqQuzRlQSW5NcjDJE0O1f5vkW0n+LMl/TvKuoWPbk+xP8nSSi4fqFyZ5vB27IUla/aQkd7X63iRrhtpsTvJM2zYfrxctSerfm5lB3QZsnFHbA1xQVX8P+F/AdoAk5wGbgPNbmxuTLGttbgK2AmvbNt3nFuClqjoXuB64rvV1OnAN8H5gHXBNktN+9JcoSVqI5gyoqnoIODyj9uWqOtKefg1Y1fYvBe6sqteq6llgP7AuydnAKVX1cFUVcDtw2VCbnW3/bmBDm11dDOypqsNV9RKDUJwZlJKkRep4fAb168B9bX8l8MLQsalWW9n2Z9aPatNC72XgjGP09UOSbE0ymWTy0KFDI70YSVIfRgqoJL8LHAE+P12a5bQ6Rn2+bY4uVt1cVRNVNbFixYpjD1qStCDMO6DaooWPAv+oXbaDwSxn9dBpq4AXW33VLPWj2iRZDpzK4JLiG/UlSVoC5hVQSTYC/wL4WFX91dCh3cCmtjLvHAaLIR6pqgPAK0nWt8+XrgTuGWozvULvcuDBFnj3AxclOa0tjrio1SRJS8DyuU5I8gXgQ8CZSaYYrKzbDpwE7Gmrxb9WVb9RVU8m2QU8xeDS39VV9Xrr6ioGKwJPZvCZ1fTnVrcAdyTZz2DmtAmgqg4n+QzwaDvv01V11GINSdLilb+9Orc4TExM1OTk5LiHoWNYs+3ecQ9BS8hz114y7iFoDkkeq6qJmfU5Z1CStJDN5z9EhlofvNWRJKlLBpQkqUsGlCSpSwaUJKlLBpQkqUsGlCSpSwaUJKlLBpQkqUsGlCSpSwaUJKlLBpQkqUsGlCSpSwaUJKlLBpQkqUsGlCSpSwaUJKlLBpQkqUsGlCSpSwaUJKlLBpQkqUtzBlSSW5McTPLEUO30JHuSPNMeTxs6tj3J/iRPJ7l4qH5hksfbsRuSpNVPSnJXq+9Nsmaozeb2M55Jsvm4vWpJUvfezAzqNmDjjNo24IGqWgs80J6T5DxgE3B+a3NjkmWtzU3AVmBt26b73AK8VFXnAtcD17W+TgeuAd4PrAOuGQ5CSdLiNmdAVdVDwOEZ5UuBnW1/J3DZUP3Oqnqtqp4F9gPrkpwNnFJVD1dVAbfPaDPd193Ahja7uhjYU1WHq+olYA8/HJSSpEVqvp9BvbuqDgC0x7NafSXwwtB5U622su3PrB/VpqqOAC8DZxyjL0nSEnC8F0lkllodoz7fNkf/0GRrkskkk4cOHXpTA5Uk9W2+AfWddtmO9niw1aeA1UPnrQJebPVVs9SPapNkOXAqg0uKb9TXD6mqm6tqoqomVqxYMc+XJEnqyXwDajcwvapuM3DPUH1TW5l3DoPFEI+0y4CvJFnfPl+6ckab6b4uBx5sn1PdD1yU5LS2OOKiVpMkLQHL5zohyReADwFnJplisLLuWmBXki3A88AVAFX1ZJJdwFPAEeDqqnq9dXUVgxWBJwP3tQ3gFuCOJPsZzJw2tb4OJ/kM8Gg779NVNXOxhiRpkZozoKrq429waMMbnL8D2DFLfRK4YJb6q7SAm+XYrcCtc41RkrT4eCcJSVKXDChJUpcMKElSlwwoSVKXDChJUpcMKElSlwwoSVKXDChJUpcMKElSlwwoSVKXDChJUpcMKElSlwwoSVKX5rybuXQsa7bdO+4hSFqkDChJmmE+//F67tpL3oKRLG1e4pMkdcmAkiR1yYCSJHXJgJIkdcmAkiR1yYCSJHXJgJIkdWmkgEryT5M8meSJJF9I8vYkpyfZk+SZ9nja0Pnbk+xP8nSSi4fqFyZ5vB27IUla/aQkd7X63iRrRhmvJGnhmHdAJVkJ/BYwUVUXAMuATcA24IGqWgs80J6T5Lx2/HxgI3BjkmWtu5uArcDatm1s9S3AS1V1LnA9cN18xytJWlhGvcS3HDg5yXLgHcCLwKXAznZ8J3BZ278UuLOqXquqZ4H9wLokZwOnVNXDVVXA7TPaTPd1N7BhenYlSVrc5h1QVfVt4N8BzwMHgJer6svAu6vqQDvnAHBWa7ISeGGoi6lWW9n2Z9aPalNVR4CXgTNmjiXJ1iSTSSYPHTo035ckSerIKJf4TmMwwzkH+EngJ5J84lhNZqnVMerHanN0oermqpqoqokVK1Yce+CSpAVhlEt8HwaerapDVfV94IvALwDfaZftaI8H2/lTwOqh9qsYXBKcavsz60e1aZcRTwUOjzBmSdICMUpAPQ+sT/KO9rnQBmAfsBvY3M7ZDNzT9ncDm9rKvHMYLIZ4pF0GfCXJ+tbPlTPaTPd1OfBg+5xKkrTIzfvXbVTV3iR3A18HjgDfAG4G3gnsSrKFQYhd0c5/Msku4Kl2/tVV9Xrr7irgNuBk4L62AdwC3JFkP4OZ06b5jleStLCM9Pugquoa4JoZ5dcYzKZmO38HsGOW+iRwwSz1V2kBJ0laWryThCSpSwaUJKlLBpQkqUsGlCSpSwaUJKlLBpQkqUsGlCSpSwaUJKlLBpQkqUsGlCSpSwaUJKlLBpQkqUsGlCSpSwaUJKlLBpQkqUsGlCSpSwaUJKlLBpQkqUsGlCSpSwaUJKlLBpQkqUsjBVSSdyW5O8m3kuxL8oEkpyfZk+SZ9nja0Pnbk+xP8nSSi4fqFyZ5vB27IUla/aQkd7X63iRrRhmvJGnhGHUG9fvAf62qnwZ+FtgHbAMeqKq1wAPtOUnOAzYB5wMbgRuTLGv93ARsBda2bWOrbwFeqqpzgeuB60YcryRpgZh3QCU5BfhF4BaAqvrrqvoL4FJgZzttJ3BZ278UuLOqXquqZ4H9wLokZwOnVNXDVVXA7TPaTPd1N7BhenYlSVrcRplB/R3gEPCHSb6R5HNJfgJ4d1UdAGiPZ7XzVwIvDLWfarWVbX9m/ag2VXUEeBk4Y+ZAkmxNMplk8tChQyO8JElSL0YJqOXAzwM3VdX7gP9Hu5z3Bmab+dQx6sdqc3Sh6uaqmqiqiRUrVhx71JKkBWGUgJoCpqpqb3t+N4PA+k67bEd7PDh0/uqh9quAF1t91Sz1o9okWQ6cChweYcySpAVi3gFVVf8HeCHJe1tpA/AUsBvY3GqbgXva/m5gU1uZdw6DxRCPtMuAryRZ3z5funJGm+m+LgcebJ9TSZIWueUjtv9N4PNJfhz4c+AfMwi9XUm2AM8DVwBU1ZNJdjEIsSPA1VX1euvnKuA24GTgvrbBYAHGHUn2M5g5bRpxvJKkBWKkgKqqbwITsxza8Abn7wB2zFKfBC6Ypf4qLeAkSUuLd5KQJHXJgJIkdcmAkiR1yYCSJHXJgJIkdcmAkiR1yYCSJHVp1C/qahFZs+3ecQ9BWrDm8/fnuWsveQtGsng4g5IkdcmAkiR1yYCSJHXJgJIkdcmAkiR1yYCSJHXJgJIkdcmAkiR1yYCSJHXJgJIkdcmAkiR1yYCSJHXJgJIkdcmAkiR1aeSASrIsyTeS/El7fnqSPUmeaY+nDZ27Pcn+JE8nuXiofmGSx9uxG5Kk1U9Kcler702yZtTxSpIWhuMxg/oksG/o+TbggapaCzzQnpPkPGATcD6wEbgxybLW5iZgK7C2bRtbfQvwUlWdC1wPXHccxitJWgBGCqgkq4BLgM8NlS8Fdrb9ncBlQ/U7q+q1qnoW2A+sS3I2cEpVPVxVBdw+o810X3cDG6ZnV5KkxW3UGdTvAZ8CfjBUe3dVHQBoj2e1+krghaHzplptZdufWT+qTVUdAV4Gzpg5iCRbk0wmmTx06NCIL0mS1IN5B1SSjwIHq+qxN9tkllodo36sNkcXqm6uqomqmlixYsWbHI4kqWfLR2j7QeBjST4CvB04JckfAd9JcnZVHWiX7w6286eA1UPtVwEvtvqqWerDbaaSLAdOBQ6PMGZJ0gIx7xlUVW2vqlVVtYbB4ocHq+oTwG5gczttM3BP298NbGor885hsBjikXYZ8JUk69vnS1fOaDPd1+XtZ/zQDEqStPiMMoN6I9cCu5JsAZ4HrgCoqieT7AKeAo4AV1fV663NVcBtwMnAfW0DuAW4I8l+BjOnTW/BeCVJHTouAVVVXwW+2vb/L7DhDc7bAeyYpT4JXDBL/VVawEmSlhbvJCFJ6pIBJUnqkgElSeqSASVJ6pIBJUnqkgElSeqSASVJ6pIBJUnqkgElSeqSASVJ6pIBJUnqkgElSeqSASVJ6pIBJUnqkgElSeqSASVJ6pIBJUnqkgElSeqSASVJ6pIBJUnq0vJxD0BvjTXb7h33ECTNYT5/T5+79pK3YCR9mvcMKsnqJF9Jsi/Jk0k+2eqnJ9mT5Jn2eNpQm+1J9id5OsnFQ/ULkzzejt2QJK1+UpK7Wn1vkjUjvFZJ0gIyyiW+I8DvVNXPAOuBq5OcB2wDHqiqtcAD7Tnt2CbgfGAjcGOSZa2vm4CtwNq2bWz1LcBLVXUucD1w3QjjlSQtIPMOqKo6UFVfb/uvAPuAlcClwM522k7gsrZ/KXBnVb1WVc8C+4F1Sc4GTqmqh6uqgNtntJnu625gw/TsSpK0uB2XRRLt0tv7gL3Au6vqAAxCDDirnbYSeGGo2VSrrWz7M+tHtamqI8DLwBmz/PytSSaTTB46dOh4vCRJ0piNHFBJ3gn8MfDbVfWXxzp1llodo36sNkcXqm6uqomqmlixYsVcQ5YkLQAjBVSStzEIp89X1Rdb+Tvtsh3t8WCrTwGrh5qvAl5s9VWz1I9qk2Q5cCpweJQxS5IWhlFW8QW4BdhXVZ8dOrQb2Nz2NwP3DNU3tZV55zBYDPFIuwz4SpL1rc8rZ7SZ7uty4MH2OZUkaZEb5XtQHwR+DXg8yTdb7V8C1wK7kmwBngeuAKiqJ5PsAp5isALw6qp6vbW7CrgNOBm4r20wCMA7kuxnMHPaNMJ4JUkLyLwDqqr+B7N/RgSw4Q3a7AB2zFKfBC6Ypf4qLeAkSUuLtzqSJHXJgJIkdcmAkiR1yYCSJHXJgJIkdcmAkiR1yYCSJHXJgJIkdcmAkiR1yYCSJHXJgJIkdcmAkiR1yYCSJHXJgJIkdWmU3welE2TNtnvHPQRJnZjPvwfPXXvJWzCSt54zKElSlwwoSVKXDChJUpcMKElSlwwoSVKXDChJUpcMKElSlxbE96CSbAR+H1gGfK6qrh3zkCRpwZjvdynH/f2p7gMqyTLgPwC/DEwBjybZXVVPjXdk8+OXbiXpzek+oIB1wP6q+nOAJHcClwJjDyjDRtJiNu67ViyEgFoJvDD0fAp4//AJSbYCW9vT7yV5+gSN7UzguyfoZ/XK98D3AHwPwPcAgFw3r/fhp2YrLoSAyiy1OupJ1c3AzSdmOH8ryWRVTZzon9sT3wPfA/A9AN+DacfzfVgIq/imgNVDz1cBL45pLJKkE2QhBNSjwNok5yT5cWATsHvMY5IkvcW6v8RXVUeS/BPgfgbLzG+tqifHPKxpJ/yyYod8D3wPwPcAfA+mHbf3IVU191mSJJ1gC+ESnyRpCTKgJEldMqBGkOSKJE8m+UGSJbW8NMnGJE8n2Z9k27jHMw5Jbk1yMMkT4x7LuCRZneQrSfa1vwufHPeYTrQkb0/ySJI/be/Bvx73mMYlybIk30jyJ8ejPwNqNE8A/wB4aNwDOZGGbj/1K8B5wMeTnDfeUY3FbcDGcQ9izI4Av1NVPwOsB65egn8WXgN+qap+Fvg5YGOS9eMd0th8Eth3vDozoEZQVfuq6kTdtaInf3P7qar6a2D69lNLSlU9BBwe9zjGqaoOVNXX2/4rDP5xWjneUZ1YNfC99vRtbVtyq8+SrAIuAT53vPo0oDQfs91+akn9o6QflmQN8D5g75iHcsK1S1vfBA4Ce6pqyb0HwO8BnwJ+cLw6NKDmkOS/JXlilm3JzRiGzHn7KS0tSd4J/DHw21X1l+Mez4lWVa9X1c8xuNPNuiQXjHlIJ1SSjwIHq+qx49lv91/UHbeq+vC4x9Ahbz+lv5HkbQzC6fNV9cVxj2ecquovknyVwWeTS2nxzAeBjyX5CPB24JQkf1RVnxilU2dQmg9vPyUAkgS4BdhXVZ8d93jGIcmKJO9q+ycDHwa+NdZBnWBVtb2qVlXVGgb/Hjw4ajiBATWSJL+aZAr4AHBvkvvHPaYToaqOANO3n9oH7Oro9lMnTJIvAA8D700ylWTLuMc0Bh8Efg34pSTfbNtHxj2oE+xs4CtJ/ozBf972VNVxWWa91HmrI0lSl5xBSZK6ZEBJkrpkQEmSumRASZK6ZEBJkrpkQEkLQJL/OcfxC5M83u4uf0P7fpK0oBlQ0gJQVb8wxyk3AVuBtW1b6ndZ1yJgQEljkuRLSR5rv0Noa5KfSvJMkjOT/FiS/57konbu99rj2Ukeal+IfSLJ309yNnBKVT1cgy823g5cNr5XJh0f3otPGp9fr6rD7fY4jzK4n911wH9kcEfwp6rqyzPa/EPg/qra0X4v1zuA9zK4P+I07y6vRcGAksbnt5L8attfDaytqs8luQL4DQa//G6mR4Fb2w1av1RV33yDz5u8RYwWPC/xSWOQ5EMMbir6gfabWL8BvD3JOxjcHR7gnTPbtV+S+IvAt4E7klzJYMa0aug07y6vRcGAksbjVOClqvqrJD/N4Nelw+AS3+eBfwX8wcxGSX6Kwe/d+QMGdxH/+ao6ALySZH2bTV0J3HMiXoT0VvJmsdIYJDkJ+BKDz4qeBlYAXwF+BfhgVb2e5IvAf6mqP0zyvap6Z5LNwD8Hvg98D7iyqp5NMgHcBpwM3Af8ZvmXWwucASVJ6pKX+CRJXTKgJEldMqAkSV0yoCRJXTKgJEldMqAkSV0yoCRJXfr/BtHTb7GAsZAAAAAASUVORK5CYII=\n", "text/plain": [ "
" ] }, "metadata": { "needs_background": "light" }, "output_type": "display_data" } ], "source": [ "# Use chunks to create a 1D histogram\n", "ha = d1(chunked2, \"fixed_width\", bin_width=0.2)\n", "check_ha = h1(million2, \"fixed_width\", bin_width=0.2)\n", "ok = (ha == check_ha)\n", "print(\"Check: \", ok)\n", "ha.plot()\n", "ha" ] }, { "cell_type": "code", "execution_count": 5, "metadata": {}, "outputs": [ { "name": "stderr", "output_type": "stream", "text": [ "/home/honza/code/my/physt/physt/util.py:74: FutureWarning:\n", "\n", "histogramdd is deprecated, use h instead\n", "\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ "Check: True\n" ] }, { "data": { "text/plain": [ "Histogram2D(bins=(5, 28), total=2097152, dtype=int64)" ] }, "execution_count": 5, "metadata": {}, "output_type": "execute_result" }, { "data": { "image/png": "iVBORw0KGgoAAAANSUhEUgAAAZwAAAEYCAYAAABoYED3AAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjQuMiwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy8rg+JYAAAACXBIWXMAAAsTAAALEwEAmpwYAAAew0lEQVR4nO3dfZRddX3v8fdnZvJMniQIJCEklgeLLDpAitSngrSXAC4SulBTKgFWXFlafOi1XAO0t3qLePGuWpULYqNQQGkDRQlBiYhBpCogAafhIRLDUxgTwCSQxGAmmTnf+8fZ8Y5xMnMGzt6/fc75vNY6K+fss88+n03CfOf327/fbysiMDMzy1tb6gBmZtYaXHDMzKwQLjhmZlYIFxwzMyuEC46ZmRWiI3WA4ZgyZUrMnDkzdQwzG8LWnh56dzX/CNhn1z6+KSIOqMex5kixqcZ9H4a7ImJOPb63SA1VcGbOnMmqVatSxzCzIVx/71qeO/mI1DFy92n0XL2OtQlY1V5bp5P6KlPq9b1FaqiCY2bWtAS0q7Z9+3JNkhsXHDOzUhDU2MJp1IrjgmNmVgYCOmps4TQoFxwzszIQw2jhNKbkBUdSO7AK+GVEvCd1HjOzZGq9htOgkhcc4OPAGmBC6iBmZsloONdwGlPSs5M0HTgD+FrKHGZmye3pUqvl0aBSt3C+CHwSGL+vHSQtAhYBzJgxo5hUZva67Jj0a0bd1pU6Rv7OqvPx3KWWD0nvAV6KiIclnbSv/SJiCbAEYPbs2c0/ddmsCYx7ZT82ndX8Ez/rqgW61FK2cN4OnCnpdGA0MEHSNyLiAwkzmZmlIWBUe+oUuUpWTiPikoiYHhEzgfnAPS42ZtaypGqXWi2PBpX6Go6Zme3hLrX8RcS9wL2JY5iZpeOJn2ZmVgg1dndZLVxwzMzKwi0cM6uHO55ex+b1ldQxCvHC2C2ehzNcw7k9QYNywTEryOb1lZa4KRnAiOVdbF/QmTpGg/E8HDMzK4JbOGZmVgiPUjMzs8K4hWNmZrmTYGRz/0hu7rMzM2sUvoZjZmaF8GrRZmZWGLdwzKweWuamZEDP1FcZt7QrdYz8nVHHY3mUmpnVSyvdlKyysou1nz0mdYzG4rXUzMysMG7hmJlZ7jxKzczMiuFRamZmVgS3cMzMrBAepWZmZsVwl5qZ1UkrzcN5ZdoODvl8V+oY+TuxjscSMKK9foeT/jvwQSCAR4ELgLHAzcBM4FngfRHxcrb/JcBCoA/4WETclW0/HrgeGAPcCXw8IkLSKOBG4HhgM/D+iHh2sEwuOGYFaaV5ONvWruLaDTNTx2gsdbyGI2ka8DHgqIj4jaRbgPnAUcDKiLhC0sXAxcBiSUdl778FmAp8X9IREdEHXAMsAh6gWnDmACuoFqeXI+IwSfOBzwHvHyxXc7ffzMwaRtalVsujNh3AGEkdVFs2G4C5wA3Z+zcA87Lnc4GlEdETEc8A64ATJB0MTIiI+yMiqLZo+n9mz7FuBU6RNGjFdMExMyuDPS2cWh4wRdKqfo9F/Q8VEb8E/glYD2wEtkbE94ADI2Jjts9G4I3ZR6YBz/c7RHe2bVr2fO/tv/OZiOgFtgL7D3aK7lIzMyuLtprbAJsiYva+3pQ0mWoLZBbwCvAfkj4wyPEGapnEINsH+8w+uYVjZlYGqrF1U9t1nj8DnomIX0XEbuBbwNuAF7NuMrI/X8r27wYO6ff56VS74Lqz53tv/53PZN12E4Etg4VywTEzK4M983Dqcw1nPXCipLHZdZVTgDXAcuC8bJ/zgNuz58uB+ZJGSZoFHA78NOt22y7pxOw4C/b6zJ5jnQ3ck13n2Sd3qZmZlUWdRqlFxIOSbgUeAXqBnwFLgP2AWyQtpFqU3pvt/3g2ku2JbP8LsxFqAB/m/w+LXpE9AK4Fvi5pHdWWzfyhcrngWFJ3PL2OzesrqWMU4oWxW1pmHs7Yydv54Dt+njpG7r5cz4NJw7mGM6SI+BTwqb0291Bt7Qy0/+XA5QNsXwUcPcD2nWQFq1YuOJbU5vUVnju5NeamjFjexfYFnaljFOKg9XczcuLm1DEaj9dSMzOz3HktNTMzK4RU16VtysgFx8ysLNrcpWZmZnlzl1p+JI0G7gNGZTluzUZVmJm1ILmFk6Me4N0R8WtJI4AfSVoREQ8kzGRmloZbOPnJZqT+Ons5InsMOkvVzKypuYWTH0ntwMPAYcDVEfHgAPssonovBmbMmFFsQMtdK92UrGfqq4xb2pU6RjG0g7bfNPeIq7qT7/iZq2zphE5Jk4DbJB0dEY/ttc8SqksyMHv2bLeAmkwr3ZQsVnbx5GXHpI5RiNOu+RnnfOPa1DFy99F6H7DJWzilKKcR8QpwL9U7yZmZtZ76Lt5ZSsmSSzoga9kgaQzV5bSbf/ElM7MBZaPUank0qJRdagcDN2TXcdqAWyLi2wnzmJmlI+q6eGcZpRyltho4NtX3m5mVipe2MTOzwjRwd1ktXHDMzMpA+PYEZnlqpXk4r0zdwbQvdKWOUYjtY4Nlf3l26hj5+/yP63s8X8Mxy08rzcPZvnYV1/1yVuoYhfjKr+9j3hkXpY7RWCQq7lIzM7O8BVBxC8fMzIrgFo6ZmeUuJPoaeBWBWrjgmJmVhFs4ZmaWP0H4Go6ZmeWtOmjALRwzM8ubh0VbKnc8vY7N6yupY+TuhbFbWmbi55jJ21n4ztZYEH3nS2NYdvv/Sh0jf6f9fd0OFUBvh9dSswQ2r6/w3MnNPyFyxPIuti/oTB2jEAeuv5sREzekjlGISVth3p1e/H04QqJPvoZjZmYFcJeamZkVwgXHzMxyFx4WbWZmxfAoNTMzK4K8eKeZmRUggIrcwrEEWuXGZD1TX2Xc0q7UMQoh7aDjN839G+weWyaO4+aPnp86Rv7+7wN1PZy71CyJVrkxWWVlF2s/e0zqGIU448pHOOf6r6aOUYirrvgkGw8YkzpGQwnJXWpmZlaMPnepmZlZ3gLR1+6lbczMLG/yNRwzMytAdZSar+GYmVkBmr2F09zl1MysUUhUanzUdjhNknSrpJ9LWiPpTyS9QdLdkn6R/Tm53/6XSFon6UlJp/bbfrykR7P3rpSqASSNknRztv1BSTOHyuQWTkm1yjycV6bt4JDPd6WOUYjt42DZue9LHaMQL0cHbbtHpY7RUALoq++w6C8B342IsyWNBMYClwIrI+IKSRcDFwOLJR0FzAfeAkwFvi/piIjoA64BFgEPAHcCc4AVwELg5Yg4TNJ84HPA+wcL5IJTUq0yD2fb2lVcu2Fm6hiFWLL1h8w79ROpYxTivrV3c/Vjx6WO0XDqtdKApAnAu4DzASJiF7BL0lzgpGy3G4B7gcXAXGBpRPQAz0haB5wg6VlgQkTcnx33RmAe1YIzF/h0dqxbgaskKSJiX7ncpWZmVgJ7lrapU5fam4BfAf8q6WeSviZpHHBgRGwEyP58Y7b/NOD5fp/vzrZNy57vvf13PhMRvcBWYP/BQrngmJmVgUS0tdX0AKZIWtXvsWivo3UAxwHXRMSxwA6q3Wf7/PYBtsUg2wf7zD65S83MrCSG0aW2KSJmD/J+N9AdEQ9mr2+lWnBelHRwRGyUdDDwUr/9D+n3+enAhmz79AG29/9Mt6QOYCKwZbDQbuGYmZVAPbvUIuIF4HlJR2abTgGeAJYD52XbzgNuz54vB+ZnI89mAYcDP8263bZLOjEbnbZgr8/sOdbZwD2DXb+BhC0cSYcANwIHARVgSUR8KVUeM7PU6nx7go8CN2Uj1J4GLqDayLhF0kJgPfBegIh4XNItVItSL3BhNkIN4MPA9cAYqoMFVmTbrwW+ng0w2EJ1lNugUnap9QJ/GxGPSBoPPCzp7oh4ImEmM7MkQqK3rX5rqUVEFzBQt9sp+9j/cuDyAbavAo4eYPtOsoJVq2QFJ2uq7RktsV3SGqqjHlxwzKzlVOfhNPdKA6UYNJDNUD0WeHCA9xZRnXTEAVOncv29a4sNl8gLY7e0xMTPsZO388F3/Dx1jELsfHEMy+68LHWMQrSPhgs6n0odI3f/Us+DSV5LLW+S9gO+CfxNRGzb+/2IWAIsAZh15NHx3MnNPxkSYOTtXfz6nM7UMXJ30Ia76Zg46MCWpjFpaxvzVn43dYxCrJnwp2xYfHzqGA0nfD+c/EgaQbXY3BQR30qZxcwspQAqA05taR4pR6mJ6iiHNRHxz6lymJmVRZ1HqZVOyg7DtwPnAu+W1JU9Tk+Yx8wsoeo1nFoejSrlKLUfMfDSCGZmLWfPxM9mlnzQgJmZAYI+FxwzM8ubWzgl09deaYm5KQA9U19lzH90pY5RgB10/KZx+6SHY8vEsdz8oQWpYxRiV0+Fwy5bnTpG/t5dz4OJSpMvb9lQBae9r42eszpTxyhErOziyc8ckzpG7k67+hHOuf6rqWMU4uorPsmv9h+ZOkYhRv54LNvmdqaO0VByuONn6TRUwTEza2aeh2NmZrkLvLSNmZkVxC0cMzPLXcij1MzMrCB9buGYmVneqtdwXHDMzKwA4RZOebTSxM9Xpu5g2he6UsfI3fZxsOzc96WOUYiXo4O23rGpYxSid3wP42/qSh0jf2fW93AepVYirTTxc9vaVVy7cWbqGLlbsu2HzDv1E6ljFOK+tXdz9WPHpY5RiEvHPcfuMztTx2govh+OmZkVRC44ZmZWDBccMzPLXQC7fQ3HzMzyFl4t2szMitLsw6KHLKeSPiJpchFhzMxaWSUbODDUo1HV0sI5CHhI0iPAdcBdERH5xhpYK83DGTt5Ox98x89Tx8jdzhfHsOzOy1LHKET7aLig86nUMYqxeVdr/L96Vv0O5WHRQET8vaT/Cfw34ALgKkm3ANdGRKH/97RV2ti6oLPIr0zmoPXfY/SEX6WOkbtJW9uYt/K7qWMUYs2EP+WFi45PHaMQIy5d3TJz5uqp2QtOTVeoshbNC9mjF5gM3Crp/+SYzcysZQSir8ZHoxqyhSPpY8B5wCbga8D/iIjdktqAXwCfzDeimVlraPZBA7Vcw5kC/EVEPNd/Y0RUJL0nn1hmZq2n2bvUarmG8w+DvLemvnHMzFpTAH3R4gXHzMyK0fItHDMzy1910EB76hi5csExMyuJirvUyqPSXmHc0q7UMQryKto5InWI3G2ZOJabP7QgdYxC7Oqp8KbLV6eOUYi+ca964ucwBTT0kOdaNFTBUbSx5rPHpI5RiDOufIRzrv9q6hi5u+qKT/LilNGpYxRi5I/Gsm1eZ+oYhRhzY5cnfg6bCLdwzMwsb17axszMihEeFm1mZgVohRZO0rv9SLpO0kuSHkuZw8ysDCJU06NRpb693PXAnMQZzMxKoLZ74QynFSSpXdLPJH07e/0GSXdL+kX25+R++14iaZ2kJyWd2m/78ZIezd67UpKy7aMk3Zxtf1DSzKHyJC04EXEfsCVlBjOzMgiq83BqeQzDx4H+S5BdDKyMiMOBldlrJB0FzAfeQrUR8GVJe2ahXgMsAg7PHnsaCQuBlyPiMOALwOeGClP6aziSFlE9WSZNPYhD/rUrbaCCbB8Hy859X+oYuXslOtDucaljFKJvfA/jb+pKHaMYB+7k0B+sTZ0ifyfX93D1HDQgaTpwBnA58Ils81zgpOz5DcC9wOJs+9KI6AGekbQOOEHSs8CEiLg/O+aNwDxgRfaZT2fHupXqvdI02A06S19wImIJsARgWudb4kvdsxInKsZXtt3HvNMuSh0jd/etuYurHzsudYxCXDruOXaf2Zk6RiEO/cFazj/piNQxcndBHY9VXbyz5k6nKZJW9Xu9JPtZ2d8Xqd4+Zny/bQdGxEaAiNgo6Y3Z9mnAA/3268627c6e7719z2eez47VK2krsD/VW9kMqPQFx8ysJYSG08LZFBGz9/VmduuYlyLiYUkn1XC8gb44Btk+2Gf2yQXHzKwEqtdw6na4twNnSjodGA1MkPQN4EVJB2etm4OBl7L9u4FD+n1+OrAh2z59gO39P9MtqQOYyBDX5FMPi/534H7gSEndkhamzGNmllK9hkVHxCURMT0iZlIdDHBPRHwAWE71Ds5kf96ePV8OzM9Gns2iOjjgp1n323ZJJ2aj0xbs9Zk9xzo7+47ytnAi4i9Tfr+ZWVnsGaWWsyuAW7Jf7tcD7wWIiMcl3QI8AfQCF0ZEX/aZD1OdwjKG6mCBFdn2a4GvZwMMtlAtbINyl5qZWUnksdJARNxLdTQaEbEZOGUf+11OdUTb3ttXAUcPsH0nWcGqlQuOmVkJ+BbTZdNWYeE71wy9XxPY+dJolt3xj6lj5K59NFzQ+VTqGMXYvKs17hED7JhUSR2h8TT4sjW1aKiCM6otePOU7qF3bAKTtop5K76dOkbu1kx4FxsWH586RiFmLF7dMveIOagVJn3moFJxwTEzs5y5S83MzIoRhYxSS8oFx8ysJHwNx8zMcheIXl/DMTOzvEVAnwuOmZkVwV1qZmZWCA8aKJFKpYJ2jkgdoxBbJo7j5o+enzpG7nbtqnDYZatTxyjGxBa5KRmw/4zUd69vPIG71Epl1O4+Lvy7f0odoxBXX34Rv5oyMnWM3I388Vi2ze1MHaMQh36nNW5KZq/R8G8f3XAaquCYmTWrAKLJVwRywTEzKwm3cMzMLH/htdTMzKwAXkvNzMwKE27hmJlZ3iLE7j4XnNLoJVj2V2enjlGIl6ODtt6xqWPkrnd8D+Nv6kodoxCvTmjyIUj2ugS+hlMqHbsqzDvxotQxCvGfa+7iy6ub/8ZkF49/lt1ndqaOUYj9WmTSp712XtrGzMzyF1Bp8kawC46ZWQm4S83MzIrh2xOYmVkRArmFY2ZmxfBaamZmlr/wSgNmZlYADxoomd4RYtkd/5g6RiHaRovzjn0qdYz8belh1G1dqVMUYsekJu8vsdfNw6JLpENtzPvenaljFGLNhHfxwkXHpY6RuxGXrqbnrM7UMQpxkCd+2mDCa6mZmVkBImB3rwuOmZnlzMOizcysGAHR5KtFt6X8cklzJD0paZ2ki1NmMTNLKYC+Sm2PRpWshSOpHbga+HOgG3hI0vKIeCJVJjOzlJq9Sy1lC+cEYF1EPB0Ru4ClwNyEeczM0slWi67l0ahSXsOZBjzf73U38Na9d5K0CFgEMGX6Adz814uLSZfYrl0V3nT56tQx8jdpJ4e2yHDh/Wck7cG2BqAaWziRc468pCw4A/2X/b3/jhGxBFgC8KbZb44XD3tD3rlKYeSPxrJtXmfqGLk79DtrOf+kI1LHMEsvoL3GQQO9OUfJS8qC0w0c0u/1dGBDoixmZkkJaGvg7rJapCw4DwGHS5oF/BKYD5yTMI+ZWToh2pp80ECyghMRvZI+AtwFtAPXRcTjqfKYmaWmvtQJ8pV04mdE3Am0xuJoZmaDUMCIGpe2+U3OWfLiYTNmZiWggLY+1fQY8ljSIZJ+IGmNpMclfTzb/gZJd0v6Rfbn5H6fuSSbhP+kpFP7bT9e0qPZe1dKUrZ9lKSbs+0PSpo5VC4XHDOzklBFNT1q0Av8bUT8IXAicKGko4CLgZURcTiwMntN9t584C3AHODL2eR8gGuoTk05PHvMybYvBF6OiMOALwCfGypUQ62ltjtg++4JqWMUom98D+Nv6kodI3evTmjyYTlmNVJAe52u4UTERmBj9ny7pDVU5z7OBU7KdrsBuBdYnG1fGhE9wDOS1gEnSHoWmBAR9wNIuhGYB6zIPvPp7Fi3AldJUkTsc5pQQxWcSl87n7njj1PHKMTF459l95mdqWPkbr8WmfRpNrRhjVKbImlVv9dLsjmLv3/UalfXscCDwIFZMSIiNkp6Y7bbNOCBfh/rzrbtzp7vvX3PZ57PjtUraSuwP7BpX6EbquCYmTWtgLbaWzibImL2UDtJ2g/4JvA3EbEtu/wy4K4DJxp0gn5Nk/f78zUcM7MSEHW9hoOkEVSLzU0R8a1s84uSDs7ePxh4Kdu+r4n43dnzvbf/zmckdQATgS2DZXLBMTMrg+waTi2PoWQjya4F1kTEP/d7azlwXvb8POD2ftvnZyPPZlEdHPDTrPttu6QTs2Mu2Osze451NnDPYNdvwF1qZmalUOelbd4OnAs8Kqkr23YpcAVwi6SFwHrgvQAR8bikW4AnqI5wuzAi9pS2DwPXA2OoDhZYkW2/Fvh6NsBgC9VRboNywTEzK4NsHk5dDhXxIwa+xgJwyj4+czlw+QDbVwFHD7B9J1nBqpULjplZSajJZwm44JiZlYCGcXuCRtVYBScqnHfsU6lTFGNLD6Nu60qdInc7JjX5r3RmNVJAx24XnNLo2N3O9gXHpY5RiBmLV9NzVmfqGLk7yBM/zX7Lq0WbmVn+Qu5SMzOz/Gl4Kw00JBccM7OS8C2mzcwsfwFyl5qZmeVN1O/2BGXlgmNmVga+hlMulbYKh122OnWMQvSNfdXzcMxaiKjf0jZl1VAFp723jV1zO1PHKMSYG7s8D8eslYSXtjEzswL4Go6ZmRWjAh273KVmZmY5q17DSZ0iXy44ZmZlUMf74ZSVC46ZWQm4hWNmZsXwPBwzMyuEC065VNorjL+pK3WMYhy4k0NbYI7K/jPaUkcwKwUhX8Mpk/beNnaf2Zk6RiEO/cFazj/piNQxzKwobuGYmVkRfD8cMzMrjAuOmZnlTi0wDyfJFVtJ75X0uKSKpNkpMpiZlUpAx67aHo0qVQvnMeAvgH9J9P1mZqXiazg5iYg1AFJzNx/NzIbDBScxSYuARQCTDz6oJW5KBr4xmVmraYVrOLkVHEnfBw4a4K2/i4jbaz1ORCwBlgDMOvLoaIWbkoFvTGbWitzCeY0i4s/yOraZWdPxNRwzMytCKwwaSDUs+ixJ3cCfAN+RdFeKHGZmZdLWV9ujUaUapXYbcFuK7zYzK6NWaOG4S83MrAwC2npTh8iXC46ZWUl4WHSJ9LVXPA/HzJqSu9RKpr2vDc/DMbNmpGjsddJq0VAFx8ysabmFY2ZmRXHBMTOz3PkajpmZFaMFhkUnWWnAzMx+l6jvSgOS5kh6UtI6SRfnGr5GbuGYmZVBHbvUJLUDVwN/DnQDD0laHhFP1OcbXhsXHDOzkqjjNZwTgHUR8TSApKXAXCBpwVFEpPz+YZG0HXgydY6CTAE2pQ5RgFY5T/C5NqMjI2J8PQ4k6btU/7vVYjSws9/rJdm9w/Yc62xgTkR8MHt9LvDWiPhIPbK+Vo3WwnkyImanDlEESata4Vxb5TzB59qMJK2q17EiYk69jkX1ktDvfUUdj/+aeNCAmVnz6QYO6fd6OrAhUZbfcsExM2s+DwGHS5olaSQwH1ieOFPDdaktGXqXptEq59oq5wk+12ZUyvOMiF5JHwHuAtqB6yLi8cSxGmvQgJmZNS53qZmZWSFccMzMrBClLDhDLcmgqiuz91dLOi5FzterhvP8q+z8Vkv6iaQ/SpGzHmpdZkPSH0vqy+YRNKRazlXSSZK6JD0u6YdFZ6yHGv79TpR0h6T/ys7zghQ5Xy9J10l6SdJj+3i/KX4eFSIiSvWgeoHrKeBNwEjgv4Cj9trndGAF1bHmJwIPps6d03m+DZicPT+tEc+z1nPtt989wJ3A2alz5/j3OonqjO8Z2es3ps6d03leCnwue34AsAUYmTr7azjXdwHHAY/t4/2G/3lU1KOMLZzfLskQEbuAPUsy9DcXuDGqHgAmSTq46KCv05DnGRE/iYiXs5cPUB1L34hq+TsF+CjwTeClIsPVWS3neg7wrYhYDxARjXi+tZxnAOMlCdiPasFpuPWQI+I+qtn3pRl+HhWijAVnGvB8v9fd2bbh7lN2wz2HhVR/i2pEQ56rpGnAWcBXCsyVh1r+Xo8AJku6V9LDkhYUlq5+ajnPq4A/pDrh8FHg4xFRKSZeoZrh51EhyjgPp5YlGUq5bMMw1XwOkk6mWnDekWui/NRyrl8EFkdEX/UX4oZVy7l2AMcDpwBjgPslPRARa/MOV0e1nOepQBfwbuAPgLsl/WdEbMs5W9Ga4edRIcpYcGpZkqGUyzYMU03nIOkY4GvAaRGxuaBs9VbLuc4GlmbFZgpwuqTeiFhWSML6qfXf76aI2AHskHQf8EdAIxWcWs7zAuCKqF7oWCfpGeDNwE+LiViYZvh5VIgydqnVsiTDcmBBNjrkRGBrRGwsOujrNOR5SpoBfAs4t8F++93bkOcaEbMiYmZEzARuBf66AYsN1Pbv93bgnZI6JI0F3gqsKTjn61XLea6n2opD0oHAkcDThaYsRjP8PCpE6Vo4sY8lGSR9KHv/K1RHMZ0OrANepfqbVEOp8Tz/Adgf+HL2m39vNOAKvDWea1Oo5VwjYk22FP1qoAJ8LSIGHHJbVjX+nV4GXC/pUardTosjouFuWSDp34GTgCmSuoFPASOgeX4eFcVL25iZWSHK2KVmZmZNyAXHzMwK4YJjZmaFcMExM7NCuOCYmVkhXHDMzKwQLjhmZlYIFxxrGdm9dlZLGi1pXHaPlqNT5zJrFZ74aS1F0meA0VQXzeyOiP+dOJJZy3DBsZaSrfv1ELATeFtE9CWOZNYy3KVmreYNVG8GNp5qS8fMCuIWjrUUScup3p1yFnBwRHwkcSSzllG61aLN8pLdWbM3Iv5NUjvwE0nvjoh7UmczawVu4ZiZWSF8DcfMzArhgmNmZoVwwTEzs0K44JiZWSFccMzMrBAuOGZmVggXHDMzK8T/A6fUrBh3xqu4AAAAAElFTkSuQmCC\n", "text/plain": [ "
" ] }, "metadata": { "needs_background": "light" }, "output_type": "display_data" } ], "source": [ "# Use chunks to create a 2D histogram\n", "hb = d2(chunked, chunked2, \"fixed_width\", bin_width=.2, axis_names=[\"x\", \"y\"])\n", "check_hb = h2(million, million2, \"fixed_width\", bin_width=.2, axis_names=[\"x\", \"y\"])\n", "hb.plot(show_zero=False, cmap=\"rainbow\")\n", "ok = (hb == check_hb)\n", "print(\"Check: \", ok)\n", "hb" ] }, { "cell_type": "code", "execution_count": 6, "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "Check: True\n" ] }, { "data": { "image/png": "iVBORw0KGgoAAAANSUhEUgAAAagAAAEYCAYAAAAJeGK1AAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjQuMiwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy8rg+JYAAAACXBIWXMAAAsTAAALEwEAmpwYAAAWzUlEQVR4nO3df6zd9X3f8eerdkpIIwg/DKM22WXD6gpobcqV4y7SFNUpeCEKbALJ0VK8zZI1xNZ06pTZ6x9oiTyBNoWWaSChwDA0AyyaCauIpR4kiiYRk0uSlh8O46owuIXFTkwo3QSpyXt/nM9tj28uvuQec8/n3vt8SEfne97f7+dzP+cr2y9/vudzvjdVhSRJvfmZcQ9AkqT5GFCSpC4ZUJKkLhlQkqQuGVCSpC6tHfcATrazzz67JiYmxj0MSdI79MQTT3y/qtbNra+4gJqYmGBqamrcw5AkvUNJ/vd8dS/xSZK6ZEBJkrpkQEmSumRASZK6ZEBJkrpkQEmSumRASZK6ZEBJkrpkQEmSumRASZK6ZEBJkrpkQEmSumRASZK6ZEBJkrpkQEmSumRASZK6ZEBJkrq0YEAluTPJ4SRPzbPvXyepJGcP1XYnmU7ybJLLh+qXJnmy7bslSVr9lCT3t/rBJBNDbbYnea49to/8biVJy8Y7mUHdBWydW0xyPvDrwItDtYuAbcDFrc2tSda03bcBO4GN7THb5w7g1aq6ELgZuKn1dSZwA/BhYBNwQ5Izfrq3J0larhYMqKr6OnB0nl03A58Faqh2JXBfVb1ZVc8D08CmJOcBp1XVY1VVwN3AVUNt9rbtB4AtbXZ1OXCgqo5W1avAAeYJSknSyrR2MY2SfBL4s6r643albtZ64BtDr2da7S/b9tz6bJuXAKrqWJLXgLOG6/O0kVadiV0PLcnPeeHGK5bk50gL+akDKsn7gN8BLptv9zy1OkF9sW3mjmkng8uHfPCDH5zvEOlds1TB0bPFnAODUAtZzAzqbwMXALOzpw3At5JsYjDLOX/o2A3Ay62+YZ46Q21mkqwFTmdwSXEG+OicNl+bb0BVdTtwO8Dk5OS8IabVx+BYHM+bevFTB1RVPQmcM/s6yQvAZFV9P8l+4L8m+QLw8wwWQzxeVW8leT3JZuAgcC3wn1oX+4HtwGPA1cCjVVVJvgL8+6GFEZcBuxfzJiX1x1mXFrJgQCW5l8FM5uwkM8ANVXXHfMdW1dNJ9gHPAMeA66vqrbb7OgYrAk8FHm4PgDuAe5JMM5g5bWt9HU3yeeCb7bjPVdV8izUkSSvQggFVVZ9aYP/EnNd7gD3zHDcFXDJP/Q3gmrfp+07gzoXGKElaebyThCSpSwaUJKlLBpQkqUsGlCSpSwaUJKlLi7rVkbTU/PKotPoYUJKWDb/cu7p4iU+S1CUDSpLUJQNKktQlA0qS1CUDSpLUJQNKktQlA0qS1CUDSpLUJQNKktQlA0qS1CUDSpLUJQNKktQlA0qS1CUDSpLUJQNKktSlBQMqyZ1JDid5aqj2H5J8N8mfJPlvST4wtG93kukkzya5fKh+aZIn275bkqTVT0lyf6sfTDIx1GZ7kufaY/vJetOSpP69kxnUXcDWObUDwCVV9XeB/wXsBkhyEbANuLi1uTXJmtbmNmAnsLE9ZvvcAbxaVRcCNwM3tb7OBG4APgxsAm5IcsZP/xYlScvRggFVVV8Hjs6p/VFVHWsvvwFsaNtXAvdV1ZtV9TwwDWxKch5wWlU9VlUF3A1cNdRmb9t+ANjSZleXAweq6mhVvcogFOcGpSRphToZn0H9M+Dhtr0eeGlo30yrrW/bc+vHtWmh9xpw1gn6+glJdiaZSjJ15MiRkd6MJKkPIwVUkt8BjgFfmi3Nc1idoL7YNscXq26vqsmqmly3bt2JBy1JWhYWHVBt0cIngH/cLtvBYJZz/tBhG4CXW33DPPXj2iRZC5zO4JLi2/UlSVoFFhVQSbYC/wb4ZFX9v6Fd+4FtbWXeBQwWQzxeVa8AryfZ3D5fuhZ4cKjN7Aq9q4FHW+B9BbgsyRltccRlrSZJWgXWLnRAknuBjwJnJ5lhsLJuN3AKcKCtFv9GVf3zqno6yT7gGQaX/q6vqrdaV9cxWBF4KoPPrGY/t7oDuCfJNIOZ0zaAqjqa5PPAN9txn6uq4xZrSJJWrvz11bmVYXJysqampsY9DJ3AxK6Hxj0ErSIv3HjFuIegBSR5oqom59YXnEFJ0nK2mP8QGWp98FZHkqQuGVCSpC4ZUJKkLhlQkqQuGVCSpC4ZUJKkLhlQkqQuGVCSpC4ZUJKkLhlQkqQuGVCSpC4ZUJKkLhlQkqQuGVCSpC4ZUJKkLhlQkqQuGVCSpC4ZUJKkLhlQkqQuGVCSpC4tGFBJ7kxyOMlTQ7UzkxxI8lx7PmNo3+4k00meTXL5UP3SJE+2fbckSaufkuT+Vj+YZGKozfb2M55Lsv2kvWtJUvfeyQzqLmDrnNou4JGq2gg80l6T5CJgG3Bxa3NrkjWtzW3ATmBje8z2uQN4taouBG4Gbmp9nQncAHwY2ATcMByEkqSVbcGAqqqvA0fnlK8E9rbtvcBVQ/X7qurNqnoemAY2JTkPOK2qHquqAu6e02a2rweALW12dTlwoKqOVtWrwAF+MiglSSvUYj+DOreqXgFoz+e0+nrgpaHjZlptfdueWz+uTVUdA14DzjpBX5KkVeBkL5LIPLU6QX2xbY7/ocnOJFNJpo4cOfKOBipJ6ttiA+p77bId7flwq88A5w8dtwF4udU3zFM/rk2StcDpDC4pvl1fP6Gqbq+qyaqaXLdu3SLfkiSpJ4sNqP3A7Kq67cCDQ/VtbWXeBQwWQzzeLgO+nmRz+3zp2jltZvu6Gni0fU71FeCyJGe0xRGXtZokaRVYu9ABSe4FPgqcnWSGwcq6G4F9SXYALwLXAFTV00n2Ac8Ax4Drq+qt1tV1DFYEngo83B4AdwD3JJlmMHPa1vo6muTzwDfbcZ+rqrmLNSRJK9SCAVVVn3qbXVve5vg9wJ556lPAJfPU36AF3Dz77gTuXGiMkqSVxztJSJK6ZEBJkrpkQEmSumRASZK6ZEBJkrpkQEmSumRASZK6ZEBJkrpkQEmSumRASZK6ZEBJkrpkQEmSumRASZK6tODdzKUTmdj10LiHIGmFMqAkaY7F/MfrhRuveBdGsrp5iU+S1CUDSpLUJQNKktQlA0qS1CUDSpLUJQNKktQlA0qS1KWRAirJv0rydJKnktyb5L1JzkxyIMlz7fmMoeN3J5lO8mySy4fqlyZ5su27JUla/ZQk97f6wSQTo4xXkrR8LDqgkqwHfhOYrKpLgDXANmAX8EhVbQQeaa9JclHbfzGwFbg1yZrW3W3ATmBje2xt9R3Aq1V1IXAzcNNixytJWl5GvcS3Fjg1yVrgfcDLwJXA3rZ/L3BV274SuK+q3qyq54FpYFOS84DTquqxqirg7jltZvt6ANgyO7uSJK1siw6oqvoz4D8CLwKvAK9V1R8B51bVK+2YV4BzWpP1wEtDXcy02vq2Pbd+XJuqOga8Bpw1dyxJdiaZSjJ15MiRxb4lSVJHRrnEdwaDGc4FwM8DP5fk0ydqMk+tTlA/UZvjC1W3V9VkVU2uW7fuxAOXJC0Lo1zi+xjwfFUdqaq/BL4M/D3ge+2yHe35cDt+Bjh/qP0GBpcEZ9r23PpxbdplxNOBoyOMWZK0TIwSUC8Cm5O8r30utAU4BOwHtrdjtgMPtu39wLa2Mu8CBoshHm+XAV9Psrn1c+2cNrN9XQ082j6nkiStcIv+dRtVdTDJA8C3gGPAt4HbgfcD+5LsYBBi17Tjn06yD3imHX99Vb3VursOuAs4FXi4PQDuAO5JMs1g5rRtseOVJC0vI/0+qKq6AbhhTvlNBrOp+Y7fA+yZpz4FXDJP/Q1awEmSVhfvJCFJ6pIBJUnqkgElSeqSASVJ6pIBJUnqkgElSeqSASVJ6pIBJUnqkgElSeqSASVJ6pIBJUnqkgElSeqSASVJ6pIBJUnqkgElSeqSASVJ6pIBJUnqkgElSeqSASVJ6pIBJUnqkgElSerSSAGV5ANJHkjy3SSHkvxqkjOTHEjyXHs+Y+j43Ummkzyb5PKh+qVJnmz7bkmSVj8lyf2tfjDJxCjjlSQtH6POoH4P+O9V9XeAXwIOAbuAR6pqI/BIe02Si4BtwMXAVuDWJGtaP7cBO4GN7bG11XcAr1bVhcDNwE0jjleStEwsOqCSnAb8feAOgKr6UVX9ELgS2NsO2wtc1bavBO6rqjer6nlgGtiU5DzgtKp6rKoKuHtOm9m+HgC2zM6uJEkr2ygzqL8FHAH+S5JvJ/likp8Dzq2qVwDa8znt+PXAS0PtZ1ptfdueWz+uTVUdA14Dzpo7kCQ7k0wlmTpy5MgIb0mS1ItRAmot8CvAbVX1IeD/0i7nvY35Zj51gvqJ2hxfqLq9qiaranLdunUnHrUkaVkYJaBmgJmqOtheP8AgsL7XLtvRng8PHX/+UPsNwMutvmGe+nFtkqwFTgeOjjBmSdIyseiAqqr/A7yU5BdaaQvwDLAf2N5q24EH2/Z+YFtbmXcBg8UQj7fLgK8n2dw+X7p2TpvZvq4GHm2fU0mSVri1I7b/l8CXkvws8KfAP2UQevuS7ABeBK4BqKqnk+xjEGLHgOur6q3Wz3XAXcCpwMPtAYMFGPckmWYwc9o24nglScvESAFVVd8BJufZteVtjt8D7JmnPgVcMk/9DVrASZJWF+8kIUnqkgElSeqSASVJ6pIBJUnqkgElSeqSASVJ6pIBJUnq0qhf1NUKMrHroXEPQVq2FvP354Ubr3gXRrJyOIOSJHXJgJIkdcmAkiR1yYCSJHXJgJIkdcmAkiR1yYCSJHXJgJIkdcmAkiR1yYCSJHXJgJIkdcmAkiR1yYCSJHXJgJIkdWnkgEqyJsm3k/xhe31mkgNJnmvPZwwduzvJdJJnk1w+VL80yZNt3y1J0uqnJLm/1Q8mmRh1vJKk5eFkzKA+Axwaer0LeKSqNgKPtNckuQjYBlwMbAVuTbKmtbkN2AlsbI+trb4DeLWqLgRuBm46CeOVJC0DIwVUkg3AFcAXh8pXAnvb9l7gqqH6fVX1ZlU9D0wDm5KcB5xWVY9VVQF3z2kz29cDwJbZ2ZUkaWUbdQb1u8BngR8P1c6tqlcA2vM5rb4eeGnouJlWW9+259aPa1NVx4DXgLPmDiLJziRTSaaOHDky4luSJPVg0QGV5BPA4ap64p02madWJ6ifqM3xharbq2qyqibXrVv3DocjSerZ2hHafgT4ZJKPA+8FTkvy+8D3kpxXVa+0y3eH2/EzwPlD7TcAL7f6hnnqw21mkqwFTgeOjjBmSdIysegZVFXtrqoNVTXBYPHDo1X1aWA/sL0dth14sG3vB7a1lXkXMFgM8Xi7DPh6ks3t86Vr57SZ7evq9jN+YgYlSVp5RplBvZ0bgX1JdgAvAtcAVNXTSfYBzwDHgOur6q3W5jrgLuBU4OH2ALgDuCfJNIOZ07Z3YbySpA6dlICqqq8BX2vbPwC2vM1xe4A989SngEvmqb9BCzhJ0urinSQkSV0yoCRJXTKgJEldMqAkSV0yoCRJXTKgJEldMqAkSV0yoCRJXTKgJEldMqAkSV0yoCRJXTKgJEldMqAkSV0yoCRJXTKgJEldMqAkSV0yoCRJXTKgJEldMqAkSV0yoCRJXVo77gHo3TGx66FxD0HSAhbz9/SFG694F0bSp0XPoJKcn+SrSQ4leTrJZ1r9zCQHkjzXns8YarM7yXSSZ5NcPlS/NMmTbd8tSdLqpyS5v9UPJpkY4b1KkpaRUS7xHQN+u6p+EdgMXJ/kImAX8EhVbQQeaa9p+7YBFwNbgVuTrGl93QbsBDa2x9ZW3wG8WlUXAjcDN40wXknSMrLogKqqV6rqW237deAQsB64EtjbDtsLXNW2rwTuq6o3q+p5YBrYlOQ84LSqeqyqCrh7TpvZvh4AtszOriRJK9tJWSTRLr19CDgInFtVr8AgxIBz2mHrgZeGms202vq2Pbd+XJuqOga8Bpw1z8/fmWQqydSRI0dOxluSJI3ZyAGV5P3AHwC/VVV/fqJD56nVCeonanN8oer2qpqsqsl169YtNGRJ0jIwUkAleQ+DcPpSVX25lb/XLtvRng+3+gxw/lDzDcDLrb5hnvpxbZKsBU4Hjo4yZknS8jDKKr4AdwCHquoLQ7v2A9vb9nbgwaH6trYy7wIGiyEeb5cBX0+yufV57Zw2s31dDTzaPqeSJK1wo3wP6iPAbwBPJvlOq/1b4EZgX5IdwIvANQBV9XSSfcAzDFYAXl9Vb7V21wF3AacCD7cHDALwniTTDGZO20YYryRpGVl0QFXV/2T+z4gAtrxNmz3AnnnqU8Al89TfoAWcJGl18VZHkqQuGVCSpC4ZUJKkLhlQkqQuGVCSpC4ZUJKkLhlQkqQuGVCSpC4ZUJKkLhlQkqQuGVCSpC4ZUJKkLhlQkqQuGVCSpC6N8vugtEQmdj007iFI6sRi/j144cYr3oWRvPucQUmSumRASZK6ZEBJkrpkQEmSumRASZK6ZEBJkrpkQEmSurQsvgeVZCvwe8Aa4ItVdeOYhyRJy8Ziv0s57u9PdR9QSdYA/xn4dWAG+GaS/VX1zHhHtjh+6VaS3pnuAwrYBExX1Z8CJLkPuBIYe0AZNpJWsnHftWI5BNR64KWh1zPAh4cPSLIT2Nle/kWSZ5dobGcD31+in9Urz4HnADwH4DkAIDct6jz8zfmKyyGgMk+tjntRdTtw+9IM568lmaqqyaX+uT3xHHgOwHMAnoNZJ/M8LIdVfDPA+UOvNwAvj2kskqQlshwC6pvAxiQXJPlZYBuwf8xjkiS9y7q/xFdVx5L8C+ArDJaZ31lVT495WLOW/LJihzwHngPwHIDnYNZJOw+pqoWPkiRpiS2HS3ySpFXIgJIkdcmAGkGSa5I8neTHSVbV8tIkW5M8m2Q6ya5xj2ccktyZ5HCSp8Y9lnFJcn6SryY51P4ufGbcY1pqSd6b5PEkf9zOwb8b95jGJcmaJN9O8ocnoz8DajRPAf8I+Pq4B7KUhm4/9Q+Ai4BPJblovKMai7uAreMexJgdA367qn4R2Axcvwr/LLwJ/FpV/RLwy8DWJJvHO6Sx+Qxw6GR1ZkCNoKoOVdVS3bWiJ391+6mq+hEwe/upVaWqvg4cHfc4xqmqXqmqb7Xt1xn847R+vKNaWjXwF+3le9pj1a0+S7IBuAL44snq04DSYsx3+6lV9Y+SflKSCeBDwMExD2XJtUtb3wEOAweqatWdA+B3gc8CPz5ZHRpQC0jyP5I8Nc9j1c0Yhix4+ymtLkneD/wB8FtV9efjHs9Sq6q3quqXGdzpZlOSS8Y8pCWV5BPA4ap64mT22/0Xdcetqj427jF0yNtP6a8keQ+DcPpSVX153OMZp6r6YZKvMfhscjUtnvkI8MkkHwfeC5yW5Per6tOjdOoMSovh7acEQJIAdwCHquoL4x7POCRZl+QDbftU4GPAd8c6qCVWVburakNVTTD49+DRUcMJDKiRJPmHSWaAXwUeSvKVcY9pKVTVMWD29lOHgH0d3X5qySS5F3gM+IUkM0l2jHtMY/AR4DeAX0vynfb4+LgHtcTOA76a5E8Y/OftQFWdlGXWq523OpIkdckZlCSpSwaUJKlLBpQkqUsGlCSpSwaUJKlLBpQkqUsGlCSpSwaU1LEknx/+HUtJ9iT5zXGOSVoqflFX6li7Q/iXq+pXkvwM8Bywqap+MN6RSe8+bxYrdayqXkjygyQfAs4Fvm04abUwoKT+fRH4J8DfAO4c71CkpeMlPqlz7Y7xTzL4Ta0bq+qtMQ9JWhLOoKTOVdWPknwV+KHhpNXEgJI61xZHbAauGfdYpKXkMnOpY0kuAqaBR6rquXGPR1pKfgYlSeqSMyhJUpcMKElSlwwoSVKXDChJUpcMKElSl/4/XQCLjg+e6mkAAAAASUVORK5CYII=\n", "text/plain": [ "
" ] }, "metadata": { "needs_background": "light" }, "output_type": "display_data" } ], "source": [ "# And another cross-check\n", "hh = hb.projection(\"y\")\n", "hh.plot()\n", "print(\"Check: \", np.array_equal(hh.frequencies, ha.frequencies)) # Just frequencies" ] }, { "cell_type": "code", "execution_count": 7, "metadata": {}, "outputs": [ { "data": { "text/plain": [ "True" ] }, "execution_count": 7, "metadata": {}, "output_type": "execute_result" } ], "source": [ "# Use dask for normal arrays (will automatically split array to chunks)\n", "d1(million2, \"fixed_width\", bin_width=0.2) == ha" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "## Some timings\n", "\n", "Your results may vary substantially. These numbers are just for illustration, on 4-core (8-thread) machine. The real gain comes when we have data that don't fit into memory." ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "### Efficiency" ] }, { "cell_type": "code", "execution_count": 8, "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "CPU times: user 412 ms, sys: 14.2 ms, total: 427 ms\n", "Wall time: 426 ms\n" ] }, { "data": { "text/plain": [ "Histogram1D(bins=(28,), total=2097152, dtype=int64)" ] }, "execution_count": 8, "metadata": {}, "output_type": "execute_result" } ], "source": [ "# Standard\n", "%time h1(million2, \"fixed_width\", bin_width=0.2)" ] }, { "cell_type": "code", "execution_count": 9, "metadata": { "scrolled": true }, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "CPU times: user 751 ms, sys: 24.3 ms, total: 775 ms\n", "Wall time: 199 ms\n" ] }, { "data": { "text/plain": [ "Histogram1D(bins=(28,), total=2097152, dtype=int64)" ] }, "execution_count": 9, "metadata": {}, "output_type": "execute_result" } ], "source": [ "# Same array, but using dask\n", "%time d1(million2, \"fixed_width\", bin_width=0.2)" ] }, { "cell_type": "code", "execution_count": 10, "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "CPU times: user 685 ms, sys: 13.5 ms, total: 699 ms\n", "Wall time: 153 ms\n" ] }, { "data": { "text/plain": [ "Histogram1D(bins=(28,), total=2097152, dtype=int64)" ] }, "execution_count": 10, "metadata": {}, "output_type": "execute_result" } ], "source": [ "# Most efficient: dask with already chunked data\n", "%time d1(chunked2, \"fixed_width\", bin_width=0.2)" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "### Different scheduling" ] }, { "cell_type": "code", "execution_count": 11, "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "CPU times: user 573 ms, sys: 37.6 ms, total: 611 ms\n", "Wall time: 135 ms\n" ] }, { "data": { "text/plain": [ "Histogram1D(bins=(28,), total=2097152, dtype=int64)" ] }, "execution_count": 11, "metadata": {}, "output_type": "execute_result" } ], "source": [ "%time d1(chunked2, \"fixed_width\", bin_width=0.2)" ] }, { "cell_type": "code", "execution_count": 12, "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "CPU times: user 442 ms, sys: 11.1 ms, total: 453 ms\n", "Wall time: 140 ms\n" ] }, { "data": { "text/plain": [ "Histogram1D(bins=(28,), total=2097152, dtype=int64)" ] }, "execution_count": 12, "metadata": {}, "output_type": "execute_result" } ], "source": [ "%%time\n", "# Hyper-threading or not?\n", "graph, name = d1(chunked2, \"fixed_width\", bin_width=0.2, compute=False)\n", "dask.threaded.get(graph, name, num_workers=4)" ] }, { "cell_type": "code", "execution_count": 13, "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "CPU times: user 42.6 ms, sys: 158 ms, total: 201 ms\n", "Wall time: 2.28 s\n" ] }, { "data": { "text/plain": [ "Histogram1D(bins=(28,), total=2097152, dtype=int64)" ] }, "execution_count": 13, "metadata": {}, "output_type": "execute_result" } ], "source": [ "# Multiprocessing not so efficient for small arrays?\n", "%time d1(chunked2, \"fixed_width\", bin_width=0.2, dask_method=dask.multiprocessing.get)" ] } ], "metadata": { "anaconda-cloud": {}, "kernelspec": { "display_name": "Python 3", "language": "python", "name": "python3" }, "language_info": { "codemirror_mode": { "name": "ipython", "version": 3 }, "file_extension": ".py", "mimetype": "text/x-python", "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", "version": "3.8.5" } }, "nbformat": 4, "nbformat_minor": 1 }