{ "cells": [ { "cell_type": "markdown", "metadata": {}, "source": [ "# Support for dask arrays\n", "\n", "It is possible to operate on dask arrays and spare the memory (or perhaps even time)." ] }, { "cell_type": "code", "execution_count": 1, "metadata": {}, "outputs": [], "source": [ "# Necessary imports\n", "import dask\n", "import dask.multiprocessing\n", "import physt\n", "import numpy as np\n", "\n", "import dask.array as da\n", "from physt import h1, h2\n", "%matplotlib inline" ] }, { "cell_type": "code", "execution_count": 2, "metadata": {}, "outputs": [], "source": [ "# Create two arrays\n", "np.random.seed(42)\n", "\n", "SIZE = 2 ** 21\n", "CHUNK = int(SIZE / 16)\n", "\n", "million = np.random.rand(SIZE)#.astype(int)\n", "million2 = (3 * million + np.random.normal(0., 0.3, SIZE))#.astype(int)\n", "\n", "# Chunk them for dask\n", "chunked = da.from_array(million, chunks=(CHUNK))\n", "chunked2 = da.from_array(million2, chunks=(CHUNK))" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "## Create histograms\n", "\n", "`h1`, `h2`, ... have their alternatives in `physt.dask_compat`. They should work similarly. Although, they are not complete and unexpected errors may occur." ] }, { "cell_type": "code", "execution_count": 3, "metadata": {}, "outputs": [], "source": [ "from physt.compat.dask import h1 as d1\n", "from physt.compat.dask import h2 as d2" ] }, { "cell_type": "code", "execution_count": 4, "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "Check: True\n" ] }, { "data": { "text/plain": [ "Histogram1D(bins=(28,), total=2097152, dtype=int64)" ] }, "execution_count": 4, "metadata": {}, "output_type": "execute_result" }, { "data": { "image/png": "iVBORw0KGgoAAAANSUhEUgAAAagAAAEYCAYAAAAJeGK1AAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4xLjMsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+AADFEAAAVs0lEQVR4nO3df6xcZ53f8fenNuHHrkISYlBqmzotFt0QdRewgrdIFSJs4iQIpysiJWo3FrVklYaWrbYCp/uHVSCVUavNbiqI5BI3DkWYKEsVaxPWdUMiVCkJcQibHxjq25Amd5Nisw7ZbBFQw7d/zHPZqT32jWecO8/1fb+k0ZzzPc8588yR7/34OfPMuakqJEnqzd+YdgckSRrFgJIkdcmAkiR1yYCSJHXJgJIkdWn5tDtwup1//vm1Zs2aaXdDkvQKPfrooz+sqhXH1s+4gFqzZg379++fdjckSa9Qkv81qu4lPklSlwwoSVKXDChJUpcMKElSlwwoSVKXDChJUpcMKElSlwwoSVKXDChJUpcMKElSlwwoSVKXDChJUpcMKElSlwwoSVKXDChJUpcMKElSlwwoSVKXDChJUpcMKElSl+YNqCQ7kxxK8uSIbf8qSSU5v60nyS1JZpI8nuRdQ203JTnYHpuG6u9O8kTb55YkafXzkuxr7fclOff0vGVJ0mLwSkZQtwMbji0mWQ38FvDsUPkKYG17bAFubW3PA7YB7wEuAbYNBc6tre3cfnOvtRW4r6rWAve1dUnSErF8vgZV9Y0ka0Zsuhn4BHD3UG0jcEdVFfBQknOSXAC8D9hXVUcAkuwDNiR5ADi7qh5s9TuAq4GvtWO9rx13F/AA8MlTenfSGWTN1ntOeZ9ntl/1KvREWhjzBtQoST4E/HlV/Vm7IjdnJfDc0Ppsq52sPjuiDvCWqnoBoKpeSPLmk/RnC4NRGG9961vHeUvS2MYJjoWyUKFmeOrVcMoBleQNwO8Dl43aPKJWY9RPSVXtAHYArFu37pT3l+b0HDYLxXOgXowzgvo7wIXA3OhpFfCtJJcwGAGtHmq7Cni+1d93TP2BVl81oj3AD5Jc0EZPFwCHxuirljB/0UqL2ykHVFU9AfzycluSZ4B1VfXDJHuAjyXZzWBCxEstYPYC/3ZoYsRlwI1VdSTJy0nWAw8D1wP/obXZA2wCtrfn4c+6JC1yXhbUfF7JNPMvAw8Cb08ym2TzSZrfCzwNzAD/EfhnAG1yxKeBR9rjU3MTJoCPAl9o+/xPBhMkYBBMv5XkIIPZgttP7a1JkhazVzKL77p5tq8ZWi7ghhO02wnsHFHfD1w8ov4XwKXz9U+SdGbyThKSpC4ZUJKkLhlQkqQuGVCSpC4ZUJKkLo11qyNpIfmFW2lpMqAkLRp+uXdp8RKfJKlLBpQkqUsGlCSpSwaUJKlLBpQkqUsGlCSpSwaUJKlLBpQkqUsGlCSpSwaUJKlLBpQkqUsGlCSpSwaUJKlLBpQkqUsGlCSpSwaUJKlL8wZUkp1JDiV5cqj275J8N8njSf5LknOGtt2YZCbJ95JcPlTf0GozSbYO1S9M8nCSg0m+kuSsVn9tW59p29ecrjctSerfKxlB3Q5sOKa2D7i4qv4e8D+AGwGSXARcC7yj7fP5JMuSLAM+B1wBXARc19oCfBa4uarWAi8Cm1t9M/BiVb0NuLm1kyQtEfMGVFV9AzhyTO2/VtXRtvoQsKotbwR2V9VPq+r7wAxwSXvMVNXTVfUzYDewMUmA9wN3tf13AVcPHWtXW74LuLS1lyQtAafjM6h/AnytLa8EnhvaNttqJ6q/CfjRUNjN1f+/Y7XtL7X2x0myJcn+JPsPHz488RuSJE3fRAGV5PeBo8CX5kojmtUY9ZMd6/hi1Y6qWldV61asWHHyTkuSFoXl4+6YZBPwQeDSqpoLjllg9VCzVcDzbXlU/YfAOUmWt1HScPu5Y80mWQ68kWMuNUqSzlxjjaCSbAA+CXyoqn48tGkPcG2bgXchsBb4JvAIsLbN2DuLwUSKPS3Y7gc+3PbfBNw9dKxNbfnDwNeHglCSdIabdwSV5MvA+4Dzk8wC2xjM2nstsK/NW3ioqv5pVT2V5E7gOwwu/d1QVT9vx/kYsBdYBuysqqfaS3wS2J3kM8BjwG2tfhvwxSQzDEZO156G9ytJWiTmDaiqum5E+bYRtbn2NwE3jajfC9w7ov40g1l+x9Z/AlwzX/8kSWemsT+DksaxZus90+6Clphx/s09s/2qV6EnOlXe6kiS1CUDSpLUJQNKktQlA0qS1CUDSpLUJQNKktQlA0qS1CUDSpLUJQNKktQlA0qS1CUDSpLUJQNKktQlA0qS1CUDSpLUJQNKktQlA0qS1CUDSpLUJQNKktQlA0qS1CUDSpLUJQNKktQlA0qS1KV5AyrJziSHkjw5VDsvyb4kB9vzua2eJLckmUnyeJJ3De2zqbU/mGTTUP3dSZ5o+9ySJCd7DUnS0vBKRlC3AxuOqW0F7quqtcB9bR3gCmBte2wBboVB2ADbgPcAlwDbhgLn1tZ2br8N87yGJGkJmDegquobwJFjyhuBXW15F3D1UP2OGngIOCfJBcDlwL6qOlJVLwL7gA1t29lV9WBVFXDHMcca9RqSpCVg3M+g3lJVLwC05ze3+krguaF2s612svrsiPrJXuM4SbYk2Z9k/+HDh8d8S5KknpzuSRIZUasx6qekqnZU1bqqWrdixYpT3V2S1KFxA+oH7fIc7flQq88Cq4farQKen6e+akT9ZK8hSVoCxg2oPcDcTLxNwN1D9evbbL71wEvt8txe4LIk57bJEZcBe9u2l5Osb7P3rj/mWKNeQ5K0BCyfr0GSLwPvA85PMstgNt524M4km4FngWta83uBK4EZ4MfARwCq6kiSTwOPtHafqqq5iRcfZTBT8PXA19qDk7yGJGkJmDegquq6E2y6dETbAm44wXF2AjtH1PcDF4+o/8Wo15AkLQ3eSUKS1CUDSpLUJQNKktQlA0qS1CUDSpLUJQNKktQlA0qS1CUDSpLUJQNKktQlA0qS1CUDSpLUJQNKktQlA0qS1CUDSpLUpXn/3IZ0Imu23jPtLkg6gxlQknSMcf7z9cz2q16FnixtXuKTJHXJgJIkdcmAkiR1yYCSJHXJgJIkdcmAkiR1yYCSJHXJgJIkdWmigEryL5M8leTJJF9O8rokFyZ5OMnBJF9JclZr+9q2PtO2rxk6zo2t/r0klw/VN7TaTJKtk/RVkrS4jB1QSVYC/wJYV1UXA8uAa4HPAjdX1VrgRWBz22Uz8GJVvQ24ubUjyUVtv3cAG4DPJ1mWZBnwOeAK4CLgutZWkrQETHqJbznw+iTLgTcALwDvB+5q23cBV7fljW2dtv3SJGn13VX106r6PjADXNIeM1X1dFX9DNjd2kqSloCxA6qq/hz498CzDILpJeBR4EdVdbQ1mwVWtuWVwHNt36Ot/ZuG68fsc6L6cZJsSbI/yf7Dhw+P+5YkSR2Z5BLfuQxGNBcCfxP4FQaX445Vc7ucYNup1o8vVu2oqnVVtW7FihXzdV2StAhMconvA8D3q+pwVf1f4KvA3wfOaZf8AFYBz7flWWA1QNv+RuDIcP2YfU5UlyQtAZME1LPA+iRvaJ8lXQp8B7gf+HBrswm4uy3vaeu07V+vqmr1a9ssvwuBtcA3gUeAtW1W4FkMJlLsmaC/kqRFZOy/B1VVDye5C/gWcBR4DNgB3APsTvKZVrut7XIb8MUkMwxGTte24zyV5E4G4XYUuKGqfg6Q5GPAXgYzBHdW1VPj9leStLhM9AcLq2obsO2Y8tMMZuAd2/YnwDUnOM5NwE0j6vcC907SR0nS4uSdJCRJXTKgJEldMqAkSV0yoCRJXTKgJEldMqAkSV0yoCRJXTKgJEldMqAkSV0yoCRJXTKgJEldMqAkSV0yoCRJXTKgJEldMqAkSV0yoCRJXTKgJEldMqAkSV0yoCRJXTKgJEldMqAkSV0yoCRJXTKgJEldmiigkpyT5K4k301yIMlvJjkvyb4kB9vzua1tktySZCbJ40neNXScTa39wSSbhurvTvJE2+eWJJmkv5KkxWPSEdQfAX9aVX8X+HXgALAVuK+q1gL3tXWAK4C17bEFuBUgyXnANuA9wCXAtrlQa222DO23YcL+SpIWibEDKsnZwD8AbgOoqp9V1Y+AjcCu1mwXcHVb3gjcUQMPAeckuQC4HNhXVUeq6kVgH7ChbTu7qh6sqgLuGDqWJOkMN8kI6m8Dh4H/lOSxJF9I8ivAW6rqBYD2/ObWfiXw3ND+s612svrsiPpxkmxJsj/J/sOHD0/wliRJvZgkoJYD7wJurap3Av+Hv76cN8qoz49qjPrxxaodVbWuqtatWLHi5L2WJC0KkwTULDBbVQ+39bsYBNYP2uU52vOhofarh/ZfBTw/T33ViLokaQkYO6Cq6n8DzyV5eytdCnwH2APMzcTbBNzdlvcA17fZfOuBl9olwL3AZUnObZMjLgP2tm0vJ1nfZu9dP3QsSdIZbvmE+/9z4EtJzgKeBj7CIPTuTLIZeBa4prW9F7gSmAF+3NpSVUeSfBp4pLX7VFUdacsfBW4HXg98rT0kSUvARAFVVd8G1o3YdOmItgXccILj7AR2jqjvBy6epI+SpMXJO0lIkrpkQEmSumRASZK6ZEBJkrpkQEmSumRASZK6ZEBJkro06Rd1dYZYs/WeaXdBWtTG+Rl6ZvtVr0JPzhyOoCRJXTKgJEldMqAkSV0yoCRJXTKgJEldMqAkSV0yoCRJXTKgJEldMqAkSV0yoCRJXTKgJEldMqAkSV0yoCRJXTKgJEldMqAkSV2aOKCSLEvyWJI/aesXJnk4ycEkX0lyVqu/tq3PtO1rho5xY6t/L8nlQ/UNrTaTZOukfZUkLR6nYwT1ceDA0PpngZurai3wIrC51TcDL1bV24CbWzuSXARcC7wD2AB8voXeMuBzwBXARcB1ra0kaQmYKKCSrAKuAr7Q1gO8H7irNdkFXN2WN7Z12vZLW/uNwO6q+mlVfR+YAS5pj5mqerqqfgbsbm0lSUvApCOoPwQ+Afyirb8J+FFVHW3rs8DKtrwSeA6gbX+ptf9l/Zh9TlQ/TpItSfYn2X/48OEJ35IkqQdjB1SSDwKHqurR4fKIpjXPtlOtH1+s2lFV66pq3YoVK07Sa0nSYrF8gn3fC3woyZXA64CzGYyozkmyvI2SVgHPt/azwGpgNsly4I3AkaH6nOF9TlSXJJ3hxh5BVdWNVbWqqtYwmOTw9ar6R8D9wIdbs03A3W15T1unbf96VVWrX9tm+V0IrAW+CTwCrG2zAs9qr7Fn3P5KkhaXSUZQJ/JJYHeSzwCPAbe1+m3AF5PMMBg5XQtQVU8luRP4DnAUuKGqfg6Q5GPAXmAZsLOqnnoV+itJ6tBpCaiqegB4oC0/zWAG3rFtfgJcc4L9bwJuGlG/F7j3dPRRkrS4eCcJSVKXDChJUpcMKElSlwwoSVKXDChJUpcMKElSlwwoSVKXDChJUpcMKElSlwwoSVKXDChJUpcMKElSlwwoSVKXDChJUpcMKElSlwwoSVKXDChJUpcMKElSlwwoSVKXDChJUpcMKElSlwwoSVKXlk+7Azr91my9Z9pdkPQKjPOz+sz2q16FnvRp7BFUktVJ7k9yIMlTST7e6ucl2ZfkYHs+t9WT5JYkM0keT/KuoWNtau0PJtk0VH93kifaPrckySRvVpK0eExyie8o8HtV9WvAeuCGJBcBW4H7qmotcF9bB7gCWNseW4BbYRBowDbgPcAlwLa5UGtttgztt2GC/kqSFpGxA6qqXqiqb7Xll4EDwEpgI7CrNdsFXN2WNwJ31MBDwDlJLgAuB/ZV1ZGqehHYB2xo286uqgerqoA7ho4lSTrDnZZJEknWAO8EHgbeUlUvwCDEgDe3ZiuB54Z2m221k9VnR9QlSUvAxAGV5FeBPwZ+t6r+8mRNR9RqjPqoPmxJsj/J/sOHD8/XZUnSIjBRQCV5DYNw+lJVfbWVf9Auz9GeD7X6LLB6aPdVwPPz1FeNqB+nqnZU1bqqWrdixYpJ3pIkqROTzOILcBtwoKr+YGjTHmBuJt4m4O6h+vVtNt964KV2CXAvcFmSc9vkiMuAvW3by0nWt9e6fuhYkqQz3CTfg3ov8DvAE0m+3Wr/GtgO3JlkM/AscE3bdi9wJTAD/Bj4CEBVHUnyaeCR1u5TVXWkLX8UuB14PfC19pAkLQFjB1RV/XdGf04EcOmI9gXccIJj7QR2jqjvBy4et4+SpMXLWx1JkrpkQEmSumRASZK6ZEBJkrpkQEmSumRASZK6ZEBJkrpkQEmSumRASZK6ZEBJkrpkQEmSumRASZK6ZEBJkrpkQEmSumRASZK6NMkfLNQCWLP1nml3QVJHxvmd8Mz2q16Fnrz6HEFJkrpkQEmSumRASZK6ZEBJkrpkQEmSumRASZK6ZEBJkrpkQEmSutT9F3WTbAD+CFgGfKGqtk+5S2PzS7eSpmGxfrm36xFUkmXA54ArgIuA65JcNN1eSZIWQu8jqEuAmap6GiDJbmAj8J2p9gpHQ5LObOP+jjudI6/eA2ol8NzQ+izwnmMbJdkCbGmrf5XkewvQt/OBHy7A6/TMczDgefAcgOcAgHx2rPPwt0YVew+ojKjVcYWqHcCOV787fy3J/qpat5Cv2RvPwYDnwXMAnoM5p/M8dP0ZFIMR0+qh9VXA81PqiyRpAfUeUI8Aa5NcmOQs4Fpgz5T7JElaAF1f4quqo0k+BuxlMM18Z1U9NeVuzVnQS4qd8hwMeB48B+A5mHPazkOqjvtIR5Kkqev9Ep8kaYkyoCRJXTKgJpDkmiRPJflFkiU1vTTJhiTfSzKTZOu0+zMNSXYmOZTkyWn3ZVqSrE5yf5ID7Wfh49Pu00JL8rok30zyZ+0c/Jtp92lakixL8liSPzkdxzOgJvMk8NvAN6bdkYXkLah+6XZgw7Q7MWVHgd+rql8D1gM3LMF/Cz8F3l9Vvw78BrAhyfop92laPg4cOF0HM6AmUFUHqmoh7lrRm1/egqqqfgbM3YJqSamqbwBHpt2PaaqqF6rqW235ZQa/nFZOt1cLqwb+qq2+pj2W3OyzJKuAq4AvnK5jGlAax6hbUC2pX0o6XpI1wDuBh6fbk4XXLm19GzgE7KuqJXcOgD8EPgH84nQd0ICaR5L/luTJEY8lN2IY8opuQaWlI8mvAn8M/G5V/eW0+7PQqurnVfUbDO52c0mSi6fdp4WU5IPAoap69HQet+sv6vagqj4w7T50yFtQ6ZeSvIZBOH2pqr467f5MU1X9KMkDDD6bXEqTZ94LfCjJlcDrgLOT/Oeq+seTHNQRlMbhLagEQJIAtwEHquoPpt2faUiyIsk5bfn1wAeA7063Vwurqm6sqlVVtYbB74OvTxpOYEBNJMk/TDIL/CZwT5K90+7TQqiqo8DcLagOAHd2dAuqBZPky8CDwNuTzCbZPO0+TcF7gd8B3p/k2+1x5bQ7tcAuAO5P8jiD/7ztq6rTMs16qfNWR5KkLjmCkiR1yYCSJHXJgJIkdcmAkiR1yYCSJHXJgJIkdcmAkiR16f8Bb8w8llaIwrIAAAAASUVORK5CYII=\n", "text/plain": [ "
" ] }, "metadata": { "needs_background": "light" }, "output_type": "display_data" } ], "source": [ "# Use chunks to create a 1D histogram\n", "ha = d1(chunked2, \"fixed_width\", 0.2)\n", "check_ha = h1(million2, \"fixed_width\", 0.2)\n", "ok = (ha == check_ha)\n", "print(\"Check: \", ok)\n", "ha.plot()\n", "ha" ] }, { "cell_type": "code", "execution_count": 5, "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "Check: True\n" ] }, { "data": { "text/plain": [ "Histogram2D(bins=(5, 28), total=2097152, dtype=int64)" ] }, "execution_count": 5, "metadata": {}, "output_type": "execute_result" }, { "data": { "image/png": "iVBORw0KGgoAAAANSUhEUgAAAZwAAAEYCAYAAABoYED3AAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4xLjMsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+AADFEAAAeoElEQVR4nO3dfbRdVXnv8e/vnJN38gZRgYSQOAi2yHDwkipWqyCKARwkdKBFKgkMvBm1oLZKBbH34hWxtLeWyjWi0aSApQ1IBYJG0xRIqV6hRMiNQCQEhHDknUCIiXk5Zz/3j72O95iel31grzXX3uv3GWON7DX33Gs/MznJkznXnHMpIjAzM8tbR+oAzMysGpxwzMysEE44ZmZWCCccMzMrhBOOmZkVoit1ACMxbdq0mDVrVuowzGwY23bvpmdP+8+AfXzTgy9ExOuaca15UrzQYN2fwuqImNeM7y1SSyWcWbNmsW7dutRhmNkwrlm7iSdOODx1GLn7PHqiWdd6AWj0XzfBtGZ9b5FaKuGYmbW1zgbvcvTW8o0jJ044ZmZlIKBTjdXtzTWS3DjhmJmVghrv4bRoxnHCMTMrgw5gTGdjdX+9N9dQ8pI84UjqpH6v7JcR8YHU8ZiZJdPokFqLSp5wgE8CG4FJqQMxM0tGIxlSa01JWydpBnAq8K2UcZiZJSfqCaeRo0Wl7uH8PfAZYOJgFSQtBhYDzJw5s6CwzOy12DHlV4y5eX3qMPJ3epOv5yG1fEj6APBcRPxU0vGD1YuIpcBSgLlz57b/0mWzNjDh5f144fT2X/jZVBUYUkvZw3kHcJqkU4CxwCRJ/xgRH0kYk5lZOm3ew0mWTiPisxExIyJmAWcCdzjZmFllCejqaOxoUanv4ZiZGWRDau3dwylFwomItcDaxGGYmaXlezhmZpa7vmnRbcwJx8ysDDykZmbNcttjm3lxS2tuKz9Sz4zf6nU4IyVgdIN7qbUoJxyzgry4pVaJh5IBjFq5nu0Lj0odRovxOhwzMyvCSJ6H06KccMzMysCTBszMrDDu4ZiZWe68l5qZmRXGCcfMzHLnSQNmZlYID6mZWbNU5qFkwO6DdzJhRQXaemqTr+cejpk1Q5UeSla7fT2bvvSW1GG0Fk+LNjOzQkje2sbMzArgSQNmZlYMTxowM7MiuIdjZmaF8KQBMzMrhofUzKxJqrQO5+XpOzjkyxVo63FNvJaAjuYNqUn6c+CjQAA/A84FDgJWAPsD9wFnR8QeSWOA64BjgReBP4qIx7PrfBY4D+gFPhERq7PyecBXgE7gWxFxxXAxOeGYFaRK63Be2bSOZU/NSh1G62lSD0fSdOATwBER8WtJNwJnAqcAV0bECklfp55Irs5+fSkiDpN0JvDXwB9JOiL73JuBg4F/k9T3Q7wEeB/QDdwraWVEPDRUXO3dfzMzaxV9kwYaORrTBYyT1AWMB54G3gPclL1/LbAgez0/Oyd7/0RJyspXRMTuiPgFsBl4a3ZsjojHImIP9V7T/OECcsIxMyuF7B5OIwdMk7Su37G4/5Ui4pfA3wJbqCeabcBPgZcjoier1g1Mz15PB57MPtuT1T+gf/k+nxmsfEgeUjMzK4ORTYt+ISLmDnopaSr1Hsds4GXgO8DJA1SNft8+0HuDlQ/UWYkByn6LE46ZWVl0NG3Q6b3ALyLieQBJ3wV+H5giqSvrxcwAnsrqdwOHAN3ZENxkYGu/8j79PzNY+aA8pGZmVgYd2V5qjRzD2wIcJ2l8di/mROAh4E7gjKzOIuDW7PXK7Jzs/TsiIrLyMyWNkTQbmAP8J3AvMEfSbEmjqU8sWDlcUO7hmJmVRZN2GoiIeyTdRH3qcw9wP7AU+D6wQtIXs7Jl2UeWAd+WtJl6z+bM7DoPZjPcHsquc35E9AJIugBYTX1a9PKIeHC4uJxwLKnbHtvMi1tqqcMoxDPjt1ZmHc74qdv56Dt/njqM3H2tmReTmjmkRkRcCly6T/Fj1GeY7Vt3F/DBQa5zOXD5AOWrgFUjickJx5J6cUuNJ06oxtqUUSvXs33hUanDKMSBW9YwevKLqcNoPd5LzczMcue91MzMrDBN3NqmjJxwzMzKQN6808zMiuIeTj4kjQXuAsZkcdyUzaowM6se38PJ1W7gPRHxK0mjgB9J+kFE3J0wJjOzROQeTl6yVay/yk5HZcewe/GYmbUl93DyJamT+g6mhwFLIuKeAeosBhYDzJw5s9gALXdVeijZ7oN3MmFFNdqKdtDx64a2YLE+Eoxq79+zpAkn2yLhKElTgJslHRkRD+xTZyn1LRmYO3eue0BtpkoPJYvb1/PwZW9JHUYhTr76fs76x2XDV2xxH2/2Bdt8SK0U/beIeBlYC8xLHIqZWRp9Q2qNPQ+nJSWLXNLrsp4NksZR3067/TdfMjMbUDZpoJGjRaUcUjsIuDa7j9MB3BgR30sYj5lZOqKpm3eWUcpZahuAo1N9v5lZ6XjzTjMzy12TH09QRk44ZmZl0cL3ZxrhhGNJVWkdzssH72D6ldVo6/bxwS0fPmP4iq3uyz9u3rWEh9TM8lSldTjbN61j+S9npw6jEF//1V0sOPXC1GG0Hg+pmZlZ7iRqHlIzM7O8BVBzD8fMzPIWEnu9l5qZmRXBQ2pmZpY/QXhIzczM8la/h+MejpmZ5c2z1CyV2x7bzItbaqnDyN0z47dWZuHnuKnbOe8PqrEh+q7nxnHLrf8zdRj5O/kvm3Ypz1KzZF7cUuOJE9p/QeSolevZvvCo1GEU4g1b1jBq8lOpwyjElG2wYJU3fx8p93DMzCx3IdEr93DMzKwA7uGYmVkhnHDMzCx34XU4ZmZWCImeLm9tY2ZmOQugJg+pWQJVeTDZ7oN3MmFF+7cTQNpB16/be8ikz9bJE7jh4+ekDiN///vupl7O93Asiao8mKx2+3o2fektqcMoxKlX3cdZ13wzdRiF+OoVn+Hp141LHUZLCckLP83MrBi9HlIzM7O8eWsbMzMriAj3cMzMLHfypAEzMytAfVp0ew+ptXfrzMxaSK1DDR2NkDRF0k2Sfi5po6S3S9pf0hpJj2S/Ts3qStJVkjZL2iDpmH7XWZTVf0TSon7lx0r6WfaZq6ThxwPdwympqqzDeXn6Dg75cvu3E2D7BLjl7A+lDqMQL0UXHXvHpA6jtUjNXvj5FeCHEXGGpNHAeOAS4PaIuELSxcDFwEXAycCc7HgbcDXwNkn7A5cCc6l3wn4qaWVEvJTVWQzcDawC5gE/GCogJ5ySqso6nFc2rWPZU7NSh1GIpdv+nQXv/1TqMApx16Y1LHngmOEr2m8EsLezOVvbSJoEvAs4ByAi9gB7JM0Hjs+qXQuspZ5w5gPXRUQAd2e9o4OyumsiYmt23TXAPElrgUkR8ZOs/DpgAcMkHA+pmZmVQN/WNo0cDXgj8DzwD5Lul/QtSROAN0TE0wDZr6/P6k8Hnuz3+e6sbKjy7gHKh+SEY2ZWBhLR0dHQAUyTtK7fsXifq3UBxwBXR8TRwA7qw2eDfvsAZfEqyofkITUzs5IYwT2cFyJi7hDvdwPdEXFPdn4T9YTzrKSDIuLpbMjsuX71D+n3+RnAU1n58fuUr83KZwxQf0ju4ZiZlUAzh9Qi4hngSUlvyopOBB4CVgJ9M80WAbdmr1cCC7PZascB27Iht9XASZKmZjPaTgJWZ+9tl3RcNjttYb9rDSpZD0fSIcB1wIFADVgaEV9JFY+ZWWpNnqX2ceD6bIbaY8C51DsZN0o6D9gCfDCruwo4BdgM7MzqEhFbJV0G3JvV+0LfBALgY8A1wDjqkwWGnDAAaYfUeoBPR8R9kiZSn263JiIeShiTmVkSITV14WdErKc+nXlfJw5QN4DzB7nOcmD5AOXrgCNHElOyhJN1yfpmS2yXtJH6LAcnHDOrJD+ArQCSZgFHA/cM8N5i6ouLeN3BB3PN2k2FxpbKM+O3VmLh5/ip2/noO3+eOoxC7Hp2HLesuix1GIXoHAvnHvVo6jBy940mXiuAXu+lli9J+wH/AvxZRLyy7/sRsRRYCjD7TUfGEye0/2JIgNG3rudXZx2VOozcHfjUGrombx2+YhuYsq2DBbf/MHUYhdg46d08ddGxqcNoLU0eUiujpAlH0ijqyeb6iPhuyljMzFLz4wlykk2lWwZsjIi/SxWHmVkZBFAbcD1l+0jZw3kHcDbwM0l9NysuiYhVCWMyM0tE9HQ0Zy+1sko5S+1HDLw9gplZ5fQt/GxnyScNmJkZIOh1wjEzs7y5h1MyvZ21SqxNAdh98E7GfacKbd1B16/beypon62Tx3PDnyxMHUYh9uyucdhlG1KHkb/3NPNiotbm21u2VMLp7O1g9+ntvzYFIG5fz8NffEvqMHJ38pL7OOuab6YOoxBLrvgMzx8wOnUYhRj94/G8Mr8af1ebydOizcwsdx5SMzOzwngdjpmZ5S7w1jZmZlYQ93DMzCx3Id/DMTOzAgRir4fUzMysCOEhtfKo0sLPlw/ewfQr27+t2yfALWd/KHUYhXgpuujoGZ86jEL0TNzNxOvb/+eX05p7OU8aKJEqLfx8ZdM6lj09K3UYuVv6yr+z4P2fSh1GIe7atIYlDxyTOoxCXDLhCfaeVo2/q83ixxOYmVlB5IRjZmbFcMIxM7PcBX48gZmZFcQ9HDMzy11U4PEEw7ZO0gWSphYRjJlZlQVq6GhVjfRwDgTulXQfsBxYHRGRb1gDq9I6nPFTt/PRd/48dRi52/XsOG5ZdVnqMArRORbOPerR1GEU48U91fi7enpzL1f5IbWI+EtJ/x04CTgX+KqkG4FlEVHo356OWgfbFlZjbv+BW/6VsZOeTx1G7qZs62DB7T9MHUYhNk56N89ceGzqMAox6pINlVkz1yxVWIfT0IBh1qN5Jjt6gKnATZL+JsfYzMwqIxB76WjoaFXD9nAkfQJYBLwAfAv4i4jYK6kDeAT4TL4hmplVQyvfn2lEI/dwpgF/GBFP9C+MiJqkD+QTlplZ9bT7kFoj93D+xxDvbWxuOGZm1RRAb1Q84ZiZWTEq38MxM7P8tfoam0Y44ZiZlUS77zTQUgmn1lljwooKLCYDYCfaNSp1ELnbOnk8N/zJwtRhFGLP7hpvvHxD6jAK0Tthpxd+vgo138MpD0UHG7/0ltRhFOLUq+7jrGu+mTqM3H31is/w7LSxqcMoxOgfjeeVBdVYDDnuuvVe+DlCAfR6SM3MzPInwj0cMzPLWxW2tnHCMTMrgQjYG+09aSBp6yQtl/ScpAdSxmFmVgYRauhoVanT6TXAvMQxmJmVgKg1eDR8RalT0v2Svpedz5Z0j6RHJN0gaXRWPiY735y9P6vfNT6blT8s6f39yudlZZslXdxIPEkTTkTcBWxNGYOZWRkE9WnRjRwj8Emg/xZkfw1cGRFzgJeA87Ly84CXIuIw4MqsHpKOAM4E3ky9c/C1LIl1AkuAk4EjgA9ndYdU+ns4khYDiwGmHHwgh/xDBeb2A9snwC1nfyh1GLl7ObrQ3gmpwyhE78TdTLy+Gj+/vGEXh965KXUU+TuhuZdr5l5qkmYApwKXA5+SJOA9wFlZlWuBzwNXA/Oz1wA3UX/umbLyFRGxG/iFpM3AW7N6myPisey7VmR1HxoqptInnIhYCiwFmH7Um+Mr3bMTR1SMr79yFwtOvjB1GLm7a+NqljxwTOowCnHJhCfYe1o11qYceucmzjn+8NRh5O7cJl9vBFvbTJO0rt/50uzfyv7+nvrjYyZm5wcAL0dET3beDUzPXk8HngSIiB5J27L604G7+12z/2ee3Kf8bcMFXfqEY2ZWBX1Dag16ISLmDvZm9uiY5yLip5KO7yse5GuHem+w8oFux8QAZb/FCcfMrAxCzRxSewdwmqRTgLHAJOo9nimSurJezgzgqax+N3AI0C2pC5hM/f56X3mf/p8ZrHxQqadF/zPwE+BNkrolnTfcZ8zM2lG9h9PYMey1Ij4bETMiYhb1m/53RMQfA3cCZ2TVFgG3Zq9XZudk798REZGVn5nNYpsNzAH+E7gXmJPNehudfcfK4eJK2sOJiA+n/H4zszIpYI3NRcAKSV8E7geWZeXLgG9nkwK2Uk8gRMSDkm6kPhmgBzg/InoBJF0ArAY6geUR8eBwX+4hNTOzEhjhPZzGrxuxFlibvX6M/z/LrH+dXcAHB/n85dRnuu1bvgpYNZJYnHDMzErCe6mVSUeN8/5g4/D12sCu58Zyy21fSB1G7jrHwrlHPZo6jGK8uKcaz4gBdkyppQ6h5USInlrqzV/y1VIJZ0xH8DvTulOHUYgp28SCH3wvdRi52zjpXTx10bGpwyjEzIs2VOYZMQdWYdFnDmo193DMzCxnQXN3GigjJxwzszIIP2LazMwK0sqPHmiEE46ZWQkEI94JuuU44ZiZlUQjuwi0MiccM7MSiIBez1IzM7Mi+B5OidRqNbRrVOowCrF18gRu+Pg5qcPI3Z49NQ67bEPqMIoxuSIPJQMOmNneCxjz4ns4JTJmby/nf+5vU4dRiCWXX8jz00anDiN3o388nlfmV2Mx5KHfr8ZDyezVCTykZmZmBYgQvb1OOGZmVgAPqZmZWf7Ce6mZmVkBvJeamZkVJtzDMTOzvNWf+Jk6iny1VMLpIbjlj89IHUYhXoouOnrGpw4jdz0TdzPx+mo8lGznJD+UzIYQ8rToMunaU2PBcRemDqMQ/7FxNV/b0P4PJrt44uPsPa0a63D2q8iiT3t1Ak8aMDOzgnhrGzMzy19Arc1HXZ1wzMxKwENqZmZWiAixt6e9Nz11wjEzK4nwkJqZmeUuvNOAmZkVwPdwSqZnlLjlti+kDqMQHWPFoqMfTR1G/rbuZszN1Vj4uWNKm4+X2GvmWWol0qUOFvzrqtRhFGLjpHfxzIXHpA4jd6Mu2cDu06ux8PNAL/y0oYT3UjMzswJ4SM3MzIoR0OshNTMzy1ugtu/hJF1lJGmepIclbZZ0ccpYzMySCoheNXS0qmQ9HEmdwBLgfUA3cK+klRHxUKqYzMxSCTyklqe3Apsj4jEASSuA+YATjplVUrsPqaVMONOBJ/uddwNv27eSpMXAYoBpM17HDX96UTHRJbZnT403Xr4hdRj5m7KLQysyXfiAme29T5a9RgHR44STl4F+Z//LA1YjYimwFOCNc38nnj1s/7zjKoXRPxrPKwvaf33Kod/fxDnHH546DLP0AjobvD/Tk3MoeUmZcLqBQ/qdzwCeShSLmVlSAjp8Dyc39wJzJM0GfgmcCZyVMB4zs3RCdPgeTj4iokfSBcBqoBNYHhEPporHzCw19aaOIF9JF35GxCqgGpujmZkNQQGdbd7D8bQZM7OS6Kg1dgxH0iGS7pS0UdKDkj6Zle8vaY2kR7Jfp2blknRVtgh/g6Rj+l1rUVb/EUmL+pUfK+ln2WeukjRstnTCMTMrAQV09KqhowE9wKcj4neB44DzJR0BXAzcHhFzgNuzc4CTgTnZsRi4GuoJCriU+pKVtwKX9iWprM7ifp+bN1xQLbWX2t6A7XsnpQ6jEL0TdzPx+vZ/TszOSW0+LcdsBNSkIbWIeBp4Onu9XdJG6msf5wPHZ9WuBdYCF2Xl10VEAHdLmiLpoKzumojYCiBpDTBP0lpgUkT8JCu/DlgA/GCouFoq4dR6O/nibb+XOoxCXDzxcfae1v7rcParyKJPs+EooLPxSQPTJK3rd740W7P4X68rzQKOBu4B3pAlIyLiaUmvz6oNtBB/+jDl3QOUD6mlEo6ZWfsa0bToFyJi7rBXlPYD/gX4s4h4ZYjbLIMtxB9p+ZB8D8fMrAQU0LlXDR0NXU8aRT3ZXB8R382Kn82Gysh+fS4rH2wh/lDlMwYoH5ITjplZGWRDao0cw8lmjC0DNkbE3/V7ayXQN9NsEXBrv/KF2Wy144Bt2dDbauAkSVOzyQInAauz97ZLOi77roX9rjUoD6mZmZVAk7e2eQdwNvAzSX2zjy4BrgBulHQesAX4YPbeKuAUYDOwEzgXICK2SrqM+s4wAF/om0AAfAy4BhhHfbLAkBMGwAnHzKwcsmnRTblUxI8Y+D4LwIkD1A/g/EGutRxYPkD5OuDIkcTlhGNmVhJq81UCTjhmZiWgETyeoFW1VsKJGouOfjR1FMXYupsxN7f/ws8dU9r8v3RmI9DhzTvLo2tvJ9sXHjN8xTYw86IN7D69/Rd+HuiFn2ZAtrVNm2/e2VIJx8ysnfnxBGZmlr+Q7+GYmVn+6rtFp44iX044ZmYl0cSFn6XkhGNmVgIK6NrjITUzM8ubh9TKpdZR47DLNqQOoxC943d6HY5ZhYjmbW1TVi2VcDp7Otgzv/3XpgCMu2691+GYVUl4axszMyuAGNETP1uSE46ZWRk0cbfosnLCMTMrA08aMDOzItQnDaSOIl9OOGZmZeAhNTMzK4J7OGZmVgzfwymXWmeNide3/2JIAN6wi0MrsEblgJkdqUMwKwWFvLVNmXT2dLD3tPZfDAlw6J2bOOf4w1OHYWZFcQ/HzMyK4McTmJlZYZxwzMwsd6rAtOgkd2wlfVDSg5JqkuamiMHMrGw6ehs7WlWqHs4DwB8C30j0/WZm5eJ7OPmIiI0AUnt3H83MGuVJAyUgaTGwGGDqQQdW4qFk4AeTmVWRE86rJOnfgAMHeOtzEXFro9eJiKXAUoDZbzoyqvBQMvCDycyqpgqTBnJLOBHx3ryubWbWjtzDMTOz/FXgHk6qadGnS+oG3g58X9LqFHGYmZWFArr2NHa0qlSz1G4Gbk7x3WZmZeRZamZmVoyAjp7UQeTLCcfMrCQ8S61EejtrXodjZm3JQ2ol09nbgdfhmFm7csIxM7PcuYdjZmbFcMIxM7OiOOGYmVnuqjCklmSnATMz20e2DqeRoxGS5kl6WNJmSRfnG3xj3MMxMyuBvq1tmnItqRNYArwP6AbulbQyIh5qzje8Ok44ZmYl0cQhtbcCmyPiMQBJK4D5QNKEo4hI+f0jImk78HDqOAoyDXghdRAFqEo7wW1tR2+KiInNuJCkH1L/fWvEWGBXv/Ol2bPD+q51BjAvIj6anZ8NvC0iLmhGrK9Wq/VwHo6IuamDKIKkdVVoa1XaCW5rO5K0rlnXioh5zboWMNAeOcl7F540YGbWfrqBQ/qdzwCeShTLbzjhmJm1n3uBOZJmSxoNnAmsTBxTyw2pLR2+StuoSlur0k5wW9tRKdsZET2SLgBWA53A8oh4MHFYrTVpwMzMWpeH1MzMrBBOOGZmVohSJpzhtmSQNEbSDdn790iaVXyUr10D7fyUpIckbZB0u6RDU8TZDI1usyHpDEkhqWWn1DbSVkkfyv5sH5T0T0XH2AwN/PzOlHSnpPuzn+FTUsTZDJKWS3pO0gODvC9JV2W/FxskHVN0jC0hIkp1UL/B9SjwRmA08H+BI/ap86fA17PXZwI3pI47p3aeAIzPXn+sFdvZaFuzehOBu4C7gbmp487xz3UOcD8wNTt/feq4c2rnUuBj2esjgMdTx/0a2vsu4BjggUHePwX4AfX1L8cB96SOuYxHGXs4v9mSISL2AH1bMvQ3H7g2e30TcKKkVnsY+LDtjIg7I2Jndno39bn0raiRP1OAy4C/4bdXULeaRtr634AlEfESQEQ8V3CMzdBIOwOYlL2eTAnWgbxaEXEXsHWIKvOB66LubmCKpIOKia51lDHhTAee7HfenZUNWCcieoBtwAGFRNc8jbSzv/Oo/w+qFQ3bVklHA4dExPeKDCwHjfy5Hg4cLunHku6W1MwV5kVppJ2fBz4iqRtYBXy8mNCSGOnf50oq4zqcRrZkKOW2DSPUcBskfQSYC7w714jyM2RbJXUAVwLnFBVQjhr5c+2iPqx2PPVe639IOjIiXs45tmZqpJ0fBq6JiC9Lejvw7aydtfzDK1w7/JuUuzL2cBrZkuE3dSR1Ue+uD9XdLaOGtp6Q9F7gc8BpEbG7oNiabbi2TgSOBNZKepz6GPjKFp040OjP760RsTcifkF9Q9o5BcXXLI208zzgRoCI+An1DScb3Zyy1ZRyK5myKWPCaWRLhpXAouz1GcAdkd25ayHDtjMbZvoG9WTTiuP8fYZsa0Rsi4hpETErImZRv191WkQ0bWPEAjXy83sL9QkhSJpGfYjtsUKjfO0aaecW4EQASb9LPeE8X2iUxVkJLMxmqx0HbIuIp1MHVTalG1KLQbZkkPQFYF1ErASWUe+eb6beszkzXcSvToPt/F/AfsB3sjkRWyLitGRBv0oNtrUtNNjW1cBJkh4CeoG/iIgX00U9cg2289PANyX9OfXhpXNa8D+GAEj6Z+pDoNOye1KXAqMAIuLr1O9RnQJsBnYC56aJtNy8tY2ZmRWijENqZmbWhpxwzMysEE44ZmZWCCccMzMrhBOOmZkVwgnHzMwK4YRjZmaFcMKxypD0e9mzSsZKmpA9i+bI1HGZVYUXflqlSPoi9S1WxgHdEfFXiUMyqwwnHKuUbN+ve6k/c+f3I6I3cUhmleEhNaua/anvTzeRek/HzAriHo5ViqSV1J9OORs4KCIuSBySWWWUbrdos7xIWgj0RMQ/SeoE/o+k90TEHaljM6sC93DMzKwQvodjZmaFcMIxM7NCOOGYmVkhnHDMzKwQTjhmZlYIJxwzMyuEE46ZmRXi/wHbIXAxVA3QjgAAAABJRU5ErkJggg==\n", "text/plain": [ "
" ] }, "metadata": { "needs_background": "light" }, "output_type": "display_data" } ], "source": [ "# Use chunks to create a 2D histogram\n", "hb = d2(chunked, chunked2, \"fixed_width\", .2, axis_names=[\"x\", \"y\"])\n", "check_hb = h2(million, million2, \"fixed_width\", .2, axis_names=[\"x\", \"y\"])\n", "hb.plot(show_zero=False, cmap=\"rainbow\")\n", "ok = (hb == check_hb)\n", "print(\"Check: \", ok)\n", "hb" ] }, { "cell_type": "code", "execution_count": 6, "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "Check: True\n" ] }, { "data": { "image/png": "iVBORw0KGgoAAAANSUhEUgAAAagAAAEYCAYAAAAJeGK1AAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4xLjMsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+AADFEAAAWFklEQVR4nO3df4xd5Z3f8fdn7ZAfuyJAcFJqOzVtrHQJ6m7IyPE2UhWFBAxEMa2CBGoXN7VkNSXdbLVVYrp/WM2PylGrZZcqQXLBxaQRDmJTYS2krkuIokpAGEKWH3FSTwmFWWjsxMCyjTbUybd/3MfZW3vsgXvN3Gc875d0Ned8z3PO/d4jPB/OmWfOpKqQJKk3vzLpBiRJmosBJUnqkgElSeqSASVJ6pIBJUnq0vJJN3CqnXvuubVmzZpJtyFJeoUefvjhH1fVimPrp11ArVmzhunp6Um3IUl6hZL8r7nq3uKTJHXJgJIkdcmAkiR1yYCSJHXJgJIkdcmAkiR1yYCSJHXJgJIkdcmAkiR1yYCSJHXJgJIkdcmAkiR1yYCSJHXJgJIkdcmAkiR1yYCSJHXJgJIkdWnegEqyM8nBJI/Pse1fJqkk57b1JLkxyUySR5NcNDR2U5ID7bVpqP6eJI+1fW5MklY/J8m+Nn5fkrNPzUeWJC0Gr+QK6lZgw7HFJKuBDwFPD5UvA9a21xbgpjb2HGAb8F5gHbBtKHBuamOP7nf0vbYC91bVWuDeti5JWiLmDaiq+hZweI5NNwCfAmqothG4rQYeAM5Kch5wKbCvqg5X1fPAPmBD23ZmVd1fVQXcBlw5dKxdbXnXUF2StAQsH2WnJB8B/qyq/rTdkTtqJfDM0Ppsq52sPjtHHeBtVfUcQFU9l+Sto/QqnS7WbL17Qd7nqe1XLMj7SPN51QGV5E3A7wOXzLV5jlqNUH+1PW1hcJuQt7/97a92d2ksCxUcPRvlHBiEms8oV1B/CzgfOHr1tAr4TpJ1DK6AVg+NXQU82+rvP6b+zVZfNcd4gB8lOa9dPZ0HHDxRQ1W1A9gBMDU19aoDTqcng2M0njf14lUHVFU9BvzydluSp4Cpqvpxkj3AJ5LsZjAh4sUWMHuBfzM0MeIS4PqqOpzkpSTrgQeBa4F/38bsATYB29vXu0b6hJK65FWX5vNKppnfDtwPvDPJbJLNJxl+D/AkMAP8B+CfAVTVYeCzwEPt9ZlWA/g4cHPb538CX2/17cCHkhxgMFtw+6v7aJKkxWzeK6iqumae7WuGlgu47gTjdgI756hPAxfOUf8JcPF8/UmSTk8+SUKS1CUDSpLUJQNKktQlA0qS1CUDSpLUpZEedSQtJH9xVFqaDChJi4a/3Lu0eItPktQlA0qS1CUDSpLUJQNKktQlA0qS1CUDSpLUJQNKktQlA0qS1CUDSpLUJQNKktQlA0qS1CUDSpLUJQNKktQlA0qS1CUDSpLUpXkDKsnOJAeTPD5U+7dJvp/k0ST/OclZQ9uuTzKT5AdJLh2qb2i1mSRbh+rnJ3kwyYEkX01yRqu/vq3PtO1rTtWHliT175VcQd0KbDimtg+4sKr+DvA/gOsBklwAXA28q+3zpSTLkiwDvghcBlwAXNPGAnwBuKGq1gLPA5tbfTPwfFW9A7ihjZMkLRHzBlRVfQs4fEztv1bVkbb6ALCqLW8EdlfVz6rqh8AMsK69Zqrqyap6GdgNbEwS4APAnW3/XcCVQ8fa1ZbvBC5u4yVJS8Cp+BnUPwG+3pZXAs8MbZtttRPV3wK8MBR2R+v/37Ha9hfb+OMk2ZJkOsn0oUOHxv5AkqTJGyugkvw+cAT4ytHSHMNqhPrJjnV8sWpHVU1V1dSKFStO3rQkaVFYPuqOSTYBHwYurqqjwTELrB4atgp4ti3PVf8xcFaS5e0qaXj80WPNJlkOvJljbjVKkk5fI11BJdkAfBr4SFX9dGjTHuDqNgPvfGAt8G3gIWBtm7F3BoOJFHtasN0HfLTtvwm4a+hYm9ryR4FvDAWhJOk0N+8VVJLbgfcD5yaZBbYxmLX3emBfm7fwQFX906p6IskdwPcY3Pq7rqp+3o7zCWAvsAzYWVVPtLf4NLA7yeeAR4BbWv0W4MtJZhhcOV19Cj6vJGmRyOl2UTI1NVXT09OTbkMnsGbr3ZNuQUvMU9uvmHQLmkeSh6tq6tj6yD+DkqTFYJT/KTLU+uCjjiRJXTKgJEldMqAkSV0yoCRJXTKgJEldMqAkSV0yoCRJXTKgJEldMqAkSV0yoCRJXTKgJEldMqAkSV0yoCRJXTKgJEldMqAkSV0yoCRJXTKgJEldMqAkSV0yoCRJXTKgJEldmjegkuxMcjDJ40O1c5LsS3KgfT271ZPkxiQzSR5NctHQPpva+ANJNg3V35PksbbPjUlysveQJC0Nr+QK6lZgwzG1rcC9VbUWuLetA1wGrG2vLcBNMAgbYBvwXmAdsG0ocG5qY4/ut2Ge95AkLQHzBlRVfQs4fEx5I7CrLe8Crhyq31YDDwBnJTkPuBTYV1WHq+p5YB+woW07s6rur6oCbjvmWHO9hyRpCRj1Z1Bvq6rnANrXt7b6SuCZoXGzrXay+uwc9ZO9hyRpCTjVkyQyR61GqL+6N022JJlOMn3o0KFXu7skqUOjBtSP2u052teDrT4LrB4atwp4dp76qjnqJ3uP41TVjqqaqqqpFStWjPiRJEk9GTWg9gBHZ+JtAu4aql/bZvOtB15st+f2ApckObtNjrgE2Nu2vZRkfZu9d+0xx5rrPSRJS8Dy+QYkuR14P3BuklkGs/G2A3ck2Qw8DVzVht8DXA7MAD8FPgZQVYeTfBZ4qI37TFUdnXjxcQYzBd8IfL29OMl7SJKWgHkDqqquOcGmi+cYW8B1JzjOTmDnHPVp4MI56j+Z6z0kSUuDT5KQJHXJgJIkdcmAkiR1yYCSJHXJgJIkdcmAkiR1yYCSJHXJgJIkdcmAkiR1yYCSJHXJgJIkdcmAkiR1yYCSJHVp3qeZSyeyZuvdk25B0mnMgJKkY4zyP19Pbb/iNehkafMWnySpSwaUJKlLBpQkqUsGlCSpSwaUJKlLBpQkqUsGlCSpS2MFVJJ/keSJJI8nuT3JG5Kcn+TBJAeSfDXJGW3s69v6TNu+Zug417f6D5JcOlTf0GozSbaO06skaXEZOaCSrAR+B5iqqguBZcDVwBeAG6pqLfA8sLntshl4vqreAdzQxpHkgrbfu4ANwJeSLEuyDPgicBlwAXBNGytJWgLGvcW3HHhjkuXAm4DngA8Ad7btu4Ar2/LGtk7bfnGStPruqvpZVf0QmAHWtddMVT1ZVS8Du9tYSdISMHJAVdWfAf8OeJpBML0IPAy8UFVH2rBZYGVbXgk80/Y90sa/Zbh+zD4nqh8nyZYk00mmDx06NOpHkiR1ZJxbfGczuKI5H/jrwK8yuB13rDq6ywm2vdr68cWqHVU1VVVTK1asmK91SdIiMM4tvg8CP6yqQ1X1f4GvAX8XOKvd8gNYBTzblmeB1QBt+5uBw8P1Y/Y5UV2StASME1BPA+uTvKn9LOli4HvAfcBH25hNwF1teU9bp23/RlVVq1/dZvmdD6wFvg08BKxtswLPYDCRYs8Y/UqSFpGR/9xGVT2Y5E7gO8AR4BFgB3A3sDvJ51rtlrbLLcCXk8wwuHK6uh3niSR3MAi3I8B1VfVzgCSfAPYymCG4s6qeGLVfSdLiMtbfg6qqbcC2Y8pPMpiBd+zYvwSuOsFxPg98fo76PcA94/QoSVqcfJKEJKlLBpQkqUsGlCSpSwaUJKlLBpQkqUsGlCSpSwaUJKlLBpQkqUsGlCSpSwaUJKlLBpQkqUsGlCSpSwaUJKlLBpQkqUsGlCSpSwaUJKlLBpQkqUsGlCSpSwaUJKlLBpQkqUsGlCSpS2MFVJKzktyZ5PtJ9if5rSTnJNmX5ED7enYbmyQ3JplJ8miSi4aOs6mNP5Bk01D9PUkea/vcmCTj9CtJWjzGvYL6I+C/VNXfBn4D2A9sBe6tqrXAvW0d4DJgbXttAW4CSHIOsA14L7AO2HY01NqYLUP7bRizX0nSIjFyQCU5E/h7wC0AVfVyVb0AbAR2tWG7gCvb8kbgthp4ADgryXnApcC+qjpcVc8D+4ANbduZVXV/VRVw29CxJEmnuXGuoP4mcAj4j0keSXJzkl8F3lZVzwG0r29t41cCzwztP9tqJ6vPzlE/TpItSaaTTB86dGiMjyRJ6sU4AbUcuAi4qareDfwf/up23lzm+vlRjVA/vli1o6qmqmpqxYoVJ+9akrQojBNQs8BsVT3Y1u9kEFg/arfnaF8PDo1fPbT/KuDZeeqr5qhLkpaAkQOqqv438EySd7bSxcD3gD3A0Zl4m4C72vIe4No2m2898GK7BbgXuCTJ2W1yxCXA3rbtpSTr2+y9a4eOJUk6zS0fc/9/DnwlyRnAk8DHGITeHUk2A08DV7Wx9wCXAzPAT9tYqupwks8CD7Vxn6mqw23548CtwBuBr7eXJGkJGCugquq7wNQcmy6eY2wB153gODuBnXPUp4ELx+lRkrQ4+SQJSVKXDChJUpcMKElSlwwoSVKXDChJUpcMKElSlwwoSVKXxv1FXZ0m1my9e9ItSIvaKP+Gntp+xWvQyenDKyhJUpcMKElSlwwoSVKXDChJUpcMKElSlwwoSVKXDChJUpcMKElSlwwoSVKXDChJUpcMKElSlwwoSVKXDChJUpcMKElSl8YOqCTLkjyS5E/a+vlJHkxyIMlXk5zR6q9v6zNt+5qhY1zf6j9IculQfUOrzSTZOm6vkqTF41RcQX0S2D+0/gXghqpaCzwPbG71zcDzVfUO4IY2jiQXAFcD7wI2AF9qobcM+CJwGXABcE0bK0laAsYKqCSrgCuAm9t6gA8Ad7Yhu4Ar2/LGtk7bfnEbvxHYXVU/q6ofAjPAuvaaqaonq+plYHcbK0laAsa9gvpD4FPAL9r6W4AXqupIW58FVrbllcAzAG37i238L+vH7HOi+nGSbEkynWT60KFDY34kSVIPRg6oJB8GDlbVw8PlOYbWPNtebf34YtWOqpqqqqkVK1acpGtJ0mKxfIx93wd8JMnlwBuAMxlcUZ2VZHm7SloFPNvGzwKrgdkky4E3A4eH6kcN73OiuiTpNDfyFVRVXV9Vq6pqDYNJDt+oqn8I3Ad8tA3bBNzVlve0ddr2b1RVtfrVbZbf+cBa4NvAQ8DaNivwjPYee0btV5K0uIxzBXUinwZ2J/kc8AhwS6vfAnw5yQyDK6erAarqiSR3AN8DjgDXVdXPAZJ8AtgLLAN2VtUTr0G/kqQOnZKAqqpvAt9sy08ymIF37Ji/BK46wf6fBz4/R/0e4J5T0aMkaXHxSRKSpC4ZUJKkLhlQkqQuGVCSpC4ZUJKkLhlQkqQuGVCSpC4ZUJKkLhlQkqQuGVCSpC4ZUJKkLhlQkqQuGVCSpC4ZUJKkLhlQkqQuGVCSpC4ZUJKkLhlQkqQuGVCSpC4ZUJKkLi2fdAM69dZsvXvSLUh6BUb5t/rU9iteg076NPIVVJLVSe5Lsj/JE0k+2ernJNmX5ED7enarJ8mNSWaSPJrkoqFjbWrjDyTZNFR/T5LH2j43Jsk4H1aStHiMc4vvCPB7VfXrwHrguiQXAFuBe6tqLXBvWwe4DFjbXluAm2AQaMA24L3AOmDb0VBrY7YM7bdhjH4lSYvIyAFVVc9V1Xfa8kvAfmAlsBHY1YbtAq5syxuB22rgAeCsJOcBlwL7qupwVT0P7AM2tG1nVtX9VVXAbUPHkiSd5k7JJIkka4B3Aw8Cb6uq52AQYsBb27CVwDNDu8222snqs3PU53r/LUmmk0wfOnRo3I8jSerA2AGV5NeAPwZ+t6r+/GRD56jVCPXji1U7qmqqqqZWrFgxX8uSpEVgrIBK8joG4fSVqvpaK/+o3Z6jfT3Y6rPA6qHdVwHPzlNfNUddkrQEjDOLL8AtwP6q+oOhTXuAozPxNgF3DdWvbbP51gMvtluAe4FLkpzdJkdcAuxt215Ksr6917VDx5IknebG+T2o9wG/DTyW5Lut9q+A7cAdSTYDTwNXtW33AJcDM8BPgY8BVNXhJJ8FHmrjPlNVh9vyx4FbgTcCX28vSdISMHJAVdV/Z+6fEwFcPMf4Aq47wbF2AjvnqE8DF47aoyRp8fJRR5KkLhlQkqQuGVCSpC4ZUJKkLhlQkqQuGVCSpC4ZUJKkLhlQkqQuGVCSpC4ZUJKkLhlQkqQuGVCSpC4ZUJKkLhlQkqQujfP3oLQA1my9e9ItSOrIKN8Tntp+xWvQyWvPKyhJUpcMKElSlwwoSVKXDChJUpcMKElSlwwoSVKXDChJUpe6/z2oJBuAPwKWATdX1fYJtyRJi8pi/d2prgMqyTLgi8CHgFngoSR7qup7k+1sNP7SrSS9cl0HFLAOmKmqJwGS7AY2AhMPKMNG0uls1O9xp/LKq/eAWgk8M7Q+C7z32EFJtgBb2upfJPnBAvR2LvDjBXifnnkOBjwPngPwHACQL4x0Hv7GXMXeAypz1Oq4QtUOYMdr385fSTJdVVML+Z698RwMeB48B+A5OOpUnofeZ/HNAquH1lcBz06oF0nSAuo9oB4C1iY5P8kZwNXAngn3JElaAF3f4quqI0k+AexlMM18Z1U9MeG2jlrQW4qd8hwMeB48B+A5OOqUnYdUHfcjHUmSJq73W3ySpCXKgJIkdcmAGkOSq5I8keQXSZbU9NIkG5L8IMlMkq2T7mcSkuxMcjDJ45PuZVKSrE5yX5L97d/CJyfd00JL8oYk307yp+0c/OtJ9zQpSZYleSTJn5yK4xlQ43kc+AfAtybdyEIaegTVZcAFwDVJLphsVxNxK7Bh0k1M2BHg96rq14H1wHVL8L+FnwEfqKrfAH4T2JBk/YR7mpRPAvtP1cEMqDFU1f6qWoinVvTml4+gqqqXgaOPoFpSqupbwOFJ9zFJVfVcVX2nLb/E4JvTysl2tbBq4C/a6uvaa8nNPkuyCrgCuPlUHdOA0ijmegTVkvqmpOMlWQO8G3hwsp0svHZr67vAQWBfVS25cwD8IfAp4Ben6oAG1DyS/Lckj8/xWnJXDENe0SOotHQk+TXgj4Hfrao/n3Q/C62qfl5Vv8ngaTfrklw46Z4WUpIPAwer6uFTedyuf1G3B1X1wUn30CEfQaVfSvI6BuH0lar62qT7maSqeiHJNxn8bHIpTZ55H/CRJJcDbwDOTPKfquofjXNQr6A0Ch9BJQCSBLgF2F9VfzDpfiYhyYokZ7XlNwIfBL4/2a4WVlVdX1WrqmoNg+8H3xg3nMCAGkuSv59kFvgt4O4keyfd00KoqiPA0UdQ7Qfu6OgRVAsmye3A/cA7k8wm2TzpnibgfcBvAx9I8t32unzSTS2w84D7kjzK4H/e9lXVKZlmvdT5qCNJUpe8gpIkdcmAkiR1yYCSJHXJgJIkdcmAkiR1yYCSJHXJgJIkdcmAkjqW5LPDf2MpyeeT/M4ke5IWir+oK3WsPSH8a1V1UZJfAQ4A66rqJxNtTFoAPixW6lhVPZXkJ0neDbwNeMRw0lJhQEn9uxn4x8BfA3ZOthVp4XiLT+pce2L8Ywz+Uuvaqvr5hFuSFoRXUFLnqurlJPcBLxhOWkoMKKlzbXLEeuCqSfciLSSnmUsdS3IBMAPcW1UHJt2PtJD8GZQkqUteQUmSumRASZK6ZEBJkrpkQEmSumRASZK69P8A7710FgSg1ooAAAAASUVORK5CYII=\n", "text/plain": [ "
" ] }, "metadata": { "needs_background": "light" }, "output_type": "display_data" } ], "source": [ "# And another cross-check\n", "hh = hb.projection(\"y\")\n", "hh.plot()\n", "print(\"Check: \", np.array_equal(hh.frequencies, ha.frequencies)) # Just frequencies" ] }, { "cell_type": "code", "execution_count": 7, "metadata": {}, "outputs": [ { "data": { "text/plain": [ "True" ] }, "execution_count": 7, "metadata": {}, "output_type": "execute_result" } ], "source": [ "# Use dask for normal arrays (will automatically split array to chunks)\n", "d1(million2, \"fixed_width\", 0.2) == ha" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "## Some timings\n", "\n", "Your results may vary substantially. These numbers are just for illustration, on 4-core (8-thread) machine. The real gain comes when we have data that don't fit into memory." ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "### Efficiency" ] }, { "cell_type": "code", "execution_count": 8, "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "CPU times: user 485 ms, sys: 391 ms, total: 876 ms\n", "Wall time: 268 ms\n" ] }, { "data": { "text/plain": [ "Histogram1D(bins=(28,), total=2097152, dtype=int64)" ] }, "execution_count": 8, "metadata": {}, "output_type": "execute_result" } ], "source": [ "# Standard\n", "%time h1(million2, \"fixed_width\", 0.2)" ] }, { "cell_type": "code", "execution_count": 9, "metadata": { "scrolled": true }, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "CPU times: user 389 ms, sys: 10 ms, total: 399 ms\n", "Wall time: 95.4 ms\n" ] }, { "data": { "text/plain": [ "Histogram1D(bins=(28,), total=2097152, dtype=int64)" ] }, "execution_count": 9, "metadata": {}, "output_type": "execute_result" } ], "source": [ "# Same array, but using dask\n", "%time d1(million2, \"fixed_width\", 0.2)" ] }, { "cell_type": "code", "execution_count": 10, "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "CPU times: user 412 ms, sys: 9.86 ms, total: 422 ms\n", "Wall time: 73.1 ms\n" ] }, { "data": { "text/plain": [ "Histogram1D(bins=(28,), total=2097152, dtype=int64)" ] }, "execution_count": 10, "metadata": {}, "output_type": "execute_result" } ], "source": [ "# Most efficient: dask with already chunked data\n", "%time d1(chunked2, \"fixed_width\", 0.2)" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "### Different scheduling" ] }, { "cell_type": "code", "execution_count": 11, "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "CPU times: user 383 ms, sys: 7.54 ms, total: 391 ms\n", "Wall time: 74.3 ms\n" ] }, { "data": { "text/plain": [ "Histogram1D(bins=(28,), total=2097152, dtype=int64)" ] }, "execution_count": 11, "metadata": {}, "output_type": "execute_result" } ], "source": [ "%time d1(chunked2, \"fixed_width\", 0.2)" ] }, { "cell_type": "code", "execution_count": 12, "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "CPU times: user 381 ms, sys: 13.4 ms, total: 394 ms\n", "Wall time: 128 ms\n" ] }, { "data": { "text/plain": [ "Histogram1D(bins=(28,), total=2097152, dtype=int64)" ] }, "execution_count": 12, "metadata": {}, "output_type": "execute_result" } ], "source": [ "%%time\n", "# Hyper-threading or not?\n", "graph, name = d1(chunked2, \"fixed_width\", 0.2, compute=False)\n", "dask.threaded.get(graph, name, num_workers=4)" ] }, { "cell_type": "code", "execution_count": 13, "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "CPU times: user 262 ms, sys: 360 ms, total: 622 ms\n", "Wall time: 714 ms\n" ] }, { "data": { "text/plain": [ "Histogram1D(bins=(28,), total=2097152, dtype=int64)" ] }, "execution_count": 13, "metadata": {}, "output_type": "execute_result" } ], "source": [ "# Multiprocessing not so efficient for small arrays?\n", "%time d1(chunked2, \"fixed_width\", 0.2, dask_method=dask.multiprocessing.get)" ] } ], "metadata": { "anaconda-cloud": {}, "kernelspec": { "display_name": "Python 3", "language": "python", "name": "python3" }, "language_info": { "codemirror_mode": { "name": "ipython", "version": 3 }, "file_extension": ".py", "mimetype": "text/x-python", "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", "version": "3.7.3" } }, "nbformat": 4, "nbformat_minor": 1 }