cluster:(root)

{ "cells": [ { "cell_type": "markdown", "metadata": {}, "source": [ "## This notebook tests experimental features that will likely change." ] }, { "cell_type": "code", "execution_count": 1, "metadata": {}, "outputs": [ { "data": { "image/svg+xml": [ "\n", "\n", "\n", "\n", "\n" ], "text/plain": [ "" ] }, "metadata": {}, "output_type": "display_data" } ], "source": [ "from lale.lib.rasl import SimpleImputer as SIM\n", "from lale.lib.rasl import MinMaxScaler as MMS\n", "from lale.lib.rasl import BatchedBaggingClassifier as BBC\n", "from lale.lib.sklearn import SGDClassifier as SGD\n", "from lale.lib.sklearn import RandomForestClassifier as RFC\n", "\n", "def fresh_pipeline(final_est=\"sgd\"): #create new to guarantee fresh state after earlier fit/partial_fit, if any\n", " if final_est == \"sgd\":\n", " est = SGD(random_state=42)\n", " elif final_est == \"rfc\":\n", " est = RFC(random_state=42)\n", " elif final_est == \"bbc\":\n", " est = BBC()\n", " return SIM(strategy=\"mean\") >> MMS() >> est\n", "\n", "fresh_pipeline().visualize()" ] }, { "cell_type": "code", "execution_count": 2, "metadata": {}, "outputs": [], "source": [ "from lale.lib.rasl import is_associative, is_incremental, PrioBatch, PrioStep, PrioResourceAware\n", "from lale.lib.rasl import mockup_data_loader, fit_with_batches, cross_val_score\n", "from sklearn.model_selection import KFold\n", "\n", "assert is_associative(SIM)\n", "assert is_associative(MMS)\n", "assert is_incremental(SGD)\n", "assert not is_incremental(RFC)" ] }, { "cell_type": "code", "execution_count": 3, "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "unique_class_labels ['good', 'bad']\n", "train_data_space 53600\n" ] }, { "data": { "text/html": [ "

\n", "\n", "\n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", "

	class	duration	credit_amount	installment_commitment	residence_since	age	existing_credits	num_dependents
993	good	36.0	3959.0	4.0	3.0	30.0	1.0	1.0
859	good	9.0	3577.0	1.0	2.0	26.0	1.0	2.0
298	good	18.0	2515.0	3.0	4.0	43.0	1.0	1.0
...	...	...	...	...	...	...	...	...
503	bad	24.0	1216.0	4.0	4.0	38.0	2.0	2.0
263	good	12.0	2748.0	2.0	4.0	57.0	3.0	1.0
808	good	42.0	9283.0	1.0	2.0	55.0	1.0	1.0

\n", "

330 rows × 8 columns

\n", "

" ], "text/plain": [ " class duration credit_amount installment_commitment residence_since \\\n", "993 good 36.0 3959.0 4.0 3.0 \n", "859 good 9.0 3577.0 1.0 2.0 \n", "298 good 18.0 2515.0 3.0 4.0 \n", ".. ... ... ... ... ... \n", "503 bad 24.0 1216.0 4.0 4.0 \n", "263 good 12.0 2748.0 2.0 4.0 \n", "808 good 42.0 9283.0 1.0 2.0 \n", "\n", " age existing_credits num_dependents \n", "993 30.0 1.0 1.0 \n", "859 26.0 1.0 2.0 \n", "298 43.0 1.0 1.0 \n", ".. ... ... ... \n", "503 38.0 2.0 2.0 \n", "263 57.0 3.0 1.0 \n", "808 55.0 1.0 1.0 \n", "\n", "[330 rows x 8 columns]" ] }, "execution_count": 3, "metadata": {}, "output_type": "execute_result" } ], "source": [ "import lale.datasets.openml\n", "import math\n", "import pandas as pd\n", "from lale.lib.lale import Project\n", "\n", "(raw_train_X, train_y), (raw_test_X, test_y) = lale.datasets.openml.fetch(\"credit-g\", \"classification\", preprocess=False)\n", "project = Project(columns={\"type\": \"number\"}).fit(raw_train_X)\n", "train_X, test_X = project.transform(raw_train_X), project.transform(raw_test_X)\n", "unique_class_labels = list(train_y.unique())\n", "print(f\"unique_class_labels {unique_class_labels}\")\n", "train_data_space = train_X.memory_usage().sum() + train_y.memory_usage()\n", "print(f\"train_data_space {train_data_space}\")\n", "pd.set_option('display.max_rows', 6)\n", "pd.concat([test_y, test_X], axis=1)" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "### Example: no RASL" ] }, { "cell_type": "code", "execution_count": 4, "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "majority class 70.91%\n", "accuracy 70.91%\n" ] } ], "source": [ "import lale.lib.rasl\n", "\n", "accuracy_scorer = lale.lib.rasl.get_scorer(\"accuracy\")\n", "\n", "print(f\"majority class {len(test_y[test_y == 'good']) / len(test_y):.2%}\")\n", "trained = fresh_pipeline().fit(train_X, train_y)\n", "print(f\"accuracy {accuracy_scorer(trained, test_X, test_y):.2%}\")" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "### Example: no batching, no cross-validation\n", "- starting point\n", "- behavior equivalent to sklearn\n", "- purposely setting max_resident too low to demonstrate cache warnings" ] }, { "cell_type": "code", "execution_count": 5, "metadata": {}, "outputs": [ { "data": { "image/svg+xml": [ "\n", "\n", "\n", "\n", "\n" ], "text/plain": [ "" ] }, "metadata": {}, "output_type": "display_data" }, { "name": "stdout", "output_type": "stream", "text": [ "scan INP(d0)\n" ] }, { "data": { "image/svg+xml": [ "\n", "\n", "\n", "\n", "\n" ], "text/plain": [ "" ] }, "metadata": {}, "output_type": "display_data" }, { "name": "stdout", "output_type": "stream", "text": [ "scan INP(d1)\n", "spill /tmp/tmp5r63ky4x/X_-1_d0_None.pkl /tmp/tmp5r63ky4x/y_-1_d0_None.pkl\n" ] }, { "data": { "image/svg+xml": [ "\n", "\n", "\n", "\n", "\n" ], "text/plain": [ "" ] }, "metadata": {}, "output_type": "display_data" }, { "name": "stdout", "output_type": "stream", "text": [ "to_monoid SimpleImputer(d0)\n", "load /tmp/tmp5r63ky4x/X_-1_d0_None.pkl /tmp/tmp5r63ky4x/y_-1_d0_None.pkl\n" ] }, { "data": { "image/svg+xml": [ "\n", "\n", "\n", "\n", "\n" ], "text/plain": [ "" ] }, "metadata": {}, "output_type": "display_data" }, { "name": "stdout", "output_type": "stream", "text": [ "combine SimpleImputer(d*)\n" ] }, { "data": { "image/svg+xml": [ "\n", "\n", "\n", "\n", "\n" ], "text/plain": [ "" ] }, "metadata": {}, "output_type": "display_data" }, { "name": "stderr", "output_type": "stream", "text": [ "aborted spill of batch -1_d0_None\n", "ensure_space() failed, amount_needed 53600, no_spill_space 53600, min_resident 107200, max_resident 53600\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ "transform SimpleImputer(d0)\n" ] }, { "data": { "image/svg+xml": [ "\n", "\n", "\n", "\n", "\n" ], "text/plain": [ "" ] }, "metadata": {}, "output_type": "display_data" }, { "name": "stdout", "output_type": "stream", "text": [ "to_monoid MinMaxScaler(d0)\n" ] }, { "data": { "image/svg+xml": [ "\n", "\n", "\n", "\n", "\n" ], "text/plain": [ "" ] }, "metadata": {}, "output_type": "display_data" }, { "name": "stdout", "output_type": "stream", "text": [ "combine MinMaxScaler(d*)\n" ] }, { "data": { "image/svg+xml": [ "\n", "\n", "\n", "\n", "\n" ], "text/plain": [ "" ] }, "metadata": {}, "output_type": "display_data" }, { "name": "stderr", "output_type": "stream", "text": [ "aborted spill of batch 0_d0_None\n", "ensure_space() failed, amount_needed 53600, no_spill_space 53600, min_resident 107200, max_resident 53600\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ "transform MinMaxScaler(d0)\n" ] }, { "data": { "image/svg+xml": [ "\n", "\n", "\n", "\n", "\n" ], "text/plain": [ "" ] }, "metadata": {}, "output_type": "display_data" }, { "name": "stdout", "output_type": "stream", "text": [ "partial_fit SGDClassifier(d0)\n" ] }, { "data": { "image/svg+xml": [ "\n", "\n", "\n", "\n", "\n" ], "text/plain": [ "" ] }, "metadata": {}, "output_type": "display_data" }, { "name": "stdout", "output_type": "stream", "text": [ "partial_fit SGDClassifier(d*)\n" ] }, { "data": { "image/svg+xml": [ "\n", "\n", "\n", "\n", "\n" ], "text/plain": [ "" ] }, "metadata": {}, "output_type": "display_data" }, { "name": "stdout", "output_type": "stream", "text": [ "{\n", " \"spill_count\": 1,\n", " \"load_count\": 1,\n", " \"spill_space\": 53600,\n", " \"load_space\": 53600,\n", " \"min_resident\": 107200,\n", " \"max_resident\": 53600,\n", " \"train_count\": 6,\n", " \"apply_count\": 4,\n", " \"metric_count\": 0,\n", " \"train_time\": 0.04813194274902344,\n", " \"apply_time\": 0.0444943904876709,\n", " \"metric_time\": 0,\n", " \"critical_count\": 1,\n", " \"critical_time\": 0.0268857479095459,\n", "}\n", "accuracy 63.33%\n" ] }, { "data": { "image/svg+xml": [ "\n", "\n", "\n", "\n", "\n" ], "text/plain": [ "" ] }, "metadata": {}, "output_type": "display_data" } ], "source": [ "n_batches = 1\n", "batches = mockup_data_loader(train_X, train_y, n_batches, \"pandas\")\n", "max_resident = 1 * math.ceil(train_data_space / n_batches)\n", "trained = fit_with_batches(fresh_pipeline(), batches, None, None, unique_class_labels,\n", " max_resident, PrioStep(), partial_transform=False, verbose=3, progress_callback=None)\n", "print(f\"accuracy {accuracy_scorer(trained, test_X, test_y):.2%}\")\n", "trained.visualize()" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "### Example: batching using associativity where possible, non-incremental final estimator" ] }, { "cell_type": "code", "execution_count": 6, "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "spill /tmp/tmplzol_tj8/X_-1_d1_None.pkl /tmp/tmplzol_tj8/y_-1_d1_None.pkl\n", "spill /tmp/tmplzol_tj8/X_-1_d2_None.pkl /tmp/tmplzol_tj8/y_-1_d2_None.pkl\n", "load /tmp/tmplzol_tj8/X_-1_d1_None.pkl /tmp/tmplzol_tj8/y_-1_d1_None.pkl\n", "load /tmp/tmplzol_tj8/X_-1_d2_None.pkl /tmp/tmplzol_tj8/y_-1_d2_None.pkl\n", "spill /tmp/tmplzol_tj8/X_-1_d2_None.pkl /tmp/tmplzol_tj8/y_-1_d2_None.pkl\n", "spill /tmp/tmplzol_tj8/X_-1_d1_None.pkl /tmp/tmplzol_tj8/y_-1_d1_None.pkl\n", "load /tmp/tmplzol_tj8/X_-1_d1_None.pkl /tmp/tmplzol_tj8/y_-1_d1_None.pkl\n", "spill /tmp/tmplzol_tj8/X_0_d0_None.pkl /tmp/tmplzol_tj8/y_0_d0_None.pkl\n", "load /tmp/tmplzol_tj8/X_-1_d2_None.pkl /tmp/tmplzol_tj8/y_-1_d2_None.pkl\n", "load /tmp/tmplzol_tj8/X_0_d0_None.pkl /tmp/tmplzol_tj8/y_0_d0_None.pkl\n", "spill /tmp/tmplzol_tj8/X_0_d2_None.pkl /tmp/tmplzol_tj8/y_0_d2_None.pkl\n", "spill /tmp/tmplzol_tj8/X_0_d1_None.pkl /tmp/tmplzol_tj8/y_0_d1_None.pkl\n", "load /tmp/tmplzol_tj8/X_0_d1_None.pkl /tmp/tmplzol_tj8/y_0_d1_None.pkl\n", "spill /tmp/tmplzol_tj8/X_1_d0_None.pkl /tmp/tmplzol_tj8/y_1_d0_None.pkl\n", "load /tmp/tmplzol_tj8/X_0_d2_None.pkl /tmp/tmplzol_tj8/y_0_d2_None.pkl\n", "load /tmp/tmplzol_tj8/X_1_d0_None.pkl /tmp/tmplzol_tj8/y_1_d0_None.pkl\n" ] }, { "data": { "image/svg+xml": [ "\n", "\n", "\n", "\n", "\n" ], "text/plain": [ "" ] }, "metadata": {}, "output_type": "display_data" }, { "name": "stdout", "output_type": "stream", "text": [ "{\n", " \"spill_count\": 8,\n", " \"load_count\": 8,\n", " \"spill_space\": 142880,\n", " \"load_space\": 142880,\n", " \"min_resident\": 71520,\n", " \"max_resident\": 53601,\n", " \"train_count\": 9,\n", " \"apply_count\": 10,\n", " \"metric_count\": 0,\n", " \"train_time\": 0.2980036735534668,\n", " \"apply_time\": 0.08468341827392578,\n", " \"metric_time\": 0,\n", " \"critical_count\": 1,\n", " \"critical_time\": 0.21265220642089844,\n", "}\n", "accuracy 70.30%\n" ] } ], "source": [ "n_batches = 3\n", "max_resident = 3 * math.ceil(train_data_space / n_batches)\n", "batches = mockup_data_loader(train_X, train_y, n_batches, \"pandas\")\n", "trained = fit_with_batches(fresh_pipeline(\"rfc\"), batches, None, None, unique_class_labels,\n", " max_resident, PrioStep(), partial_transform=False, verbose=2, progress_callback=None)\n", "print(f\"accuracy {accuracy_scorer(trained, test_X, test_y):.2%}\")" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "### Example: batching using associativity where possible, partial_fit otherwise\n", "- unlike our batching patent, which always used partial_fit\n", "- behavior still equivalent to sklearn\n", "- potential advantages: less resident data; parallelism\n", "- below are three runs with three different scheduling priorities" ] }, { "cell_type": "code", "execution_count": 7, "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "spill /tmp/tmp2ijsdu2b/X_-1_d1_None.pkl /tmp/tmp2ijsdu2b/y_-1_d1_None.pkl\n", "spill /tmp/tmp2ijsdu2b/X_-1_d2_None.pkl /tmp/tmp2ijsdu2b/y_-1_d2_None.pkl\n", "load /tmp/tmp2ijsdu2b/X_-1_d1_None.pkl /tmp/tmp2ijsdu2b/y_-1_d1_None.pkl\n", "load /tmp/tmp2ijsdu2b/X_-1_d2_None.pkl /tmp/tmp2ijsdu2b/y_-1_d2_None.pkl\n", "spill /tmp/tmp2ijsdu2b/X_-1_d2_None.pkl /tmp/tmp2ijsdu2b/y_-1_d2_None.pkl\n", "spill /tmp/tmp2ijsdu2b/X_-1_d1_None.pkl /tmp/tmp2ijsdu2b/y_-1_d1_None.pkl\n", "load /tmp/tmp2ijsdu2b/X_-1_d1_None.pkl /tmp/tmp2ijsdu2b/y_-1_d1_None.pkl\n", "spill /tmp/tmp2ijsdu2b/X_0_d0_None.pkl /tmp/tmp2ijsdu2b/y_0_d0_None.pkl\n", "load /tmp/tmp2ijsdu2b/X_-1_d2_None.pkl /tmp/tmp2ijsdu2b/y_-1_d2_None.pkl\n", "load /tmp/tmp2ijsdu2b/X_0_d0_None.pkl /tmp/tmp2ijsdu2b/y_0_d0_None.pkl\n", "spill /tmp/tmp2ijsdu2b/X_0_d2_None.pkl /tmp/tmp2ijsdu2b/y_0_d2_None.pkl\n", "spill /tmp/tmp2ijsdu2b/X_0_d1_None.pkl /tmp/tmp2ijsdu2b/y_0_d1_None.pkl\n", "load /tmp/tmp2ijsdu2b/X_0_d1_None.pkl /tmp/tmp2ijsdu2b/y_0_d1_None.pkl\n", "spill /tmp/tmp2ijsdu2b/X_1_d0_None.pkl /tmp/tmp2ijsdu2b/y_1_d0_None.pkl\n", "load /tmp/tmp2ijsdu2b/X_0_d2_None.pkl /tmp/tmp2ijsdu2b/y_0_d2_None.pkl\n", "load /tmp/tmp2ijsdu2b/X_1_d0_None.pkl /tmp/tmp2ijsdu2b/y_1_d0_None.pkl\n" ] }, { "data": { "image/svg+xml": [ "\n", "\n", "\n", "\n", "\n" ], "text/plain": [ "" ] }, "metadata": {}, "output_type": "display_data" }, { "name": "stdout", "output_type": "stream", "text": [ "{\n", " \"spill_count\": 8,\n", " \"load_count\": 8,\n", " \"spill_space\": 142880,\n", " \"load_space\": 142880,\n", " \"min_resident\": 35840,\n", " \"max_resident\": 53601,\n", " \"train_count\": 12,\n", " \"apply_count\": 10,\n", " \"metric_count\": 0,\n", " \"train_time\": 0.10301017761230469,\n", " \"apply_time\": 0.10129833221435547,\n", " \"metric_time\": 0,\n", " \"critical_count\": 1,\n", " \"critical_time\": 0.0216214656829834,\n", "}\n", "accuracy 62.42%\n" ] } ], "source": [ "n_batches = 3\n", "max_resident = 3 * math.ceil(train_data_space / n_batches)\n", "batches = mockup_data_loader(train_X, train_y, n_batches, \"pandas\")\n", "trained = fit_with_batches(fresh_pipeline(), batches, None, None, unique_class_labels, max_resident,\n", " PrioStep(), partial_transform=False, verbose=2, progress_callback=None)\n", "print(f\"accuracy {accuracy_scorer(trained, test_X, test_y):.2%}\")" ] }, { "cell_type": "code", "execution_count": 8, "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "spill /tmp/tmpocs5ad_r/X_-1_d1_None.pkl /tmp/tmpocs5ad_r/y_-1_d1_None.pkl\n", "spill /tmp/tmpocs5ad_r/X_-1_d2_None.pkl /tmp/tmpocs5ad_r/y_-1_d2_None.pkl\n", "load /tmp/tmpocs5ad_r/X_-1_d1_None.pkl /tmp/tmpocs5ad_r/y_-1_d1_None.pkl\n", "spill /tmp/tmpocs5ad_r/X_0_d0_None.pkl /tmp/tmpocs5ad_r/y_0_d0_None.pkl\n", "load /tmp/tmpocs5ad_r/X_-1_d2_None.pkl /tmp/tmpocs5ad_r/y_-1_d2_None.pkl\n", "load /tmp/tmpocs5ad_r/X_0_d0_None.pkl /tmp/tmpocs5ad_r/y_0_d0_None.pkl\n", "spill /tmp/tmpocs5ad_r/X_0_d2_None.pkl /tmp/tmpocs5ad_r/y_0_d2_None.pkl\n", "spill /tmp/tmpocs5ad_r/X_0_d1_None.pkl /tmp/tmpocs5ad_r/y_0_d1_None.pkl\n", "load /tmp/tmpocs5ad_r/X_0_d1_None.pkl /tmp/tmpocs5ad_r/y_0_d1_None.pkl\n", "load /tmp/tmpocs5ad_r/X_0_d2_None.pkl /tmp/tmpocs5ad_r/y_0_d2_None.pkl\n" ] }, { "data": { "image/svg+xml": [ "\n", "\n", "\n", "\n", "\n" ], "text/plain": [ "" ] }, "metadata": {}, "output_type": "display_data" }, { "name": "stdout", "output_type": "stream", "text": [ "{\n", " \"spill_count\": 5,\n", " \"load_count\": 5,\n", " \"spill_space\": 89280,\n", " \"load_space\": 89280,\n", " \"min_resident\": 35840,\n", " \"max_resident\": 53601,\n", " \"train_count\": 12,\n", " \"apply_count\": 10,\n", " \"metric_count\": 0,\n", " \"train_time\": 0.10233783721923828,\n", " \"apply_time\": 0.07904672622680664,\n", " \"metric_time\": 0,\n", " \"critical_count\": 1,\n", " \"critical_time\": 0.025415658950805664,\n", "}\n", "accuracy 62.42%\n" ] } ], "source": [ "n_batches = 3\n", "max_resident = 3 * math.ceil(train_data_space / n_batches)\n", "batches = mockup_data_loader(train_X, train_y, n_batches, \"pandas\")\n", "trained = fit_with_batches(fresh_pipeline(), batches, None, None, unique_class_labels, max_resident,\n", " PrioBatch(), partial_transform=False, verbose=2, progress_callback=None)\n", "print(f\"accuracy {accuracy_scorer(trained, test_X, test_y):.2%}\")" ] }, { "cell_type": "code", "execution_count": 9, "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "spill /tmp/tmp32eeypzi/X_-1_d1_None.pkl /tmp/tmp32eeypzi/y_-1_d1_None.pkl\n", "spill /tmp/tmp32eeypzi/X_-1_d2_None.pkl /tmp/tmp32eeypzi/y_-1_d2_None.pkl\n", "load /tmp/tmp32eeypzi/X_-1_d1_None.pkl /tmp/tmp32eeypzi/y_-1_d1_None.pkl\n", "spill /tmp/tmp32eeypzi/X_0_d0_None.pkl /tmp/tmp32eeypzi/y_0_d0_None.pkl\n", "load /tmp/tmp32eeypzi/X_-1_d2_None.pkl /tmp/tmp32eeypzi/y_-1_d2_None.pkl\n", "load /tmp/tmp32eeypzi/X_0_d0_None.pkl /tmp/tmp32eeypzi/y_0_d0_None.pkl\n", "spill /tmp/tmp32eeypzi/X_1_d2_None.pkl /tmp/tmp32eeypzi/y_1_d2_None.pkl\n", "load /tmp/tmp32eeypzi/X_1_d2_None.pkl /tmp/tmp32eeypzi/y_1_d2_None.pkl\n" ] }, { "data": { "image/svg+xml": [ "\n", "\n", "\n", "\n", "\n" ], "text/plain": [ "" ] }, "metadata": {}, "output_type": "display_data" }, { "name": "stdout", "output_type": "stream", "text": [ "{\n", " \"spill_count\": 4,\n", " \"load_count\": 4,\n", " \"spill_space\": 71440,\n", " \"load_space\": 71440,\n", " \"min_resident\": 35840,\n", " \"max_resident\": 53601,\n", " \"train_count\": 12,\n", " \"apply_count\": 10,\n", " \"metric_count\": 0,\n", " \"train_time\": 0.08783912658691406,\n", " \"apply_time\": 0.06755399703979492,\n", " \"metric_time\": 0,\n", " \"critical_count\": 1,\n", " \"critical_time\": 0.01776576042175293,\n", "}\n", "accuracy 62.42%\n" ] } ], "source": [ "n_batches = 3\n", "max_resident = 3 * math.ceil(train_data_space / n_batches)\n", "batches = mockup_data_loader(train_X, train_y, n_batches, \"pandas\")\n", "trained = fit_with_batches(fresh_pipeline(), batches, None, None, unique_class_labels, max_resident,\n", " PrioResourceAware(), partial_transform=False, verbose=2, progress_callback=None)\n", "print(f\"accuracy {accuracy_scorer(trained, test_X, test_y):.2%}\")" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "### Example: frozen trained prefix\n", "\n", "- pre-train on the first batch, then freeze, and train again on remaining batches\n", "- in the second training, SGD partial_fit *i* happens before scan *i+1*\n", "- to test and demonstrate that, setting partial_transform=\"score\"" ] }, { "cell_type": "code", "execution_count": 10, "metadata": {}, "outputs": [ { "data": { "image/svg+xml": [ "\n", "\n", "\n", "\n", "\n" ], "text/plain": [ "" ] }, "metadata": {}, "output_type": "display_data" }, { "name": "stdout", "output_type": "stream", "text": [ "{\n", " \"spill_count\": 0,\n", " \"load_count\": 0,\n", " \"spill_space\": 0,\n", " \"load_space\": 0,\n", " \"min_resident\": 35840,\n", " \"max_resident\": 9223372036854775807,\n", " \"train_count\": 6,\n", " \"apply_count\": 4,\n", " \"metric_count\": 0,\n", " \"train_time\": 0.029951095581054688,\n", " \"apply_time\": 0.018345117568969727,\n", " \"metric_time\": 0,\n", " \"critical_count\": 1,\n", " \"critical_time\": 0.019626379013061523,\n", "}\n" ] } ], "source": [ "class DemoProgressCallback: # ideally, should create a dynamically updating plot in Jupyter\n", " def __init__(self):\n", " self.scores = []\n", " \n", " def __call__(self, score_train, score_valid, n_batches_scanned, end_of_scanned_batches):\n", " self.scores.append(score_train)\n", " scores_s = \", \".join(f\"{s:.2%}\" for s in self.scores)\n", " print(f\"scanned {n_batches_scanned}, found end {end_of_scanned_batches}, callback state {scores_s}\")\n", " assert len(self.scores) == n_batches_scanned + (1 if end_of_scanned_batches else 0)\n", "\n", "n_batches = 3\n", "batches = mockup_data_loader(train_X, train_y, n_batches, \"pandas\")\n", "first_batch = next(iter(batches))\n", "trainable1 = fresh_pipeline()\n", "trained1 = fit_with_batches(trainable1, [first_batch], None, None, unique_class_labels, None,\n", " PrioBatch(), partial_transform=False, verbose=2, progress_callback=None)" ] }, { "cell_type": "code", "execution_count": 11, "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "scanned 1, found end False, callback state 69.96%\n", "scanned 2, found end False, callback state 69.96%, 69.51%\n" ] }, { "data": { "image/svg+xml": [ "\n", "\n", "\n", "\n", "\n" ], "text/plain": [ "" ] }, "metadata": {}, "output_type": "display_data" }, { "name": "stdout", "output_type": "stream", "text": [ "{\n", " \"spill_count\": 0,\n", " \"load_count\": 0,\n", " \"spill_space\": 0,\n", " \"load_space\": 0,\n", " \"min_resident\": 35680,\n", " \"max_resident\": 9223372036854775807,\n", " \"train_count\": 3,\n", " \"apply_count\": 9,\n", " \"metric_count\": 3,\n", " \"train_time\": 0.0074999332427978516,\n", " \"apply_time\": 0.05312967300415039,\n", " \"metric_time\": 0.012913227081298828,\n", " \"critical_count\": 1,\n", " \"critical_time\": 0.009376764297485352,\n", "}\n" ] } ], "source": [ "prefix2 = trained1.remove_last().freeze_trained()\n", "suffix2 = trained1.get_last()\n", "trainable2 = prefix2 >> suffix2\n", "trained = fit_with_batches(trainable2, batches, None, accuracy_scorer, unique_class_labels, None,\n", " PrioBatch(), partial_transform=\"score\", verbose=2, progress_callback=DemoProgressCallback())" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "### Example: partial_transform, start using transformer before it has seen all data\n", "- behavior not equivalent to sklearn, instead more similar to DL\n", "- but may be \"close enough\" if operators are \"convergent\"\n", "- advantage: need much less resident data\n", "- amenable to early stopping, extracting a checkpoint" ] }, { "cell_type": "code", "execution_count": 12, "metadata": {}, "outputs": [ { "data": { "image/svg+xml": [ "\n", "\n", "\n", "\n", "\n" ], "text/plain": [ "" ] }, "metadata": {}, "output_type": "display_data" }, { "name": "stdout", "output_type": "stream", "text": [ "scan INP(d0)\n" ] }, { "data": { "image/svg+xml": [ "\n", "\n", "\n", "\n", "\n" ], "text/plain": [ "" ] }, "metadata": {}, "output_type": "display_data" }, { "name": "stdout", "output_type": "stream", "text": [ "to_monoid SimpleImputer(d0)\n" ] }, { "data": { "image/svg+xml": [ "\n", "\n", "\n", "\n", "\n" ], "text/plain": [ "" ] }, "metadata": {}, "output_type": "display_data" }, { "name": "stdout", "output_type": "stream", "text": [ "transform SimpleImputer(d0)\n" ] }, { "data": { "image/svg+xml": [ "\n", "\n", "\n", "\n", "\n" ], "text/plain": [ "" ] }, "metadata": {}, "output_type": "display_data" }, { "name": "stdout", "output_type": "stream", "text": [ "to_monoid MinMaxScaler(d0)\n" ] }, { "data": { "image/svg+xml": [ "\n", "\n", "\n", "\n", "\n" ], "text/plain": [ "" ] }, "metadata": {}, "output_type": "display_data" }, { "name": "stdout", "output_type": "stream", "text": [ "transform MinMaxScaler(d0)\n" ] }, { "data": { "image/svg+xml": [ "\n", "\n", "\n", "\n", "\n" ], "text/plain": [ "" ] }, "metadata": {}, "output_type": "display_data" }, { "name": "stdout", "output_type": "stream", "text": [ "partial_fit SGDClassifier(d0)\n" ] }, { "data": { "image/svg+xml": [ "\n", "\n", "\n", "\n", "\n" ], "text/plain": [ "" ] }, "metadata": {}, "output_type": "display_data" }, { "name": "stdout", "output_type": "stream", "text": [ "scan INP(d1)\n" ] }, { "data": { "image/svg+xml": [ "\n", "\n", "\n", "\n", "\n" ], "text/plain": [ "" ] }, "metadata": {}, "output_type": "display_data" }, { "name": "stdout", "output_type": "stream", "text": [ "to_monoid SimpleImputer(d1)\n" ] }, { "data": { "image/svg+xml": [ "\n", "\n", "\n", "\n", "\n" ], "text/plain": [ "" ] }, "metadata": {}, "output_type": "display_data" }, { "name": "stdout", "output_type": "stream", "text": [ "combine SimpleImputer(d0,d1)\n" ] }, { "data": { "image/svg+xml": [ "\n", "\n", "\n", "\n", "\n" ], "text/plain": [ "" ] }, "metadata": {}, "output_type": "display_data" }, { "name": "stdout", "output_type": "stream", "text": [ "transform SimpleImputer(d1)\n" ] }, { "data": { "image/svg+xml": [ "\n", "\n", "\n", "\n", "\n" ], "text/plain": [ "" ] }, "metadata": {}, "output_type": "display_data" }, { "name": "stdout", "output_type": "stream", "text": [ "to_monoid MinMaxScaler(d1)\n" ] }, { "data": { "image/svg+xml": [ "\n", "\n", "\n", "\n", "\n" ], "text/plain": [ "" ] }, "metadata": {}, "output_type": "display_data" }, { "name": "stdout", "output_type": "stream", "text": [ "combine MinMaxScaler(d0,d1)\n" ] }, { "data": { "image/svg+xml": [ "\n", "\n", "\n", "\n", "\n" ], "text/plain": [ "" ] }, "metadata": {}, "output_type": "display_data" }, { "name": "stdout", "output_type": "stream", "text": [ "transform MinMaxScaler(d1)\n" ] }, { "data": { "image/svg+xml": [ "\n", "\n", "\n", "\n", "\n" ], "text/plain": [ "" ] }, "metadata": {}, "output_type": "display_data" }, { "name": "stdout", "output_type": "stream", "text": [ "partial_fit SGDClassifier(d0,d1)\n" ] }, { "data": { "image/svg+xml": [ "\n", "\n", "\n", "\n", "\n" ], "text/plain": [ "" ] }, "metadata": {}, "output_type": "display_data" }, { "name": "stdout", "output_type": "stream", "text": [ "scan INP(d2)\n" ] }, { "data": { "image/svg+xml": [ "\n", "\n", "\n", "\n", "\n" ], "text/plain": [ "" ] }, "metadata": {}, "output_type": "display_data" }, { "name": "stdout", "output_type": "stream", "text": [ "to_monoid SimpleImputer(d2)\n" ] }, { "data": { "image/svg+xml": [ "\n", "\n", "\n", "\n", "\n" ], "text/plain": [ "" ] }, "metadata": {}, "output_type": "display_data" }, { "name": "stdout", "output_type": "stream", "text": [ "combine SimpleImputer(d0,d1,d2)\n" ] }, { "data": { "image/svg+xml": [ "\n", "\n", "\n", "\n", "\n" ], "text/plain": [ "" ] }, "metadata": {}, "output_type": "display_data" }, { "name": "stdout", "output_type": "stream", "text": [ "transform SimpleImputer(d2)\n" ] }, { "data": { "image/svg+xml": [ "\n", "\n", "\n", "\n", "\n" ], "text/plain": [ "" ] }, "metadata": {}, "output_type": "display_data" }, { "name": "stdout", "output_type": "stream", "text": [ "to_monoid MinMaxScaler(d2)\n" ] }, { "data": { "image/svg+xml": [ "\n", "\n", "\n", "\n", "\n" ], "text/plain": [ "" ] }, "metadata": {}, "output_type": "display_data" }, { "name": "stdout", "output_type": "stream", "text": [ "combine MinMaxScaler(d0,d1,d2)\n" ] }, { "data": { "image/svg+xml": [ "\n", "\n", "\n", "\n", "\n" ], "text/plain": [ "" ] }, "metadata": {}, "output_type": "display_data" }, { "name": "stdout", "output_type": "stream", "text": [ "transform MinMaxScaler(d2)\n" ] }, { "data": { "image/svg+xml": [ "\n", "\n", "\n", "\n", "\n" ], "text/plain": [ "" ] }, "metadata": {}, "output_type": "display_data" }, { "name": "stdout", "output_type": "stream", "text": [ "partial_fit SGDClassifier(d0,d1,d2)\n" ] }, { "data": { "image/svg+xml": [ "\n", "\n", "\n", "\n", "\n" ], "text/plain": [ "" ] }, "metadata": {}, "output_type": "display_data" }, { "name": "stdout", "output_type": "stream", "text": [ "scan INP(d3)\n" ] }, { "data": { "image/svg+xml": [ "\n", "\n", "\n", "\n", "\n" ], "text/plain": [ "" ] }, "metadata": {}, "output_type": "display_data" }, { "name": "stdout", "output_type": "stream", "text": [ "combine SimpleImputer(d*)\n" ] }, { "data": { "image/svg+xml": [ "\n", "\n", "\n", "\n", "\n" ], "text/plain": [ "" ] }, "metadata": {}, "output_type": "display_data" }, { "name": "stdout", "output_type": "stream", "text": [ "combine MinMaxScaler(d*)\n" ] }, { "data": { "image/svg+xml": [ "\n", "\n", "\n", "\n", "\n" ], "text/plain": [ "" ] }, "metadata": {}, "output_type": "display_data" }, { "name": "stdout", "output_type": "stream", "text": [ "partial_fit SGDClassifier(d*)\n" ] }, { "data": { "image/svg+xml": [ "\n", "\n", "\n", "\n", "\n" ], "text/plain": [ "" ] }, "metadata": {}, "output_type": "display_data" }, { "name": "stdout", "output_type": "stream", "text": [ "{\n", " \"spill_count\": 0,\n", " \"load_count\": 0,\n", " \"spill_space\": 0,\n", " \"load_space\": 0,\n", " \"min_resident\": 35840,\n", " \"max_resident\": 53601,\n", " \"train_count\": 16,\n", " \"apply_count\": 10,\n", " \"metric_count\": 0,\n", " \"train_time\": 0.11674237251281738,\n", " \"apply_time\": 0.092437744140625,\n", " \"metric_time\": 0,\n", " \"critical_count\": 1,\n", " \"critical_time\": 0.022518157958984375,\n", "}\n", "accuracy 70.30%\n" ] } ], "source": [ "n_batches = 3\n", "max_resident = 3 * math.ceil(train_data_space / n_batches)\n", "batches = mockup_data_loader(train_X, train_y, n_batches, \"pandas\")\n", "trained = fit_with_batches(fresh_pipeline(), batches, None, None, unique_class_labels,\n", " max_resident, PrioBatch(), partial_transform=True, verbose=3, progress_callback=None)\n", "print(f\"accuracy {accuracy_scorer(trained, test_X, test_y):.2%}\")" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "### Example: learning curve during fit_with_batches with partial_transform" ] }, { "cell_type": "code", "execution_count": 13, "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "spill /tmp/tmpv3zynub2/X_-1_d0_None.pkl /tmp/tmpv3zynub2/y_-1_d0_None.pkl\n", "load /tmp/tmpv3zynub2/X_-1_d0_None.pkl /tmp/tmpv3zynub2/y_-1_d0_None.pkl\n", "scanned 1, found end False, callback state 68.75%\n", "scanned 2, found end False, callback state 68.75%, 69.96%\n", "scanned 3, found end False, callback state 68.75%, 69.96%, 71.30%\n" ] }, { "data": { "image/svg+xml": [ "\n", "\n", "\n", "\n", "\n" ], "text/plain": [ "" ] }, "metadata": {}, "output_type": "display_data" }, { "name": "stdout", "output_type": "stream", "text": [ "{\n", " \"spill_count\": 1,\n", " \"load_count\": 1,\n", " \"spill_space\": 17920,\n", " \"load_space\": 17920,\n", " \"min_resident\": 53760,\n", " \"max_resident\": 53601,\n", " \"train_count\": 16,\n", " \"apply_count\": 13,\n", " \"metric_count\": 4,\n", " \"train_time\": 0.09845256805419922,\n", " \"apply_time\": 0.10098576545715332,\n", " \"metric_time\": 0.025612354278564453,\n", " \"critical_count\": 1,\n", " \"critical_time\": 0.02614283561706543,\n", "}\n", "accuracy 70.30%\n" ] } ], "source": [ "n_batches = 3\n", "max_resident = 3 * math.ceil(train_data_space / n_batches)\n", "batches = mockup_data_loader(train_X, train_y, n_batches, \"pandas\")\n", "trained = fit_with_batches(fresh_pipeline(), batches, None, accuracy_scorer, unique_class_labels,\n", " max_resident, PrioBatch(), partial_transform=True, verbose=2, progress_callback=DemoProgressCallback())\n", "print(f\"accuracy {accuracy_scorer(trained, test_X, test_y):.2%}\")" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "### Example: cross-validation without batching, using same-fold data for training\n", "- behavior equivalent to sklearn\n", "- reuse of lifted coefficients limited to first operator\n", "- disambiguate some tasks by `\\f`, where f is the left-out fold\n", "- using associativity where possible, partial_fit otherwise" ] }, { "cell_type": "code", "execution_count": 14, "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "spill /tmp/tmpaduz2lqh/X_-1_d0_None.pkl /tmp/tmpaduz2lqh/y_-1_d0_None.pkl\n", "spill /tmp/tmpaduz2lqh/X_0_e0_f.pkl /tmp/tmpaduz2lqh/y_0_e0_f.pkl\n", "spill /tmp/tmpaduz2lqh/X_-1_e0_None.pkl /tmp/tmpaduz2lqh/y_-1_e0_None.pkl\n", "load /tmp/tmpaduz2lqh/X_-1_e0_None.pkl /tmp/tmpaduz2lqh/y_-1_e0_None.pkl\n", "spill /tmp/tmpaduz2lqh/X_-1_f0_None.pkl /tmp/tmpaduz2lqh/y_-1_f0_None.pkl\n", "spill /tmp/tmpaduz2lqh/X_0_f0_e.pkl /tmp/tmpaduz2lqh/y_0_f0_e.pkl\n", "load /tmp/tmpaduz2lqh/X_-1_d0_None.pkl /tmp/tmpaduz2lqh/y_-1_d0_None.pkl\n", "spill /tmp/tmpaduz2lqh/X_0_f0_f.pkl /tmp/tmpaduz2lqh/y_0_f0_f.pkl\n", "spill /tmp/tmpaduz2lqh/X_-1_e0_None.pkl /tmp/tmpaduz2lqh/y_-1_e0_None.pkl\n", "spill /tmp/tmpaduz2lqh/X_-1_d0_None.pkl /tmp/tmpaduz2lqh/y_-1_d0_None.pkl\n", "load /tmp/tmpaduz2lqh/X_0_e0_f.pkl /tmp/tmpaduz2lqh/y_0_e0_f.pkl\n", "load /tmp/tmpaduz2lqh/X_0_f0_e.pkl /tmp/tmpaduz2lqh/y_0_f0_e.pkl\n", "load /tmp/tmpaduz2lqh/X_-1_e0_None.pkl /tmp/tmpaduz2lqh/y_-1_e0_None.pkl\n", "load /tmp/tmpaduz2lqh/X_0_f0_f.pkl /tmp/tmpaduz2lqh/y_0_f0_f.pkl\n", "load /tmp/tmpaduz2lqh/X_-1_f0_None.pkl /tmp/tmpaduz2lqh/y_-1_f0_None.pkl\n", "load /tmp/tmpaduz2lqh/X_-1_d0_None.pkl /tmp/tmpaduz2lqh/y_-1_d0_None.pkl\n" ] }, { "data": { "image/svg+xml": [ "\n", "\n", "\n", "\n", "\n" ], "text/plain": [ "" ] }, "metadata": {}, "output_type": "display_data" }, { "name": "stdout", "output_type": "stream", "text": [ "{\n", " \"spill_count\": 8,\n", " \"load_count\": 8,\n", " \"spill_space\": 142880,\n", " \"load_space\": 142880,\n", " \"min_resident\": 107200,\n", " \"max_resident\": 107200,\n", " \"train_count\": 24,\n", " \"apply_count\": 26,\n", " \"metric_count\": 6,\n", " \"train_time\": 0.16528654098510742,\n", " \"apply_time\": 0.5534472465515137,\n", " \"metric_time\": 0.02211618423461914,\n", " \"critical_count\": 1,\n", " \"critical_time\": 0.1302642822265625,\n", "}\n", "accuracies 54.91%, 58.30%, 53.81%\n" ] } ], "source": [ "n_batches = 1\n", "max_resident = 2 * math.ceil(train_data_space / n_batches)\n", "batches = mockup_data_loader(train_X, train_y, n_batches, \"pandas\")\n", "scores = cross_val_score(fresh_pipeline(\"bbc\"), batches, accuracy_scorer, KFold(3),\n", " unique_class_labels, max_resident, PrioResourceAware(), same_fold=True, verbose=2)\n", "print(\"accuracies \" + \", \".join(f\"{score:.2%}\" for score in scores))" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "### Example: cross-validation with batching, using same-fold data for training\n", "- compared to cross-validation without batching, requires less resident memory" ] }, { "cell_type": "code", "execution_count": 15, "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "spill /tmp/tmpxmtt9rld/X_-1_*1_None.pkl /tmp/tmpxmtt9rld/y_-1_*1_None.pkl\n", "spill /tmp/tmpxmtt9rld/X_-1_d0_None.pkl /tmp/tmpxmtt9rld/y_-1_d0_None.pkl\n", "spill /tmp/tmpxmtt9rld/X_-1_e0_None.pkl /tmp/tmpxmtt9rld/y_-1_e0_None.pkl\n", "load /tmp/tmpxmtt9rld/X_-1_*1_None.pkl /tmp/tmpxmtt9rld/y_-1_*1_None.pkl\n", "spill /tmp/tmpxmtt9rld/X_-1_f0_None.pkl /tmp/tmpxmtt9rld/y_-1_f0_None.pkl\n", "spill /tmp/tmpxmtt9rld/X_-1_d1_None.pkl /tmp/tmpxmtt9rld/y_-1_d1_None.pkl\n", "spill /tmp/tmpxmtt9rld/X_0_e1_f.pkl /tmp/tmpxmtt9rld/y_0_e1_f.pkl\n", "spill /tmp/tmpxmtt9rld/X_-1_e1_None.pkl /tmp/tmpxmtt9rld/y_-1_e1_None.pkl\n", "load /tmp/tmpxmtt9rld/X_-1_f0_None.pkl /tmp/tmpxmtt9rld/y_-1_f0_None.pkl\n", "spill /tmp/tmpxmtt9rld/X_-1_f1_None.pkl /tmp/tmpxmtt9rld/y_-1_f1_None.pkl\n", "spill /tmp/tmpxmtt9rld/X_0_f1_f.pkl /tmp/tmpxmtt9rld/y_0_f1_f.pkl\n", "spill /tmp/tmpxmtt9rld/X_0_f1_e.pkl /tmp/tmpxmtt9rld/y_0_f1_e.pkl\n", "load /tmp/tmpxmtt9rld/X_-1_d0_None.pkl /tmp/tmpxmtt9rld/y_-1_d0_None.pkl\n", "spill /tmp/tmpxmtt9rld/X_0_f1_d.pkl /tmp/tmpxmtt9rld/y_0_f1_d.pkl\n", "spill /tmp/tmpxmtt9rld/X_-1_f0_None.pkl /tmp/tmpxmtt9rld/y_-1_f0_None.pkl\n", "spill /tmp/tmpxmtt9rld/X_0_f0_d.pkl /tmp/tmpxmtt9rld/y_0_f0_d.pkl\n", "spill /tmp/tmpxmtt9rld/X_0_f0_e.pkl /tmp/tmpxmtt9rld/y_0_f0_e.pkl\n", "spill /tmp/tmpxmtt9rld/X_0_f0_f.pkl /tmp/tmpxmtt9rld/y_0_f0_f.pkl\n", "load /tmp/tmpxmtt9rld/X_-1_d1_None.pkl /tmp/tmpxmtt9rld/y_-1_d1_None.pkl\n", "spill /tmp/tmpxmtt9rld/X_-1_d0_None.pkl /tmp/tmpxmtt9rld/y_-1_d0_None.pkl\n", "spill /tmp/tmpxmtt9rld/X_0_d0_d.pkl /tmp/tmpxmtt9rld/y_0_d0_d.pkl\n", "spill /tmp/tmpxmtt9rld/X_0_d1_d.pkl /tmp/tmpxmtt9rld/y_0_d1_d.pkl\n", "spill /tmp/tmpxmtt9rld/X_-1_d1_None.pkl /tmp/tmpxmtt9rld/y_-1_d1_None.pkl\n", "load /tmp/tmpxmtt9rld/X_0_f0_e.pkl /tmp/tmpxmtt9rld/y_0_f0_e.pkl\n", "load /tmp/tmpxmtt9rld/X_0_f1_e.pkl /tmp/tmpxmtt9rld/y_0_f1_e.pkl\n", "load /tmp/tmpxmtt9rld/X_-1_e0_None.pkl /tmp/tmpxmtt9rld/y_-1_e0_None.pkl\n", "spill /tmp/tmpxmtt9rld/X_0_d1_f.pkl /tmp/tmpxmtt9rld/y_0_d1_f.pkl\n", "spill /tmp/tmpxmtt9rld/X_-1_e0_None.pkl /tmp/tmpxmtt9rld/y_-1_e0_None.pkl\n", "load /tmp/tmpxmtt9rld/X_0_f0_f.pkl /tmp/tmpxmtt9rld/y_0_f0_f.pkl\n", "load /tmp/tmpxmtt9rld/X_0_f1_f.pkl /tmp/tmpxmtt9rld/y_0_f1_f.pkl\n", "load /tmp/tmpxmtt9rld/X_0_d1_f.pkl /tmp/tmpxmtt9rld/y_0_d1_f.pkl\n", "load /tmp/tmpxmtt9rld/X_-1_e0_None.pkl /tmp/tmpxmtt9rld/y_-1_e0_None.pkl\n", "load /tmp/tmpxmtt9rld/X_-1_e1_None.pkl /tmp/tmpxmtt9rld/y_-1_e1_None.pkl\n", "spill /tmp/tmpxmtt9rld/X_1_f1_f.pkl /tmp/tmpxmtt9rld/y_1_f1_f.pkl\n", "load /tmp/tmpxmtt9rld/X_0_f0_d.pkl /tmp/tmpxmtt9rld/y_0_f0_d.pkl\n", "load /tmp/tmpxmtt9rld/X_0_f1_d.pkl /tmp/tmpxmtt9rld/y_0_f1_d.pkl\n", "load /tmp/tmpxmtt9rld/X_0_d0_d.pkl /tmp/tmpxmtt9rld/y_0_d0_d.pkl\n", "load /tmp/tmpxmtt9rld/X_-1_d0_None.pkl /tmp/tmpxmtt9rld/y_-1_d0_None.pkl\n", "load /tmp/tmpxmtt9rld/X_0_d1_d.pkl /tmp/tmpxmtt9rld/y_0_d1_d.pkl\n", "load /tmp/tmpxmtt9rld/X_-1_d1_None.pkl /tmp/tmpxmtt9rld/y_-1_d1_None.pkl\n", "load /tmp/tmpxmtt9rld/X_0_e1_f.pkl /tmp/tmpxmtt9rld/y_0_e1_f.pkl\n", "load /tmp/tmpxmtt9rld/X_-1_f0_None.pkl /tmp/tmpxmtt9rld/y_-1_f0_None.pkl\n", "load /tmp/tmpxmtt9rld/X_1_f1_f.pkl /tmp/tmpxmtt9rld/y_1_f1_f.pkl\n", "load /tmp/tmpxmtt9rld/X_-1_f1_None.pkl /tmp/tmpxmtt9rld/y_-1_f1_None.pkl\n" ] }, { "data": { "image/svg+xml": [ "\n", "\n", "\n", "\n", "\n" ], "text/plain": [ "" ] }, "metadata": {}, "output_type": "display_data" }, { "name": "stdout", "output_type": "stream", "text": [ "{\n", " \"spill_count\": 22,\n", " \"load_count\": 22,\n", " \"spill_space\": 214160,\n", " \"load_space\": 214160,\n", " \"min_resident\": 53600,\n", " \"max_resident\": 53600,\n", " \"train_count\": 39,\n", " \"apply_count\": 51,\n", " \"metric_count\": 9,\n", " \"train_time\": 0.24051737785339355,\n", " \"apply_time\": 0.9827837944030762,\n", " \"metric_time\": 0.03426671028137207,\n", " \"critical_count\": 1,\n", " \"critical_time\": 0.21920561790466309,\n", "}\n", "accuracies 66.07%, 62.05%, 63.06%\n" ] } ], "source": [ "n_batches = 2\n", "max_resident = 2 * math.ceil(train_data_space / n_batches)\n", "batches = mockup_data_loader(train_X, train_y, n_batches, \"pandas\")\n", "scores = cross_val_score(fresh_pipeline(\"bbc\"), batches, accuracy_scorer, KFold(3),\n", " unique_class_labels, max_resident, PrioResourceAware(), same_fold=True, verbose=2)\n", "print(\"accuracies \" + \", \".join(f\"{score:.2%}\" for score in scores))" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "### Example: cross-validation with out-of-fold samples\n", "- not sklearn compatible\n", "- might generalize better by following best practices for stacking ensembles\n", "- depends on transformers being convergent\n", "- fewer tasks overall than for cross-validation with in-fold samples" ] }, { "cell_type": "code", "execution_count": 16, "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "spill /tmp/tmpwewe5gvy/X_-1_d0_None.pkl /tmp/tmpwewe5gvy/y_-1_d0_None.pkl\n", "spill /tmp/tmpwewe5gvy/X_-1_e0_None.pkl /tmp/tmpwewe5gvy/y_-1_e0_None.pkl\n", "load /tmp/tmpwewe5gvy/X_-1_e0_None.pkl /tmp/tmpwewe5gvy/y_-1_e0_None.pkl\n", "load /tmp/tmpwewe5gvy/X_-1_d0_None.pkl /tmp/tmpwewe5gvy/y_-1_d0_None.pkl\n", "spill /tmp/tmpwewe5gvy/X_-1_f0_None.pkl /tmp/tmpwewe5gvy/y_-1_f0_None.pkl\n", "spill /tmp/tmpwewe5gvy/X_-1_e0_None.pkl /tmp/tmpwewe5gvy/y_-1_e0_None.pkl\n", "load /tmp/tmpwewe5gvy/X_-1_e0_None.pkl /tmp/tmpwewe5gvy/y_-1_e0_None.pkl\n", "load /tmp/tmpwewe5gvy/X_-1_f0_None.pkl /tmp/tmpwewe5gvy/y_-1_f0_None.pkl\n" ] }, { "data": { "image/svg+xml": [ "\n", "\n", "\n", "\n", "\n" ], "text/plain": [ "" ] }, "metadata": {}, "output_type": "display_data" }, { "name": "stdout", "output_type": "stream", "text": [ "{\n", " \"spill_count\": 4,\n", " \"load_count\": 4,\n", " \"spill_space\": 71440,\n", " \"load_space\": 71440,\n", " \"min_resident\": 107200,\n", " \"max_resident\": 107200,\n", " \"train_count\": 18,\n", " \"apply_count\": 14,\n", " \"metric_count\": 6,\n", " \"train_time\": 0.09293031692504883,\n", " \"apply_time\": 0.49991703033447266,\n", " \"metric_time\": 0.02008819580078125,\n", " \"critical_count\": 1,\n", " \"critical_time\": 0.1726703643798828,\n", "}\n", "accuracies 60.71%, 56.50%, 58.74%\n" ] } ], "source": [ "n_batches = 1\n", "max_resident = 2 * math.ceil(train_data_space / n_batches)\n", "batches = mockup_data_loader(train_X, train_y, n_batches, \"pandas\")\n", "scores = cross_val_score(fresh_pipeline(\"bbc\"), batches, accuracy_scorer, KFold(3),\n", " unique_class_labels, max_resident, PrioResourceAware(), same_fold=False, verbose=2)\n", "print(\"accuracies \" + \", \".join(f\"{score:.2%}\" for score in scores))" ] }, { "cell_type": "code", "execution_count": 17, "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "spill /tmp/tmpgwdyqcux/X_-1_*1_None.pkl /tmp/tmpgwdyqcux/y_-1_*1_None.pkl\n", "spill /tmp/tmpgwdyqcux/X_-1_d0_None.pkl /tmp/tmpgwdyqcux/y_-1_d0_None.pkl\n", "spill /tmp/tmpgwdyqcux/X_-1_e0_None.pkl /tmp/tmpgwdyqcux/y_-1_e0_None.pkl\n", "load /tmp/tmpgwdyqcux/X_-1_*1_None.pkl /tmp/tmpgwdyqcux/y_-1_*1_None.pkl\n", "spill /tmp/tmpgwdyqcux/X_-1_f0_None.pkl /tmp/tmpgwdyqcux/y_-1_f0_None.pkl\n", "spill /tmp/tmpgwdyqcux/X_-1_d1_None.pkl /tmp/tmpgwdyqcux/y_-1_d1_None.pkl\n", "spill /tmp/tmpgwdyqcux/X_-1_e1_None.pkl /tmp/tmpgwdyqcux/y_-1_e1_None.pkl\n", "load /tmp/tmpgwdyqcux/X_-1_f0_None.pkl /tmp/tmpgwdyqcux/y_-1_f0_None.pkl\n", "load /tmp/tmpgwdyqcux/X_-1_d0_None.pkl /tmp/tmpgwdyqcux/y_-1_d0_None.pkl\n", "spill /tmp/tmpgwdyqcux/X_-1_f1_None.pkl /tmp/tmpgwdyqcux/y_-1_f1_None.pkl\n", "load /tmp/tmpgwdyqcux/X_-1_d1_None.pkl /tmp/tmpgwdyqcux/y_-1_d1_None.pkl\n", "spill /tmp/tmpgwdyqcux/X_0_f1_f.pkl /tmp/tmpgwdyqcux/y_0_f1_f.pkl\n", "spill /tmp/tmpgwdyqcux/X_-1_f0_None.pkl /tmp/tmpgwdyqcux/y_-1_f0_None.pkl\n", "spill /tmp/tmpgwdyqcux/X_0_f0_f.pkl /tmp/tmpgwdyqcux/y_0_f0_f.pkl\n", "load /tmp/tmpgwdyqcux/X_-1_e0_None.pkl /tmp/tmpgwdyqcux/y_-1_e0_None.pkl\n", "spill /tmp/tmpgwdyqcux/X_-1_d1_None.pkl /tmp/tmpgwdyqcux/y_-1_d1_None.pkl\n", "spill /tmp/tmpgwdyqcux/X_-1_e0_None.pkl /tmp/tmpgwdyqcux/y_-1_e0_None.pkl\n", "load /tmp/tmpgwdyqcux/X_-1_e1_None.pkl /tmp/tmpgwdyqcux/y_-1_e1_None.pkl\n", "spill /tmp/tmpgwdyqcux/X_1_e0_e.pkl /tmp/tmpgwdyqcux/y_1_e0_e.pkl\n", "spill /tmp/tmpgwdyqcux/X_-1_e1_None.pkl /tmp/tmpgwdyqcux/y_-1_e1_None.pkl\n", "load /tmp/tmpgwdyqcux/X_0_f0_f.pkl /tmp/tmpgwdyqcux/y_0_f0_f.pkl\n", "spill /tmp/tmpgwdyqcux/X_1_e1_e.pkl /tmp/tmpgwdyqcux/y_1_e1_e.pkl\n", "load /tmp/tmpgwdyqcux/X_-1_f0_None.pkl /tmp/tmpgwdyqcux/y_-1_f0_None.pkl\n", "load /tmp/tmpgwdyqcux/X_0_f1_f.pkl /tmp/tmpgwdyqcux/y_0_f1_f.pkl\n", "load /tmp/tmpgwdyqcux/X_-1_f1_None.pkl /tmp/tmpgwdyqcux/y_-1_f1_None.pkl\n", "load /tmp/tmpgwdyqcux/X_-1_d1_None.pkl /tmp/tmpgwdyqcux/y_-1_d1_None.pkl\n", "load /tmp/tmpgwdyqcux/X_1_e0_e.pkl /tmp/tmpgwdyqcux/y_1_e0_e.pkl\n", "load /tmp/tmpgwdyqcux/X_-1_e0_None.pkl /tmp/tmpgwdyqcux/y_-1_e0_None.pkl\n", "load /tmp/tmpgwdyqcux/X_1_e1_e.pkl /tmp/tmpgwdyqcux/y_1_e1_e.pkl\n", "load /tmp/tmpgwdyqcux/X_-1_e1_None.pkl /tmp/tmpgwdyqcux/y_-1_e1_None.pkl\n" ] }, { "data": { "image/svg+xml": [ "\n", "\n", "\n", "\n", "\n" ], "text/plain": [ "" ] }, "metadata": {}, "output_type": "display_data" }, { "name": "stdout", "output_type": "stream", "text": [ "{\n", " \"spill_count\": 15,\n", " \"load_count\": 15,\n", " \"spill_space\": 151840,\n", " \"load_space\": 151840,\n", " \"min_resident\": 53600,\n", " \"max_resident\": 53600,\n", " \"train_count\": 27,\n", " \"apply_count\": 27,\n", " \"metric_count\": 9,\n", " \"train_time\": 0.21664977073669434,\n", " \"apply_time\": 0.6339118480682373,\n", " \"metric_time\": 0.03318333625793457,\n", " \"critical_count\": 1,\n", " \"critical_time\": 0.10162663459777832,\n", "}\n", "accuracies 62.50%, 59.82%, 68.47%\n" ] } ], "source": [ "n_batches = 2\n", "max_resident = 2 * math.ceil(train_data_space / n_batches)\n", "batches = mockup_data_loader(train_X, train_y, n_batches, \"pandas\")\n", "scores = cross_val_score(fresh_pipeline(\"bbc\"), batches, accuracy_scorer, KFold(3),\n", " unique_class_labels, max_resident, PrioResourceAware(), same_fold=False, verbose=2)\n", "print(\"accuracies \" + \", \".join(f\"{score:.2%}\" for score in scores))" ] } ], "metadata": { "kernelspec": { "display_name": "Python 3 (ipykernel)", "language": "python", "name": "python3" }, "language_info": { "codemirror_mode": { "name": "ipython", "version": 3 }, "file_extension": ".py", "mimetype": "text/x-python", "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", "version": "3.7.13" } }, "nbformat": 4, "nbformat_minor": 4 }