{
"cells": [
{
"cell_type": "markdown",
"metadata": {},
"source": [
"# MNIST classification using Extreme Learning Machines\n"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": [
"import numpy as np\n",
"import time\n",
"from scipy.stats import uniform\n",
"from sklearn.base import clone\n",
"from sklearn.datasets import fetch_openml\n",
"from sklearn.linear_model import Ridge\n",
"from sklearn.preprocessing import MinMaxScaler\n",
"from sklearn.model_selection import RandomizedSearchCV, GridSearchCV, StratifiedKFold, ParameterGrid, cross_validate\n",
"from sklearn.utils.fixes import loguniform\n",
"from sklearn.metrics import accuracy_score\n",
"\n",
"from pyrcn.model_selection import SequentialSearchCV\n",
"from pyrcn.extreme_learning_machine import ELMClassifier"
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"# Load the dataset"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": [
"X, y = fetch_openml('mnist_784', version=1, return_X_y=True, as_frame=False)"
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"# Train test split. \n",
"Normalize to a range between [-1, 1]."
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": [
"X = MinMaxScaler(feature_range=(-1,1)).fit_transform(X=X)\n",
"X_train, X_test = X[:60000], X[60000:]\n",
"y_train, y_test = y[:60000].astype(int), y[60000:].astype(int)"
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"# Prepare sequential hyperparameter tuning"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": [
"initially_fixed_params = {'hidden_layer_size': 500,\n",
" 'input_activation': 'tanh',\n",
" 'k_in': 10,\n",
" 'bias_scaling': 0.0,\n",
" 'alpha': 1e-5,\n",
" 'random_state': 42}\n",
"\n",
"step1_params = {'input_scaling': loguniform(1e-5, 1e1)}\n",
"kwargs1 = {'random_state': 42,\n",
" 'verbose': 1,\n",
" 'n_jobs': -1,\n",
" 'n_iter': 50,\n",
" 'scoring': 'accuracy'}\n",
"step2_params = {'bias_scaling': np.linspace(0.0, 1.6, 16)}\n",
"kwargs2 = {'verbose': 5,\n",
" 'n_jobs': -1,\n",
" 'scoring': 'accuracy'}\n",
"\n",
"elm = ELMClassifier(regressor=Ridge(), **initially_fixed_params)\n",
"\n",
"# The searches are defined similarly to the steps of a sklearn.pipeline.Pipeline:\n",
"searches = [('step1', RandomizedSearchCV, step1_params, kwargs1),\n",
" ('step2', GridSearchCV, step2_params, kwargs2)]"
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"# Perform the sequential search"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": [
"sequential_search = SequentialSearchCV(elm, searches=searches).fit(X_train, y_train)"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": [
"best_params = sequential_search.best_estimator_.get_params()"
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"# Test\n",
"Increase reservoir size and compare different regression methods. Make sure that you have enough RAM for that, because all regression types without chunk size require a lot of memory. This is the reason why, especially for large datasets, the incremental regression is recommeded."
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": [
"base_elm_ridge = ELMClassifier(regressor=Ridge(), **best_params)\n",
"base_elm_inc = ELMClassifier(**best_params)\n",
"base_elm_inc_chunk = clone(base_elm_inc).set_params(chunk_size=6000)\n",
"\n",
"param_grid = {'hidden_layer_size': [500, 1000, 2000, 4000, 8000, 16000]}\n",
"\n",
"print(\"CV results\\tFit time\\tInference time\\tAccuracy score\\tSize[Bytes]\")\n",
"for params in ParameterGrid(param_grid):\n",
" elm_ridge_cv = cross_validate(clone(base_elm_ridge).set_params(**params), X=X_train, y=y_train)\n",
" t1 = time.time()\n",
" elm_ridge = clone(base_elm_ridge).set_params(**params).fit(X_train, y_train)\n",
" t_fit = time.time() - t1\n",
" mem_size = elm_ridge.__sizeof__()\n",
" t1 = time.time()\n",
" acc_score = accuracy_score(y_test, elm_ridge.predict(X_test))\n",
" t_inference = time.time() - t1\n",
" print(\"{0}\\t{1}\\t{2}\\t{3}\\t{4}\".format(elm_ridge_cv, t_fit, t_inference, acc_score, mem_size))\n",
" elm_inc_cv = cross_validate(clone(base_elm_inc).set_params(**params), X=X_train, y=y_train)\n",
" t1 = time.time()\n",
" elm_inc = clone(base_elm_inc).set_params(**params).fit(X_train, y_train)\n",
" t_fit = time.time() - t1\n",
" mem_size = elm_inc.__sizeof__()\n",
" t1 = time.time()\n",
" acc_score = accuracy_score(y_test, elm_inc.predict(X_test))\n",
" t_inference = time.time() - t1\n",
" print(\"{0}\\t{1}\\t{2}\\t{3}\\t{4}\".format(elm_inc_cv, t_fit, t_inference, acc_score, mem_size))\n",
" elm_inc_chunk_cv = cross_validate(clone(base_elm_inc_chunk).set_params(**params), X=X_train, y=y_train)\n",
" t1 = time.time()\n",
" elm_inc_chunk = clone(base_elm_inc_chunk).set_params(**params).fit(X_train, y_train)\n",
" t_fit = time.time() - t1\n",
" mem_size = elm_inc_chunk.__sizeof__()\n",
" t1 = time.time()\n",
" acc_score = accuracy_score(y_test, elm_inc_chunk.predict(X_test))\n",
" t_inference = time.time() - t1\n",
" print(\"{0}\\t{1}\\t{2}\\t{3}\\t{4}\".format(elm_inc_chunk_cv, t_fit, t_inference, acc_score, mem_size))"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": []
}
],
"metadata": {
"kernelspec": {
"display_name": "Python 3 (ipykernel)",
"language": "python",
"name": "python3"
},
"language_info": {
"codemirror_mode": {
"name": "ipython",
"version": 3
},
"file_extension": ".py",
"mimetype": "text/x-python",
"name": "python",
"nbconvert_exporter": "python",
"pygments_lexer": "ipython3",
"version": "3.10.4"
}
},
"nbformat": 4,
"nbformat_minor": 4
}