{ "cells": [ { "cell_type": "markdown", "metadata": {}, "source": [ "이 노트북의 코드에 대한 설명은 [반복 교차 검증](https://tensorflow.blog/2017/12/27/%EB%B0%98%EB%B3%B5-%EA%B5%90%EC%B0%A8-%EA%B2%80%EC%A6%9D/) 글을 참고하세요." ] }, { "cell_type": "code", "execution_count": 1, "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "CPython 3.5.6\n", "IPython 6.5.0\n", "\n", "sklearn 0.20.1\n", "numpy 1.15.2\n", "scipy 1.1.0\n" ] } ], "source": [ "%load_ext watermark\n", "%watermark -v -p sklearn,numpy,scipy" ] }, { "cell_type": "code", "execution_count": 2, "metadata": {}, "outputs": [], "source": [ "import matplotlib.pyplot as plt\n", "%matplotlib inline" ] }, { "cell_type": "code", "execution_count": 3, "metadata": {}, "outputs": [], "source": [ "from sklearn.datasets import load_iris\n", "from sklearn.model_selection import cross_val_score, KFold, StratifiedKFold\n", "from sklearn.linear_model import LogisticRegression" ] }, { "cell_type": "code", "execution_count": 4, "metadata": {}, "outputs": [], "source": [ "iris = load_iris()\n", "logreg = LogisticRegression(solver='liblinear', multi_class='auto', max_iter=1000)" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "RepeatedKFold" ] }, { "cell_type": "code", "execution_count": 5, "metadata": {}, "outputs": [ { "data": { "text/plain": [ "(array([1. , 0.93333333, 0.43333333, 0.96666667, 0.43333333]),\n", " 0.7533333333333333)" ] }, "execution_count": 5, "metadata": {}, "output_type": "execute_result" } ], "source": [ "kfold = KFold(n_splits=5)\n", "scores = cross_val_score(logreg, iris.data, iris.target, cv=kfold)\n", "scores, scores.mean()" ] }, { "cell_type": "code", "execution_count": 6, "metadata": {}, "outputs": [ { "data": { "text/plain": [ "(array([1. , 0.93333333, 0.93333333, 0.96666667, 0.96666667,\n", " 0.96666667, 0.93333333, 1. , 1. , 0.83333333,\n", " 0.93333333, 0.9 , 0.96666667, 0.9 , 0.93333333,\n", " 0.96666667, 1. , 0.93333333, 0.93333333, 0.93333333,\n", " 0.96666667, 0.9 , 1. , 0.93333333, 0.93333333]),\n", " 0.9466666666666668)" ] }, "execution_count": 6, "metadata": {}, "output_type": "execute_result" } ], "source": [ "from sklearn.model_selection import RepeatedKFold\n", "rkfold = RepeatedKFold(n_splits=5, n_repeats=5, random_state=42)\n", "scores = cross_val_score(logreg, iris.data, iris.target, cv=rkfold)\n", "scores, scores.mean()" ] }, { "cell_type": "code", "execution_count": 7, "metadata": {}, "outputs": [ { "data": { "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYQAAAD8CAYAAAB3u9PLAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDMuMC4wLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvqOYd8AAAEwJJREFUeJzt3X+MndV95/H3JwbjSosTwNMI2YDZrXdr14lgc+tktWkgiUINaoGE/sBLAqmstRoJ/9EmVUBOBHJrZStllSoRS+VsCHWUmPV2FWGpibwpGlp5RVa+FsZgkMngasNgpEzWXsQqTcDku3/cY/Yynngez4wZD32/pEfzPOec59xzJHw/POd57r2pKiRJett8D0CSdG4wECRJgIEgSWoMBEkSYCBIkhoDQZIEGAiSpMZAkCQBBoIkqTlvvgdwJpYtW1YrV66c72FI0oKyf//+H1fVyHTtFlQgrFy5kn6/P9/DkKQFJcn/6tLOJSNJEmAgSJIaA0GSBBgIkqTGQJAkAR0DIckDSX6U5KlfUJ8kX04yluRgkn89VHdHkh+07Y6h8vckebKd8+Ukmf10JEkz1fUK4UFg/WnqrwdWtW0TcD9AkouBe4D3AuuAe5Jc1M65v7U9ed7p+pcknWWdAqGq/h44dpomNwE7auD7wDuSXAr8JvC9qjpWVceB7wHrW93SqnqsBr/huQO4eVYzkSTNylx9MG058PzQ8XgrO135+BTlp0iyicGVBJdffvkcDVc6vTdrBdPfNNe5ZK5uKk/1r6dmUH5qYdX2qupVVW9kZNpPXktzoqrOaJvJOYaBzjVzFQjjwGVDxyuAo9OUr5iiXJI0T+YqEHYDt7enjd4HvFRVLwJ7gOuSXNRuJl8H7Gl1Lyd5X3u66Hbg4TkaiyRpBjrdQ0iyE7gWWJZknMGTQ+cDVNVfAt8BbgDGgJ8Af9DqjiX5U2Bf62prVZ28Of0pBk8v/RLw3bZJkuZJFtI6Zq/XK7/tVOeiJN4T0Dkryf6q6k3Xzk8qS5IAA0GS1BgIkiTAQJAkNQaCJAkwECRJjYEgSQIMBElSYyBIkgADQZLUGAiSJMBAkCQ1BoIkCTAQJEmNgSBJAgwESVJjIEiSAANBktQYCJIkwECQJDWdAiHJ+iSHk4wluWuK+iuSPJLkYJJHk6xo5R9McmBo+2mSm1vdg0n+YajuqrmdmiTpTJw3XYMki4D7gI8A48C+JLur6umhZl8EdlTVXyX5EPAF4BNVNQpc1fq5GBgD/vvQeX9SVX89N1ORJM1GlyuEdcBYVR2pqleAh4CbJrVZAzzS9kenqAf4HeC7VfWTmQ5WknT2dAmE5cDzQ8fjrWzYE8Atbf+jwIVJLpnU5lZg56SybW2Z6UtJLug4ZknSWdAlEDJFWU06/gxwTZLHgWuAF4ATr3eQXAq8C9gzdM7dwK8Cvw5cDHx2yhdPNiXpJ+lPTEx0GK4kaSa6BMI4cNnQ8Qrg6HCDqjpaVR+rqquBLa3spaEmvwd8u6peHTrnxRr4GfB1BktTp6iq7VXVq6reyMhIp0lJks5cl0DYB6xKcmWSxQyWfnYPN0iyLMnJvu4GHpjUxwYmLRe1qwaSBLgZeOrMhy9JmivTBkJVnQDuZLDc8wywq6oOJdma5MbW7FrgcJJngXcC206en2QlgyuMv5vU9TeTPAk8CSwD/mxWM5EkzUqqJt8OOHf1er3q9/vzPQzpFElYSP+W9E9Lkv1V1ZuunZ9UliQBBoIkqTEQJEmAgSBJagwESRLQ4cvtpIXu4osv5vjx42f9dQYfqTl7LrroIo4dO3ZWX0P/tBkIess7fvz4W+KR0LMdOJJLRpIkwECQJDUGgiQJMBAkSY2BIEkCDARJUmMgSJIAA0GS1BgIkiTAQJAkNQaCJAkwECRJjYEgSQIMBElS0ykQkqxPcjjJWJK7pqi/IskjSQ4meTTJiqG615IcaNvuofIrk/zPJD9I8l+SLJ6bKUmSZmLaQEiyCLgPuB5YA2xIsmZSsy8CO6rq3cBW4AtDdf9YVVe17cah8j8HvlRVq4DjwMZZzEOSNEtdrhDWAWNVdaSqXgEeAm6a1GYN8EjbH52i/g0y+KWPDwF/3Yr+Cri566AlSXOvSyAsB54fOh5vZcOeAG5p+x8FLkxySTtekqSf5PtJTr7pXwL8n6o6cZo+JUlvoi6BMNXv9k3+PcLPANckeRy4BngBOPlmf3lV9YB/B/xFkn/Rsc/BiyebWqD0JyYmOgxXkjQTXQJhHLhs6HgFcHS4QVUdraqPVdXVwJZW9tLJuvb3CPAocDXwY+AdSc77RX0O9b29qnpV1RsZGek6L0nSGeoSCPuAVe2poMXArcDu4QZJliU52dfdwAOt/KIkF5xsA/xb4Oka/OL5KPA77Zw7gIdnOxlJ0sxNGwhtnf9OYA/wDLCrqg4l2Zrk5FND1wKHkzwLvBPY1spXA/0kTzAIgP9QVU+3us8Cf5xkjME9ha/N0ZwkSTOQwf+sLwy9Xq/6/f58D0MLTBIW0n/nv8hbZR568yXZ3+7lnpafVJYkAQaCJKkxECRJgIEgSWoMBEkSYCBIkhoDQZIEGAiSpMZAkCQBBoIkqTEQJEmAgSBJagwESRJgIEiSmvOmbyItbHXPUrj37fM9jFmre5bO9xD0Fmcg6K3v3pfO+kv4WwV6K3DJSJIEGAiSpMZAkCQBBoIkqTEQJElAx0BIsj7J4SRjSe6aov6KJI8kOZjk0SQrWvlVSR5LcqjV/f7QOQ8m+YckB9p21dxNS5J0pqYNhCSLgPuA64E1wIYkayY1+yKwo6reDWwFvtDKfwLcXlW/BqwH/iLJO4bO+5OquqptB2Y5F0nSLHS5QlgHjFXVkap6BXgIuGlSmzXAI21/9GR9VT1bVT9o+0eBHwEjczFwSdLc6hIIy4Hnh47HW9mwJ4Bb2v5HgQuTXDLcIMk6YDHw3FDxtraU9KUkF0z14kk2Jekn6U9MTHQYriRpJroEQqYom/yRzM8A1yR5HLgGeAE48XoHyaXAN4A/qKqft+K7gV8Ffh24GPjsVC9eVdurqldVvZERLy4k6Wzp8tUV48BlQ8crgKPDDdpy0McAkvwz4JaqeqkdLwX+BvhcVX1/6JwX2+7PknydQahIkuZJlyuEfcCqJFcmWQzcCuwebpBkWZKTfd0NPNDKFwPfZnDD+b9OOufS9jfAzcBTs5mIJGl2pg2EqjoB3AnsAZ4BdlXVoSRbk9zYml0LHE7yLPBOYFsr/z3gA8Anp3i89JtJngSeBJYBfzZXk5IknbkspG9o7PV61e/353sY0in8tlOdy5Lsr6redO38pLIkCTAQJEmNgSBJAgwESVJjIEiSAANBktQYCJIkwECQJDUGgiQJMBAkSY2BIEkCDARJUmMgSJIAA0GS1BgIkiTAQJAkNQaCJAkwECRJjYEgSQIMBElS0ykQkqxPcjjJWJK7pqi/IskjSQ4meTTJiqG6O5L8oG13DJW/J8mTrc8vJ8ncTEmSNBPTBkKSRcB9wPXAGmBDkjWTmn0R2FFV7wa2Al9o514M3AO8F1gH3JPkonbO/cAmYFXb1s96NpKkGetyhbAOGKuqI1X1CvAQcNOkNmuAR9r+6FD9bwLfq6pjVXUc+B6wPsmlwNKqeqyqCtgB3DzLuUiSZqFLICwHnh86Hm9lw54Abmn7HwUuTHLJac5d3vZP16ck6U3UJRCmWtuvScefAa5J8jhwDfACcOI053bpc/DiyaYk/ST9iYmJDsOVZi/JGW0zOcfbZjrXdAmEceCyoeMVwNHhBlV1tKo+VlVXA1ta2UunOXe87f/CPof63l5VvarqjYyMdBiuNHtV9aZs0rmkSyDsA1YluTLJYuBWYPdwgyTLkpzs627ggba/B7guyUXtZvJ1wJ6qehF4Ocn72tNFtwMPz8F8JEkzNG0gVNUJ4E4Gb+7PALuq6lCSrUlubM2uBQ4neRZ4J7CtnXsM+FMGobIP2NrKAD4F/GdgDHgO+O5cTUqSdOaykC5be71e9fv9+R6GJC0oSfZXVW+6dn5SWZIEGAiSpMZAkCQBBoIkqTEQJEmAgSBJagwESRJgIEiSGgNBkgQYCJKkxkCQJAEGgiSpMRAkSYCBIElqDARJEmAgSJIaA0GSBBgIkqTGQJAkAQaCJKkxECRJQMdASLI+yeEkY0numqL+8iSjSR5PcjDJDa38tiQHhrafJ7mq1T3a+jxZ98tzOzVJ0pk4b7oGSRYB9wEfAcaBfUl2V9XTQ80+B+yqqvuTrAG+A6ysqm8C32z9vAt4uKoODJ13W1X152gukqRZ6HKFsA4Yq6ojVfUK8BBw06Q2BSxt+28Hjk7RzwZg50wHKkk6u7oEwnLg+aHj8VY27F7g40nGGVwdbJ6in9/n1ED4elsu+nySTPXiSTYl6SfpT0xMdBiuJGkmugTCVG/UNel4A/BgVa0AbgC+keT1vpO8F/hJVT01dM5tVfUu4Dfa9ompXryqtldVr6p6IyMjHYYrSZqJLoEwDlw2dLyCU5eENgK7AKrqMWAJsGyo/lYmXR1U1Qvt78vAtxgsTUmS5kmXQNgHrEpyZZLFDN7cd09q80PgwwBJVjMIhIl2/Dbgdxnce6CVnZdkWds/H/gt4CkkSfNm2qeMqupEkjuBPcAi4IGqOpRkK9Cvqt3Ap4GvJvkjBstJn6yqk8tKHwDGq+rIULcXAHtaGCwC/hb46pzNSpJ0xvL/37fPfb1er/p9n1KVpDORZH9V9aZr5yeVJUmAgSBJagwESRJgIEiSGgNBkgQYCJKkxkCQJAEGgiSpMRAkSYCBIElqDARJEmAgSJIaA0GSBBgIkqTGQJAkAQaCJKkxECRJgIEgSWoMBEkSYCBIkppOgZBkfZLDScaS3DVF/eVJRpM8nuRgkhta+cok/5jkQNv+cuic9yR5svX55SSZu2lJks7UtIGQZBFwH3A9sAbYkGTNpGafA3ZV1dXArcB/Gqp7rqquatsfDpXfD2wCVrVt/cynIUmarS5XCOuAsao6UlWvAA8BN01qU8DStv924OjpOkxyKbC0qh6rqgJ2ADef0cglSXOqSyAsB54fOh5vZcPuBT6eZBz4DrB5qO7KtpT0d0l+Y6jP8Wn6lCS9iboEwlRr+zXpeAPwYFWtAG4AvpHkbcCLwOVtKemPgW8lWdqxz8GLJ5uS9JP0JyYmOgxXkjQTXQJhHLhs6HgFpy4JbQR2AVTVY8ASYFlV/ayq/ncr3w88B/zL1ueKafqknbe9qnpV1RsZGekwXEnSTHQJhH3AqiRXJlnM4Kbx7kltfgh8GCDJagaBMJFkpN2UJsk/Z3Dz+EhVvQi8nOR97emi24GH52RGkqQZOW+6BlV1IsmdwB5gEfBAVR1KshXoV9Vu4NPAV5P8EYOln09WVSX5ALA1yQngNeAPq+pY6/pTwIPALwHfbZskaZ5k8JDPwtDr9arf78/3MCRpQUmyv6p607Xzk8qSJMBAkCQ1BoIkCTAQJEmNgSBJAgwESVJjIEiSAANBktQYCJIkwECQJDUGgiQJMBAkSY2BIM3Czp07Wbt2LYsWLWLt2rXs3Llzvockzdi0X38taWo7d+5ky5YtfO1rX+P9738/e/fuZePGjQBs2LBhnkcnnTm//lqaobVr1/KVr3yFD37wg6+XjY6OsnnzZp566ql5HJn0Rl2//tpAkGZo0aJF/PSnP+X8889/vezVV19lyZIlvPbaa/M4MumN/D0E6SxbvXo1e/fufUPZ3r17Wb169TyNSJodA0GaoS1btrBx40ZGR0d59dVXGR0dZePGjWzZsmW+hybNiDeVpRk6eeN48+bNPPPMM6xevZpt27Z5Q1kLlvcQJOktznsIkqQz0ikQkqxPcjjJWJK7pqi/PMlokseTHExyQyv/SJL9SZ5sfz80dM6jrc8DbfvluZuWJOlMTXsPIcki4D7gI8A4sC/J7qp6eqjZ54BdVXV/kjXAd4CVwI+B366qo0nWAnuA5UPn3VZVrgFJ0jmgyxXCOmCsqo5U1SvAQ8BNk9oUsLTtvx04ClBVj1fV0VZ+CFiS5ILZD1uSNNe6BMJy4Pmh43He+H/5APcCH08yzuDqYPMU/dwCPF5VPxsq+3pbLvp8knQftiRprnV57HSqN+rJjyZtAB6sqv+Y5N8A30iytqp+DpDk14A/B64bOue2qnohyYXAfwM+Aew45cWTTcCmdvh/kxzuMGbpzbaMwRKpdC66okujLoEwDlw2dLyCtiQ0ZCOwHqCqHkuyhME/kB8lWQF8G7i9qp47eUJVvdD+vpzkWwyWpk4JhKraDmzvMhlpviTpd3msTzqXdVky2gesSnJlksXArcDuSW1+CHwYIMlqYAkwkeQdwN8Ad1fV/zjZOMl5SZa1/fOB3wL8NjBJmkfTBkJVnQDuZPCE0DMMniY6lGRrkhtbs08D/z7JE8BO4JM1+MTbncCvAJ+f9HjpBcCeJAeBA8ALwFfnenKSpO4W1CeVpXNVkk1teVNasAwESRLgV1dIkhoDQZqFJA8k+VESH4rQgmcgSLPzIO2Ra2mhMxCkWaiqvweOzfc4pLlgIEiSAANBktQYCJIkwECQJDUGgjQLSXYCjwH/Ksl4ko3zPSZppvyksiQJ8ApBktQYCJIkwECQJDUGgiQJMBAkSY2BIEkCDARJUmMgSJIA+H9XBPSQSTxVpQAAAABJRU5ErkJggg==\n", "text/plain": [ "
" ] }, "metadata": { "needs_background": "light" }, "output_type": "display_data" } ], "source": [ "plt.boxplot(scores)\n", "plt.show()" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "RepeatedStratifiedKFold" ] }, { "cell_type": "code", "execution_count": 8, "metadata": {}, "outputs": [ { "data": { "text/plain": [ "(array([1. , 0.96666667, 0.93333333, 0.9 , 1. ]),\n", " 0.9600000000000002)" ] }, "execution_count": 8, "metadata": {}, "output_type": "execute_result" } ], "source": [ "skfold = StratifiedKFold(n_splits=5)\n", "scores = cross_val_score(logreg, iris.data, iris.target, cv=skfold)\n", "scores, scores.mean()" ] }, { "cell_type": "code", "execution_count": 9, "metadata": {}, "outputs": [ { "data": { "text/plain": [ "(array([0.96666667, 0.96666667, 0.96666667, 0.93333333, 0.96666667,\n", " 0.86666667, 0.96666667, 0.96666667, 0.93333333, 0.96666667,\n", " 1. , 1. , 0.93333333, 0.93333333, 0.93333333,\n", " 1. , 0.96666667, 0.96666667, 0.9 , 0.96666667,\n", " 0.96666667, 0.96666667, 1. , 0.9 , 0.96666667]),\n", " 0.9559999999999998)" ] }, "execution_count": 9, "metadata": {}, "output_type": "execute_result" } ], "source": [ "from sklearn.model_selection import RepeatedStratifiedKFold\n", "rskfold = RepeatedStratifiedKFold(n_splits=5, n_repeats=5, random_state=42)\n", "scores = cross_val_score(logreg, iris.data, iris.target, cv=rskfold)\n", "scores, scores.mean()" ] }, { "cell_type": "code", "execution_count": 10, "metadata": {}, "outputs": [ { "data": { "image/png": "iVBORw0KGgoAAAANSUhEUgAAAX0AAAD8CAYAAACb4nSYAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDMuMC4wLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvqOYd8AAAEU9JREFUeJzt3X+sX/V93/HnKzYuXQMpxVeow4CpRivfuqhJv3HTLa1JqlGTP0CAuuGua6iselIHf0xhE8iRYI4Q0kqnqRnr5CpeRiYZsaqdPJXNRZZp5YlI/rqeHRzXxEVafHE0bkZGxaIITN/74x4n31zf+HvuD/O1/Xk+pK90zufzOd/zPtK9r++5n3PO96aqkCS14QOTLkCS9P4x9CWpIYa+JDXE0Jekhhj6ktQQQ1+SGmLoS1JDDH1JaoihL0kNWT3pAuZbu3ZtrV+/ftJlSNJl5fDhw9+sqqlx4y650F+/fj3D4XDSZUjSZSXJ/+ozzukdSWqIoS9JDTH0Jakhhr4kNcTQl6SGjA39JLuTvJHklR/QnyS/l+RUkmNJPjLS9+kkX+ten17JwiVJi9fnTP+LwJYL9N8F3Na9tgO/D5Dkx4DHgZ8HNgGPJ7luOcVKkpZnbOhX1Z8Db15gyD3AszXny8CPJvlx4FeAF6vqzar6FvAiF/7wkCRdZCvxcNaNwOmR9Zmu7Qe1nyfJdub+SuDmm29egZKk8ZK8L/vx/1DrUrISF3IX+s2pC7Sf31i1q6oGVTWYmhr7FLG0Iqpq0a+lbCddSlYi9GeAm0bW1wFnLtAuSZqQlQj9vcBvdHfxfAx4q6q+AewD7kxyXXcB986uTZI0IWPn9JPsAe4A1iaZYe6OnKsAqurfAy8AnwJOAd8GfrPrezPJ54BD3VvtrKoLXRCWJF1kY0O/qraO6S/gn/6Avt3A7qWVJklaaT6RK0kNMfQlqSGGviQ1xNCXpIYY+pLUEENfkhpi6EtSQwx9SWqIoS9JDTH0Jakhhr4kNcTQl6SGGPqS1BBDX5IaYuhLUkMMfUlqiKEvSQ0x9CWpIYa+JDXE0JekhvQK/SRbkpxMcirJowv035Jkf5JjSV5Ksm6k718lOZ7kRJLfS5KVPABJUn9jQz/JKuAZ4C5gGtiaZHresKeBZ6vqdmAn8FS37d8F/h5wO7AR+CiwecWqlyQtSp8z/U3Aqap6rareAZ4D7pk3ZhrY3y0fGOkv4GpgDfBDwFXA/15u0ZKkpekT+jcCp0fWZ7q2UUeB+7vle4FrklxfVS8z9yHwje61r6pOLK9kSdJS9Qn9hebga976I8DmJEeYm755HTib5O8AG4B1zH1QfDLJL523g2R7kmGS4ezs7KIOQJLUX5/QnwFuGllfB5wZHVBVZ6rqvqr6MLCja3uLubP+L1fV21X1NvDfgI/N30FV7aqqQVUNpqamlngokqRx+oT+IeC2JLcmWQM8AOwdHZBkbZJz7/UYsLtb/jpzfwGsTnIVc38FOL0jSRMyNvSr6izwELCPucB+vqqOJ9mZ5O5u2B3AySSvAjcAT3btfwj8FfAV5ub9j1bVf13ZQ5Ak9ZWq+dPzkzUYDGo4HE66DGlBSbjUfmckgCSHq2owbpxP5EpSQwx9SWqIoS9JDTH0Jakhhr4kNcTQl6SGGPqS1BBDX5IaYuhLUkMMfUlqiKEvSQ0x9CWpIYa+JDXE0Jekhhj6ktSQ1ZMuQFoxT3zoou+iHr/2fdkPT7x18fehJhn6umLkX/71FfEPTpJQT0y6Cl2pnN6RpIYY+pLUEENfkhpi6EtSQ3qFfpItSU4mOZXk0QX6b0myP8mxJC8lWTfSd3OSP01yIslXk6xfufIlSYsxNvSTrAKeAe4CpoGtSabnDXsaeLaqbgd2Ak+N9D0L/E5VbQA2AW+sROGSpMXrc6a/CThVVa9V1TvAc8A988ZMA/u75QPn+rsPh9VV9SJAVb1dVd9ekcolSYvWJ/RvBE6PrM90baOOAvd3y/cC1yS5HvhJ4P8m+aMkR5L8TveXgyRpAvqEfhZom/8EzCPA5iRHgM3A68BZ5h7++sWu/6PATwAPnreDZHuSYZLh7Oxs/+olSYvSJ/RngJtG1tcBZ0YHVNWZqrqvqj4M7Oja3uq2PdJNDZ0F/gvwkfk7qKpdVTWoqsHU1NQSD0WSNE6f0D8E3Jbk1iRrgAeAvaMDkqxNcu69HgN2j2x7XZJzSf5J4KvLL1uStBRjQ787Q38I2AecAJ6vquNJdia5uxt2B3AyyavADcCT3bbvMTe1sz/JV5ibKvqDFT8KSVIvudS+oGowGNRwOJx0GboMJblyvnDtCjgOvb+SHK6qwbhxPpErSQ0x9CWpIYa+JDXE0Jekhhj6ktQQQ1+SGmLoS1JDDH1JaoihL0kNMfQlqSGGviQ1xNCXpIYY+pLUEENfkhpi6EtSQwx9SWqIoS9JDTH0Jakhhr4kNcTQl6SGGPqS1JBeoZ9kS5KTSU4leXSB/luS7E9yLMlLSdbN6782yetJ/u1KFS5JWryxoZ9kFfAMcBcwDWxNMj1v2NPAs1V1O7ATeGpe/+eAP1t+uZKk5ehzpr8JOFVVr1XVO8BzwD3zxkwD+7vlA6P9SX4OuAH40+WXK0lajj6hfyNwemR9pmsbdRS4v1u+F7gmyfVJPgD8LvDPL7SDJNuTDJMMZ2dn+1UuSVq0PqGfBdpq3vojwOYkR4DNwOvAWeC3gReq6jQXUFW7qmpQVYOpqakeJUmSlmJ1jzEzwE0j6+uAM6MDquoMcB9Akg8C91fVW0l+AfjFJL8NfBBYk+TtqjrvYrAk6eLrE/qHgNuS3MrcGfwDwK+NDkiyFnizqv4GeAzYDVBV/2hkzIPAwMCXpMkZO71TVWeBh4B9wAng+ao6nmRnkru7YXcAJ5O8ytxF2ycvUr2SpGVI1fzp+ckaDAY1HA4nXYYuQ0m41H6el+JKOQ69v5IcrqrBuHE+kStJDTH0Jakhhr4kNcTQl6SG9LllU7psJAs9S3h5ue666yZdgq5ghr6uGO/HHS/eWaPLndM7ktQQQ1+SGmLoS1JDDH1JaoihL0kNMfQlqSGGviQ1xNCXpIYY+pLUEENfkhpi6EtSQwx9SWqIoS9JDTH0JakhvUI/yZYkJ5OcSvLoAv23JNmf5FiSl5Ks69p/NsnLSY53ff9wpQ9AktTf2NBPsgp4BrgLmAa2JpmeN+xp4Nmquh3YCTzVtX8b+I2q+mlgC/BvkvzoShUvSVqcPmf6m4BTVfVaVb0DPAfcM2/MNLC/Wz5wrr+qXq2qr3XLZ4A3gKmVKFyStHh9Qv9G4PTI+kzXNuoocH+3fC9wTZLrRwck2QSsAf5qaaVKkparT+gv9E9H5/+/uEeAzUmOAJuB14Gz332D5MeBLwG/WVV/c94Oku1JhkmGs7OzvYuXJC1On9CfAW4aWV8HnBkdUFVnquq+qvowsKNrewsgybXAnwCfraovL7SDqtpVVYOqGkxNOfsjSRdLn9A/BNyW5NYka4AHgL2jA5KsTXLuvR4Ddnfta4A/Zu4i739eubIlSUsxNvSr6izwELAPOAE8X1XHk+xMcnc37A7gZJJXgRuAJ7v2fwD8EvBgkv/ZvX52pQ9CktRPquZPz0/WYDCo4XA46TKkBSXhUvudkQCSHK6qwbhxPpErSQ0x9CWpIYa+JDXE0Jekhhj6ktQQQ1+SGmLoS1JDDH1JaoihL0kNMfQlqSGGviQ1xNCXpIYY+pLUEENfkhpi6EtSQwx9SWqIoS9JDTH0Jakhhr4kNcTQl6SGGPqS1JBeoZ9kS5KTSU4leXSB/luS7E9yLMlLSdaN9H06yde616dXsnhJ0uKMDf0kq4BngLuAaWBrkul5w54Gnq2q24GdwFPdtj8GPA78PLAJeDzJdStXviRpMfqc6W8CTlXVa1X1DvAccM+8MdPA/m75wEj/rwAvVtWbVfUt4EVgy/LLliQtRZ/QvxE4PbI+07WNOgrc3y3fC1yT5Pqe25Jke5JhkuHs7Gzf2qVlSbLo11K2ky4lfUJ/oZ/amrf+CLA5yRFgM/A6cLbntlTVrqoaVNVgamqqR0nS8lXV+/KSLiWre4yZAW4aWV8HnBkdUFVngPsAknwQuL+q3koyA9wxb9uXllGvJGkZ+pzpHwJuS3JrkjXAA8De0QFJ1iY5916PAbu75X3AnUmu6y7g3tm1SZImYGzoV9VZ4CHmwvoE8HxVHU+yM8nd3bA7gJNJXgVuAJ7stn0T+BxzHxyHgJ1dmyRpAnKpzTkOBoMaDoeTLkOSLitJDlfVYNw4n8iVpIYY+pLUEENfkhpi6EtSQwx9SWqIoS9JDTH0Jakhhr4kNcTQl6SGGPqS1BBDX5IaYuhLUkMMfUlqiKEvSQ0x9CWpIYa+JDXE0Jekhhj6ktQQQ1+SGmLoS1JDeoV+ki1JTiY5leTRBfpvTnIgyZEkx5J8qmu/Ksl/TPKVJCeSPLbSByBJ6m9s6CdZBTwD3AVMA1uTTM8b9lng+ar6MPAA8O+69l8Ffqiqfgb4OeCfJFm/MqVLkharz5n+JuBUVb1WVe8AzwH3zBtTwLXd8oeAMyPtP5JkNfDDwDvAXy+7aknSkvQJ/RuB0yPrM13bqCeAX08yA7wAPNy1/yHw/4BvAF8Hnq6qN5dTsCRp6fqEfhZoq3nrW4EvVtU64FPAl5J8gLm/Et4D/jZwK/CZJD9x3g6S7UmGSYazs7OLOgBJUn99Qn8GuGlkfR3fm745ZxvwPEBVvQxcDawFfg3471X1blW9AfwPYDB/B1W1q6oGVTWYmppa/FFIknrpE/qHgNuS3JpkDXMXavfOG/N14JcBkmxgLvRnu/ZPZs6PAB8D/nKlipckLc7Y0K+qs8BDwD7gBHN36RxPsjPJ3d2wzwC/leQosAd4sKqKubt+Pgi8wtyHx3+oqmMX4TgkST1kLpsvHYPBoIbD4aTLkKTLSpLDVXXe9Pl8PpErSQ0x9CWpIYa+JDXE0Jekhhj6ktQQQ1+SGmLoS1JDDH1JaoihL0kNMfQlqSGGviQ1xNCXpIYY+lIPe/bsYePGjaxatYqNGzeyZ8+eSZckLcnqSRcgXer27NnDjh07+MIXvsDHP/5xDh48yLZt2wDYunXrhKuTFsevVpbG2LhxI5///Of5xCc+8d22AwcO8PDDD/PKK69MsDLpe/p+tbKhL42xatUqvvOd73DVVVd9t+3dd9/l6quv5r333ptgZdL3+H360grZsGEDBw8e/L62gwcPsmHDhglVJC2doS+NsWPHDrZt28aBAwd49913OXDgANu2bWPHjh2TLk1aNC/kSmOcu1j78MMPc+LECTZs2MCTTz7pRVxdlpzTl6QrgHP6kqTz9Ar9JFuSnExyKsmjC/TfnORAkiNJjiX51Ejf7UleTnI8yVeSXL2SByBJ6m/snH6SVcAzwN8HZoBDSfZW1VdHhn0WeL6qfj/JNPACsD7JauA/Af+4qo4muR54d8WPQpLUS58z/U3Aqap6rareAZ4D7pk3poBru+UPAWe65TuBY1V1FKCq/k9VeWOzJE1In9C/ETg9sj7TtY16Avj1JDPMneU/3LX/JFBJ9iX5iyT/Ypn1SpKWoc8tm1mgbf4tP1uBL1bV7yb5BeBLSTZ27/9x4KPAt4H93RXm/d+3g2Q7sL1bfTvJycUchPQ+Wgt8c9JFSAu4pc+gPqE/A9w0sr6O703fnLMN2AJQVS93F2vXdtv+WVV9EyDJC8BHgO8L/araBezqU7A0SUmGfW6Lky5VfaZ3DgG3Jbk1yRrgAWDvvDFfB34ZIMkG4GpgFtgH3J7kb3UXdTcDX0WSNBFjz/Sr6mySh5gL8FXA7qo6nmQnMKyqvcBngD9I8s+Ym/p5sOae+vpWkn/N3AdHAS9U1Z9crIORJF3YJfdErnQpS7K9m46ULkuGviQ1xK9hkKSGGPpSD0l2J3kjif8qS5c1Q1/q54t0tyVLlzNDX+qhqv4ceHPSdUjLZehLUkMMfUlqiKEvSQ0x9CWpIYa+1EOSPcDLwE8lmUmybdI1SUvhE7mS1BDP9CWpIYa+JDXE0Jekhhj6ktQQQ1+SGmLoS1JDDH1JaoihL0kN+f/7eB7g6VA2hAAAAABJRU5ErkJggg==\n", "text/plain": [ "
" ] }, "metadata": { "needs_background": "light" }, "output_type": "display_data" } ], "source": [ "plt.boxplot(scores)\n", "plt.show()" ] }, { "cell_type": "code", "execution_count": 11, "metadata": {}, "outputs": [ { "data": { "text/plain": [ "GridSearchCV(cv=,\n", " error_score='raise-deprecating',\n", " estimator=LogisticRegression(C=1.0, class_weight=None, dual=False, fit_intercept=True,\n", " intercept_scaling=1, max_iter=1000, multi_class='auto',\n", " n_jobs=None, penalty='l2', random_state=None, solver='liblinear',\n", " tol=0.0001, verbose=0, warm_start=False),\n", " fit_params=None, iid=False, n_jobs=None,\n", " param_grid={'C': [0.001, 0.01, 0.1, 1, 10, 100, 1000]},\n", " pre_dispatch='2*n_jobs', refit=True, return_train_score=True,\n", " scoring=None, verbose=0)" ] }, "execution_count": 11, "metadata": {}, "output_type": "execute_result" } ], "source": [ "from sklearn.model_selection import GridSearchCV, train_test_split\n", "X_train, X_test, y_train, y_test = train_test_split(iris.data, iris.target, random_state=42)\n", "param_grid = {'C': [0.001, 0.01, 0.1, 1, 10, 100, 1000]}\n", "grid_search = GridSearchCV(logreg, param_grid, cv=rskfold, return_train_score=True, iid=False)\n", "grid_search.fit(X_train, y_train)" ] }, { "cell_type": "code", "execution_count": 12, "metadata": {}, "outputs": [ { "data": { "text/plain": [ "(1.0, {'C': 100}, 0.9605947675512894)" ] }, "execution_count": 12, "metadata": {}, "output_type": "execute_result" } ], "source": [ "grid_search.score(X_test, y_test), grid_search.best_params_, grid_search.best_score_" ] }, { "cell_type": "code", "execution_count": 13, "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "split15_test_score [0.34782609 0.65217391 0.7826087 0.91304348 0.95652174 0.95652174\n", " 0.95652174]\n", "split5_train_score [0.33707865 0.65168539 0.80898876 0.96629213 0.97752809 0.97752809\n", " 0.97752809]\n", "split14_test_score [0.33333333 0.66666667 0.80952381 0.9047619 0.95238095 0.9047619\n", " 0.9047619 ]\n", "split16_test_score [0.34782609 0.65217391 0.91304348 0.95652174 0.95652174 1.\n", " 0.95652174]\n", "split11_train_score [0.33707865 0.65168539 0.84269663 0.94382022 0.94382022 0.96629213\n", " 0.96629213]\n", "split18_train_score [0.34444444 0.65555556 0.82222222 0.95555556 0.96666667 0.96666667\n", " 0.96666667]\n", "split2_train_score [0.33707865 0.65168539 0.7752809 0.96629213 0.98876404 0.98876404\n", " 1. ]\n", "split13_test_score [0.31818182 0.63636364 0.86363636 1. 0.95454545 0.90909091\n", " 0.90909091]\n", "split21_test_score [0.34782609 0.65217391 0.82608696 0.91304348 0.86956522 0.91304348\n", " 0.86956522]\n", "split23_train_score [0.34444444 0.65555556 0.81111111 0.96666667 0.98888889 0.98888889\n", " 0.98888889]\n", "split17_train_score [0.33707865 0.65168539 0.82022472 0.94382022 0.95505618 0.96629213\n", " 0.96629213]\n", "split16_train_score [0.33707865 0.65168539 0.80898876 0.95505618 0.96629213 0.96629213\n", " 0.96629213]\n", "split1_test_score [0.34782609 0.65217391 0.86956522 0.82608696 0.91304348 0.95652174\n", " 1. ]\n", "split18_test_score [0.31818182 0.63636364 0.72727273 1. 1. 1.\n", " 0.90909091]\n", "split4_test_score [0.33333333 0.66666667 0.85714286 0.95238095 0.95238095 0.95238095\n", " 0.95238095]\n", "split17_test_score [0.34782609 0.65217391 0.82608696 0.95652174 1. 1.\n", " 1. ]\n", "split13_train_score [0.34444444 0.65555556 0.78888889 0.95555556 0.96666667 0.96666667\n", " 0.98888889]\n", "split11_test_score [0.34782609 0.65217391 0.7826087 0.95652174 0.95652174 1.\n", " 1. ]\n", "split10_train_score [0.33707865 0.65168539 0.83146067 0.96629213 0.96629213 0.96629213\n", " 0.97752809]\n", "split7_test_score [0.34782609 0.65217391 0.7826087 0.91304348 0.91304348 0.91304348\n", " 0.91304348]\n", "split8_train_score [0.34444444 0.65555556 0.76666667 0.94444444 0.97777778 0.98888889\n", " 0.97777778]\n", "split9_test_score [0.33333333 0.66666667 0.85714286 0.95238095 0.95238095 1.\n", " 1. ]\n", "split15_train_score [0.33707865 0.65168539 0.80898876 0.97752809 0.97752809 0.97752809\n", " 0.97752809]\n", "split7_train_score [0.33707865 0.65168539 0.84269663 0.95505618 0.96629213 0.96629213\n", " 0.96629213]\n", "split24_train_score [0.34065934 0.64835165 0.83516484 0.93406593 0.96703297 0.96703297\n", " 0.96703297]\n", "split20_test_score [0.34782609 0.65217391 0.82608696 0.95652174 1. 1.\n", " 1. ]\n", "split22_train_score [0.33707865 0.65168539 0.80898876 0.95505618 0.96629213 0.96629213\n", " 0.96629213]\n", "split19_test_score [0.33333333 0.66666667 0.80952381 0.95238095 0.9047619 0.9047619\n", " 0.9047619 ]\n", "split0_test_score [0.34782609 0.65217391 0.73913043 1. 0.95652174 0.95652174\n", " 0.95652174]\n", "split8_test_score [0.31818182 0.63636364 0.86363636 0.95454545 0.95454545 0.95454545\n", " 0.95454545]\n", "split4_train_score [0.34065934 0.64835165 0.79120879 0.96703297 0.96703297 0.97802198\n", " 0.97802198]\n", "split23_test_score [0.31818182 0.63636364 0.77272727 0.95454545 0.95454545 0.95454545\n", " 0.95454545]\n", "split10_test_score [0.34782609 0.65217391 0.82608696 0.91304348 0.95652174 1.\n", " 1. ]\n", "split9_train_score [0.34065934 0.64835165 0.83516484 0.93406593 0.95604396 0.95604396\n", " 0.96703297]\n", "split14_train_score [0.34065934 0.64835165 0.8021978 0.96703297 0.95604396 0.97802198\n", " 0.96703297]\n", "split6_train_score [0.33707865 0.65168539 0.83146067 0.95505618 0.95505618 0.97752809\n", " 0.96629213]\n", "split3_train_score [0.34444444 0.65555556 0.82222222 0.95555556 0.96666667 0.96666667\n", " 0.96666667]\n", "split19_train_score [0.34065934 0.64835165 0.81318681 0.95604396 0.96703297 0.98901099\n", " 0.97802198]\n", "split6_test_score [0.34782609 0.65217391 0.7826087 0.91304348 0.95652174 0.95652174\n", " 0.95652174]\n", "split24_test_score [0.33333333 0.66666667 0.76190476 1. 1. 1.\n", " 1. ]\n", "split2_test_score [0.34782609 0.65217391 0.86956522 0.91304348 0.91304348 0.91304348\n", " 0.91304348]\n", "split12_train_score [0.33707865 0.65168539 0.80898876 0.95505618 0.98876404 0.98876404\n", " 0.98876404]\n", "split20_train_score [0.33707865 0.65168539 0.79775281 0.96629213 0.96629213 0.96629213\n", " 0.96629213]\n", "split0_train_score [0.33707865 0.65168539 0.85393258 0.93258427 0.95505618 0.97752809\n", " 0.96629213]\n", "split5_test_score [0.34782609 0.65217391 0.7826087 0.95652174 0.95652174 0.95652174\n", " 0.95652174]\n", "split3_test_score [0.31818182 0.63636364 0.72727273 1. 1. 1.\n", " 1. ]\n", "split12_test_score [0.34782609 0.65217391 0.7826087 0.95652174 0.95652174 0.95652174\n", " 0.95652174]\n", "split1_train_score [0.33707865 0.65168539 0.82022472 0.97752809 0.96629213 0.96629213\n", " 0.96629213]\n", "split22_test_score [0.34782609 0.65217391 0.91304348 0.95652174 0.95652174 0.95652174\n", " 1. ]\n", "split21_train_score [0.33707865 0.65168539 0.82022472 0.95505618 0.96629213 0.96629213\n", " 0.97752809]\n" ] } ], "source": [ "for k in grid_search.cv_results_:\n", " if 'split' in k:\n", " print(k, grid_search.cv_results_[k])" ] } ], "metadata": { "kernelspec": { "display_name": "Python 3", "language": "python", "name": "python3" }, "language_info": { "codemirror_mode": { "name": "ipython", "version": 3 }, "file_extension": ".py", "mimetype": "text/x-python", "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", "version": "3.5.6" } }, "nbformat": 4, "nbformat_minor": 2 }