{ "metadata": { "name": "" }, "nbformat": 3, "nbformat_minor": 0, "worksheets": [ { "cells": [ { "cell_type": "code", "collapsed": false, "input": [ "from sklearn.datasets import fetch_lfw_people\n", "from sklearn.cross_validation import train_test_split\n", "from sklearn.grid_search import GridSearchCV\n", "from sklearn import metrics\n", "from sklearn.decomposition import RandomizedPCA\n", "from sklearn.svm import SVC\n", "from sklearn.ensemble import ExtraTreesClassifier\n", "from sklearn.pipeline import Pipeline\n", "from sklearn.utils import shuffle\n", "\n", "import matplotlib.pyplot as plt\n", "%pylab inline --no-import-all" ], "language": "python", "metadata": {}, "outputs": [ { "output_type": "stream", "stream": "stdout", "text": [ "Populating the interactive namespace from numpy and matplotlib\n" ] } ], "prompt_number": 24 }, { "cell_type": "code", "collapsed": false, "input": [ "lfw_people = fetch_lfw_people(min_faces_per_person=70, resize = 0.4)" ], "language": "python", "metadata": {}, "outputs": [], "prompt_number": 2 }, { "cell_type": "markdown", "metadata": {}, "source": [ "## Image Data Classification by Different Models\n", "\n", "- WHITENING (e.g. PCA_WHITENING) IS CRUCIAL for models such as SVM\n", "- WHITENING may NOT be curcial for models such as decision trees\n", "- When doing PCA, DONT PUT it in a pipeline as it will refit a PCA for each cross-fold, try to FIT AS MUCH DATA AS POSSIBLE FOR UNSUPERVISED LEARNING " ] }, { "cell_type": "code", "collapsed": false, "input": [ "X = lfw_people.data\n", "y = lfw_people.target\n", "X, y = shuffle(X, y)\n", "train_X, test_X, train_y, test_y = train_test_split(X, y,\n", " test_size = 0.2)\n", "print train_X.shape, test_X.shape\n", "print np.unique(y)" ], "language": "python", "metadata": {}, "outputs": [ { "output_type": "stream", "stream": "stdout", "text": [ "(1030, 1850) (258, 1850)\n", "[0 1 2 3 4 5 6]\n" ] } ], "prompt_number": 50 }, { "cell_type": "code", "collapsed": false, "input": [ "## PCA WHITEN OR NOT (ON AS MUCH DATA AS POSSIBLE) \n", "pca_whiten = RandomizedPCA(n_components=150, whiten=True)\n", "pca = RandomizedPCA(n_components=150, whiten = False)\n", "\n", "white_train_X = pca_whiten.fit_transform(train_X)\n", "white_test_X = pca_whiten.transform(test_X)\n", "\n", "pca_train_X = pca.fit_transform(train_X)\n", "pca_test_X = pca.transform(test_X)\n", "\n", "datasets = {\n", " 'raw': [(train_X, train_y), (test_X, test_y)],\n", " 'pca': [(pca_train_X, train_y), (pca_test_X, test_y)],\n", " 'white': [(white_train_X, train_y), (white_test_X, test_y)]\n", "}" ], "language": "python", "metadata": {}, "outputs": [], "prompt_number": 35 }, { "cell_type": "code", "collapsed": false, "input": [ "trees = ExtraTreesClassifier()\n", "svc = SVC(kernel = 'rbf')\n", "\n", "trees_params = {\n", " 'n_estimators': [100, 200, 1000],\n", " 'max_features': [0.05, 0.2, 0.5, 1.0]\n", "}\n", "svc_params = {\n", " 'C': [5e2, 1e3, 5e3, 1e4, 5e4],\n", " 'gamma': [1e-4, 5e-4, 1e-3, 5e-3, 1e-2, 1e-1]\n", " \n", "}\n", "\n", "models = {\n", " 'trees': (trees, trees_params),\n", " 'svc': (svc, svc_params)\n", "}" ], "language": "python", "metadata": {}, "outputs": [], "prompt_number": 39 }, { "cell_type": "code", "collapsed": false, "input": [ "from itertools import product\n", "from sklearn.grid_search import GridSearchCV\n", "\n", "def benchmark(candidate):\n", " from sklearn.grid_search import GridSearchCV\n", " from sklearn.svm import SVC\n", " from sklearn.ensemble import ExtraTreesClassifier\n", " (data_name, data), (model_name, model_infor) = candidate\n", " train_data, test_data = data\n", " model, param_grid = model_infor\n", " gs = GridSearchCV(model, param_grid, cv = 3, n_jobs=-1)\n", " gs.fit(*train_data)\n", " cv_score = gs.best_score_\n", " test_score = gs.best_estimator_.score(*test_data)\n", " return (model_name, data_name, cv_score, test_score, gs.best_params_)" ], "language": "python", "metadata": {}, "outputs": [], "prompt_number": 40 }, { "cell_type": "code", "collapsed": false, "input": [ "candidates = product(datasets.items(), models.items())\n", "results = []\n", "for candidate in candidates:\n", " print '============================='\n", " results.append(benchmark(candidate))\n", " print results[-1]" ], "language": "python", "metadata": {}, "outputs": [ { "output_type": "stream", "stream": "stdout", "text": [ "=============================\n" ] }, { "output_type": "stream", "stream": "stdout", "text": [ "('svc', 'raw', 0.40194174757281553, 0.44961240310077522, {'C': 500.0, 'gamma': 0.0001})\n", "=============================\n" ] }, { "output_type": "stream", "stream": "stdout", "text": [ "('trees', 'raw', 0.66407766990291262, 0.68217054263565891, {'max_features': 0.2, 'n_estimators': 1000})\n", "=============================\n" ] }, { "output_type": "stream", "stream": "stdout", "text": [ "('svc', 'white', 0.80679611650485439, 0.85658914728682167, {'C': 500.0, 'gamma': 0.005})\n", "=============================\n" ] }, { "output_type": "stream", "stream": "stdout", "text": [ "('trees', 'white', 0.62038834951456312, 0.68992248062015504, {'max_features': 1.0, 'n_estimators': 1000})\n", "=============================\n" ] }, { "output_type": "stream", "stream": "stdout", "text": [ "('svc', 'pca', 0.40194174757281553, 0.44961240310077522, {'C': 500.0, 'gamma': 0.0001})\n", "=============================\n" ] }, { "output_type": "stream", "stream": "stdout", "text": [ "('trees', 'pca', 0.60485436893203881, 0.70930232558139539, {'max_features': 1.0, 'n_estimators': 1000})\n" ] } ], "prompt_number": 41 }, { "cell_type": "markdown", "metadata": {}, "source": [ "### Several Interesting Things to Observe:\n", "1. TEST **ACCURACIES** (shuffled) tend to be HIGHER than CROSS-VALIDATION ACCURACIES\n", "2. **SVC**'best performance is better than **trees** (is it because it is based on one-vs-one and trees is based on one-vs-other)?\n", "3. Tree models are more **sensitive** to number of features (the larger the better) than normalization\n", "4. SVC models are more **sensitive** to normalization and (number of features - the smaller the better)?\n", "5. **whitening** has HUGE effect on SVC, but doesnt affect SVC that much\n", "6. The question is how to improve RandomForest's accuracy? -- does it help by using MORE trees with FEWER features, so each tree will overfit ? - Using a big n_estimators and a small max_features for RAW data does NOT seem to work.\n", "\n", "***Put it simple, SVM + PCA (dimensionality reduction) + Whitening is very good for face recognition***" ] }, { "cell_type": "code", "collapsed": false, "input": [ "## HOW TO BOOST the performance of random trees - one way could be\n", "## to use more advanced features such as sparse filtering.\n", "## BUT why does a SVM with PCA white has such a higher accuracy than\n", "## random forest?\n", "from sklearn.cross_validation import cross_val_score\n", "super_trees = ExtraTreesClassifier(n_estimators=300, max_depth=None, \n", " max_features=0.5, n_jobs=-1, \n", " bootstrap=False)\n", "print cross_val_score(super_trees, train_X, train_y, cv = 3)" ], "language": "python", "metadata": {}, "outputs": [ { "output_type": "stream", "stream": "stdout", "text": [ "[ 0.67151163 0.66472303 0.67638484]\n" ] } ], "prompt_number": 44 }, { "cell_type": "code", "collapsed": false, "input": [ "## Is it because the one-vs-one (SVM) and one-vs-rest (RBF) difference?\n", "## IT SEEMS NOT!!\n", "from sklearn.multiclass import OneVsOneClassifier\n", "super_trees = OneVsOneClassifier(ExtraTreesClassifier(n_estimators=1000, max_depth=None, \n", " max_features=0.5, n_jobs=1, \n", " bootstrap=False),\n", " n_jobs = -1)\n", "print cross_val_score(super_trees, train_X, train_y, cv = 3)" ], "language": "python", "metadata": {}, "outputs": [ { "output_type": "stream", "stream": "stdout", "text": [ "[ 0.66569767 0.67346939 0.70553936]\n" ] } ], "prompt_number": 49 }, { "cell_type": "code", "collapsed": false, "input": [ "## MAKE IT BETTER BY USING TREES ON FEATURE SELECTION and USE SVM\n", "trees_selector = ExtraTreesClassifier(n_estimators=1000, n_jobs=-1, max_features=1.0)\n", "trees_selector.fit(white_train_X, train_y)" ], "language": "python", "metadata": {}, "outputs": [ { "metadata": {}, "output_type": "pyout", "prompt_number": 65, "text": [ "ExtraTreesClassifier(bootstrap=False, compute_importances=None,\n", " criterion='gini', max_depth=None, max_features=1.0,\n", " min_density=None, min_samples_leaf=1, min_samples_split=2,\n", " n_estimators=1000, n_jobs=-1, oob_score=False,\n", " random_state=None, verbose=0)" ] } ], "prompt_number": 65 }, { "cell_type": "code", "collapsed": false, "input": [ "trees_selector.estimators_[0].feature_importances_\n", "feature_importances = np.mean(np.asarray([t.feature_importances_ for t in trees_selector.estimators_]), axis = 0)\n", "figure(figsize=(32, 8))\n", "plt.bar(np.arange(feature_importances.shape[0]), feature_importances)" ], "language": "python", "metadata": {}, "outputs": [ { "metadata": {}, "output_type": "pyout", "prompt_number": 66, "text": [ "" ] }, { "metadata": {}, "output_type": "display_data", "png": "iVBORw0KGgoAAAANSUhEUgAABzAAAAHcCAYAAAC5yDxJAAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\nAAALEgAACxIB0t1+/AAAIABJREFUeJzs3V+MVGf5B/DnNLtNDWFDMXY1O0RidglrWjdGKtFosklD\nSbjY1MYoekMEE0JELozW6hVoVPDPjZIauBDLDSHeuFxsNqHW1Rthm0g0ERIWLcmySGOkW3phBNf5\nXeAPSmFndpmdmefMfD7JJJ3ue4b3vOe8M+ec73nfU1Sr1WoAAAAAAAAAJPBIuysAAAAAAAAA8P8E\nmAAAAAAAAEAaAkwAAAAAAAAgDQEmAAAAAAAAkIYAEwAAAAAAAEhDgAkAAAAAAACkUTfAnJycjI0b\nN8bQ0FAcOnTogWX27dsXQ0NDMTIyEufOnbvz/3fu3Bn9/f3x1FNP3VP++vXrsWXLltiwYUM8++yz\nMT8/3+BqAAAAAAAAAJ2gZoC5sLAQe/fujcnJyTh//nycOHEiLly4cE+ZiYmJuHTpUszMzMTRo0dj\nz549d/72pS99KSYnJ+/73IMHD8aWLVvi4sWL8cwzz8TBgwdXaHUAAAAAAACAMqsZYE5PT8fg4GCs\nX78+ent7Y/v27TE+Pn5PmVOnTsWOHTsiImLz5s0xPz8f165di4iIT3/60/H444/f97nvXGbHjh3x\n61//ekVWBgAAAAAAACi3nlp/nJubi3Xr1t15X6lU4uzZs3XLzM3Nxfvf//5FP/eNN96I/v7+iIjo\n7++PN95444HliqKovwYAAAAAAABAKtVq9aGXrRlgLjVAfHcFlhM8FkVRs3wjKwc8nP3798f+/fvb\nXQ3oOvoetIe+B+2j/0F76HvQHvoetIe+B+3R6CDFmlPIDgwMxOzs7J33s7OzUalUapa5cuVKDAwM\n1PxH+/v770wz+/e//z2eeOKJZVccAAAAAAAA6Dw1A8xNmzbFzMxMXL58OW7evBknT56MsbGxe8qM\njY3F8ePHIyLizJkzsWbNmjvTwy5mbGwsXn755YiIePnll+O5555rZB0AAAAAAACADlEzwOzp6YnD\nhw/H1q1b48Mf/nB8/vOfj+Hh4Thy5EgcOXIkIiK2bdsWH/rQh2JwcDB2794dL7300p3lv/CFL8Qn\nP/nJuHjxYqxbty6OHTsWEREvvvhinD59OjZs2BCvvvpqvPjii01cRWC5RkdH210F6Er6HrSHvgft\no/9Be+h70B76HrSHvgflVFQTP2SyKArPwAQAAAAAAIASaTTjqzkCEwAAAAAAAKCVBJgAAAAAAABA\nGgJMAAAAAAAAIA0BJgAAAAAAAJCGABMAAAAAAABIQ4AJAAAAAAAApCHABAAAAAAAANIQYAIAAAAA\nAABpCDABAAAAAACANASYAAAAAAAAQBoCTAAAAAAAACANASYAAAAAAACQhgATAAAAAAAASEOACQAA\nAAAAAKQhwAQAAAAAAADSEGACAAAAAAAAaQgwAQAAAAAAgDQEmAAAAAAAAEAaAkwAAAAAAAAgDQEm\nAAAAAAAAkIYAEwAAAAAAAEhDgAkAAAAAAACkIcAEAAAAAAAA0hBgAgAAAAAAAGkIMAEAAAAAAIA0\nBJgAAAAAAABAGgJMAAAAAAAAIA0BJgAAAAAAAJCGABMAAAAAoAv09a2Noihqvvr61ra7mgAQRbVa\nrba7EospiiISVw8AAAAAoDSKooiIetdbXZMFoHGNZnxGYAIAAAAAAABpCDABAAAAAACANASYAAAA\nAAAAQBoCTAAAAAAAACANASYAAAAAAACQhgATAAAAAAAASEOACQAAAAAAAKQhwAQAAAAAAADSEGAC\nAAAAAAAAaQgwAQAAAAAAgDQEmAAAAAAAAEAaAkwAAAAAAAAgDQEmAAAAAAAAkIYAEwAAAAAAAEhD\ngAkAAAAAAACkIcAEAAAAAAAA0hBgAgAAAAAAAGkIMAEAAAAAAIA0BJgAAAAAAABAGgJMAAAAAAAA\nIA0BJgAAAAAAAJCGABMAAAAAAABIQ4AJAAAAAAAApCHABAAAAAAAANIQYAIAAAAAAABpCDABAAAA\nAACANASYAAAAAAAAQBoCTAAAAAAAACANASYAAAAAAACQhgATAAAAAOro61sbRVHUfPX1rW13NQEA\nOkJRrVar7a7EYoqiiMTVAwAAAKBLFEUREfWuU7mWRW72YwBapdGMzwhMAAAAAAAAIA0BJgAAAAAA\nAJCGABMAAAAAAABIQ4AJAAAAAAAApCHABAAAAAAAANIQYAIAAAAAAABpCDABAAAAAACANASYAAAA\nAAAAQBoCTAAAAAAAACANASYAAAAAAACQhgATAAAAAAAASEOACQAAAAAAAKQhwAQAAAAAAADSEGAC\nAAAAAAAAaQgwAQAAAAAAgDQEmDRFX9/aKIqi5quvb227qwkAAAAAAEAyRbVarba7EospiiISV48a\niqKIiHrbzvYFAAAAysG1DjqB/RiAVmk04zMCEwAAAAAAAEhDgAkAAAAAAACkUTfAnJycjI0bN8bQ\n0FAcOnTogWX27dsXQ0NDMTIyEufOnau77PT0dHz84x+Pj370o/H000/Ha6+9tgKrAtTiuaQAAAAA\nAEAZ1AwwFxYWYu/evTE5ORnnz5+PEydOxIULF+4pMzExEZcuXYqZmZk4evRo7Nmzp+6yL7zwQnz3\nu9+Nc+fOxXe+85144YUXmrR6wP97++034/YzDhZ/3S4DAAAAAADQPjUDzOnp6RgcHIz169dHb29v\nbN++PcbHx+8pc+rUqdixY0dERGzevDnm5+fj2rVrNZf9wAc+EG+99VZERMzPz8fAwEAz1g0AAAAA\nAAAomZ5af5ybm4t169bdeV+pVOLs2bN1y8zNzcXVq1cXXfbgwYPxqU99Kr7+9a/Hf//73/jDH/6w\nIisDAAAAAAAAlFvNALMoiiV9SLVaXdY/umvXrvjpT38an/nMZ+JXv/pV7Ny5M06fPv3Asvv377/z\n36OjozE6OrqsfwsAAAAAAABonqmpqZiamlqxz6sZYA4MDMTs7Oyd97Ozs1GpVGqWuXLlSlQqlbh1\n69aiy05PT8crr7wSERGf/exn48tf/vKidXhngAkAAABAY/r61sbbb79Zs8zq1Y/HjRvXW1QjAADK\n7t2DEA8cONDQ59V8BuamTZtiZmYmLl++HDdv3oyTJ0/G2NjYPWXGxsbi+PHjERFx5syZWLNmTfT3\n99dcdnBwMH73u99FRMSrr74aGzZsaGglAAAAAFia2+FltearXsAJAADNVHMEZk9PTxw+fDi2bt0a\nCwsLsWvXrhgeHo4jR45ERMTu3btj27ZtMTExEYODg7Fq1ao4duxYzWUjIo4ePRpf+cpX4t///ne8\n5z3viaNHjzZ5NQEAAAAAAIAyKKrLfYBlCxVFsezna5LD7een1tt2tm8r2SYAAABEOD98WNqNTmA/\nBqBVGs34ak4hCwAAAAAAK6mvb20URVHz1de3tt3VBKCNjMCkKdzNlY9tAgAAQITzw4el3egEWfbj\nLPUAoHmMwAQAAAAAAAA6hgATAAAAAAAASEOACQAAAAAAAKQhwAQAAAAAAADSEGACAAAAAAAAaQgw\nAQAAAAAAgDQEmAAAAAAAAEAaAkwAAABgWfr61kZRFDVffX1r211NAACgpHraXQEAAACgXN5++82I\nqNYpU7SmMgAAQMcxAhMAAAAAgPvUG3FvtD0AzSLABFIyJRUAAABAe90dcf/g1+2/A8DKM4UskJIp\nqQAAAAAAoDsZgQkAAEB6ZugAAADoHgJMAAAA0qs3hZ1p7AAAYOW4gZB2E2ACACyi3sG6A3UAAACg\nE7mBkHYTYAIALKLewboDdQAAAMrOSLvW0M6wPEW1Wq22uxKLKYoiElePGoqiiNsXd2uWsn1bqGzb\npGz1BTpT/e8i30MAreL4MBfbo9xsv4ej3egEy92Pm3VOpD/lYnu0RtnauWz1JZ9GMz4jMAEAAAAA\nAIA0BJgAAABQIqYfKy/bDgAAlkaACQAAACVS7xnNntOcl21Ho4TgAEC3EGCuIAeRAAAAADSLEBwA\n6BY97a5AJ7l7EFmrTNGaygAAAAAAAEAJGYEJAMCKMisFAAAAAI0QYAIAsKKaNbWZYBQAAACgO5hC\nFgCAUjBdPwAAAEB3MAITAAAAAAAASEOACQAAAJSS6cWBdvM9BADNYQpZAAAAoJRMLw60m+8hAGgO\nIzABAAAAAACANASYAEBLmFoJAAAAAFgKASYALJMg7uHcnVpp8dftMgAAAABAN/MMTABYJs84AQAA\nAABoHiMwgZYxag0AAAAAAKjHCEygZYxaAwAAAAAA6jECEwAAAICOUW/2HzP/AADkJ8AEAIAmMX06\nQDn5/r6rjG1xd/afB79u/x0AgMxMIQsA3KOvb23dizqrVz8eN25cb1GNoLxMnw5QTr6/79IWAAC0\ngxGYAG1QxruY6R717lh31zpAY0xtCAAAALUJMGk7QQ7dSEAE0FyOL8jM1IYAfqsBAKjNFLJdLsM0\ngaajAQBWmuMLAMjNbzUAALUYgdmBlnMXo1FgAEA7GX0BAAAAwLsJMDuQUBIAKAvHLQAAAJ3LTavA\nwxJgAgAAAB3PBVQAaD03rTafYxw6lQATgLZykAUAQCu4gAoslfNUoEwc49Cp0geYzThYcBACkIeD\nLACgLJxLAnQH56nl5bcaoHOkDzCbcbDgIAQAaCcn1QDl5FwSWCrHe9AefqsBOkcJAkwAgM7ipBqA\nbiLIoRs53gMAaIwAE6CLuZgE3Uv/B6BVBDkAAMByCTApHRdcYeW4mESjfCeXl/4PNIPfBQAAAFZC\nT7srAMt194JrrTJFayoDCfX1ra0ZOqxe/XjcuHG9hTWik/lOBuCd/C4AAACwEozABOgw9UZVGVEF\nUH5lHOVWxjoDAJ3BcQiZ2T8BHkyACUBHcgJAp7Av8yBlnAK4jHUGADqD4xBabTnncWXbP52jAq0i\nwARgScp2gFq2EwBYjH0ZAIAyK9u5JKyEDOdxzep7GdYN6A4dE2A6GAJoLgeoAGSU4TwgQx0AICvn\nktAe+h5Qdh0TYPpCBgCA7pPhPCBDHSg3ITgA7+a3AYBu1zEBJgAAAJRRlhDcxXK6kf0+F9vjriy/\nDeSijwDdRIAJAE3k5AIAKAsXy+lG9vtclrM9nGvRjXxnAd1EgAlQgxMiGuXkAgBoJ8ezQKdyrgUA\nnU2ACVCDEyIAoCwEVTyI41kAoNs5ToZyEmACAAB0AEEVANBOQiKycpwM5STAZMkchAAAdDbHewDL\n57sT4DYhEQArSYDJkjkIaT4nvgCQW6f/Vjveu6vTtzWwcnx3kpXfsu5hWz8c7ZaPbQK8U0+7K9DN\n+vrW1jyJWb368bhx43oLa0S73T3xrVWmaE1lAID7+K3uHp2+reudi0Q4HwEou07/LeMu2/rhaLd8\nbBPgnYzAbKN6d2m6QxMAVpa7OcnM/kkrGTEGALA89Y7XHasDrCwBJgDQNVywJzP7JwB0JzcxQTkY\njJJLs747fSe3hhsCWApTyAIAAABAm5gyEWD5mvXd6Tu5Neq1szYmwghMoEHuSgIAysJxCwAszu8k\nAJCJEZhAQ9yVBACUheMWAFic30mAPPr61tadlnj16sfjxo3rLaoRtJ4RmFBi7o4EAKDMPPsGAADu\nV++Zq567SjcQYEKJNeuHTDAK7aHvAdBt6h3PuigDS+eGAABoLddxoLkEmMB93OHzcBy00Ch9D4CM\nHONAObghAABay3UcaC4BJsAKcdBCK7mYDECrOMYBAACg1QSYAFBCnX4xWUALAN3JMQAAQPk4hqMZ\netpdAciir29t3Yv9q1c/HjduXG9RjQC6192AtlaZojWVAQBaxjEAAED5OIajGYzAhP/p9NFMna7e\nXT7u8Gktd13RSvY3aI9m9j39GoB38rsAANB9jMAEOkK9u3zc4dNa7rqilexv0B7N7Hv6NQDv5HcB\nAKD7GIFZR5a7/LLUg+Wz7fKxTeg29nkAuo3fPgAAgHIzArOOLHf5ZalH2WR4rqVtl49tQrexzwPQ\nbfz2AQDA/TJcL4elEmDS0Vy4AAAAAAAA18spl7pTyE5OTsbGjRtjaGgoDh069MAy+/bti6GhoRgZ\nGYlz584tadmf/exnMTw8HE8++WR885vfbHA1AAAAIA/T2NKN7Pfdw7amlexvAN2p5gjMhYWF2Lt3\nb7zyyisxMDAQTz/9dIyNjcXw8PCdMhMTE3Hp0qWYmZmJs2fPxp49e+LMmTM1l/3tb38bp06dij//\n+c/R29sb//jHP5q+ogAAzZRhGpYMdQDgNne3043s993DtqaV7G8A3anmCMzp6ekYHByM9evXR29v\nb2zfvj3Gx8fvKXPq1KnYsWNHRERs3rw55ufn49q1azWX/fnPfx7f+ta3ore3NyIi3ve+9zVj3QAA\nWubuSfXir3rhYifUAVhZRhwAAADQjWqOwJybm4t169bdeV+pVOLs2bN1y8zNzcXVq1cXXXZmZiZ+\n//vfx7e//e147LHH4sc//nFs2rRpkVrsf8d/j/7vBQAA0PmMOAAAAKAMpqamYmpqasU+r2aAWRRL\nOxGuVmufUL/bf/7zn3jzzTfjzJkz8dprr8XnPve5+Nvf/rZI6f3L+mwAAFZevelpTU0LAAC1eeQD\nAJ1sdHQ0RkdH77w/cOBAQ59XM8AcGBiI2dnZO+9nZ2ejUqnULHPlypWoVCpx69atRZetVCrx/PPP\nR0TE008/HY888kj885//jPe+970NrQwAAM1RbxSYEWAAAFCbmRUAYOlqPgNz06ZNMTMzE5cvX46b\nN2/GyZMnY2xs7J4yY2Njcfz48YiIOHPmTKxZsyb6+/trLvvcc8/Fq6++GhERFy9ejJs3bwovARbh\n2VetoZ0BAAAAAHKoOQKzp6cnDh8+HFu3bo2FhYXYtWtXDA8Px5EjRyIiYvfu3bFt27aYmJiIwcHB\nWLVqVRw7dqzmshERO3fujJ07d8ZTTz0Vjz766J0AFID7uUOzNbQzAAAAAEAORXW5D7BsodvP4KxX\nvSKq1WrpykYsZf2WU7b5dc5QNiJHWyxHp7dFhrIR+lO3tMVy6E8PVzZCW3RLf9IW99Z5qcrYn5bD\nd0vzy0Z0dn8qY1ssR3narfn1yFA2orPbYrky9KcMZSO0hf7UWNkIbaE/3V82QlvoT42VjdAWD9uf\nKK+iaGw71pxCFgAAOl2nTyHd6esHAAAAdJ6aU8gCAECn6/QppDt9/QAAAIDOYwQmAAAAXcsoZQAA\ngHyMwAQAAKBrGaUMAACQjxGYAAAAAMkYHQwAQDczAhMAAAAgGaODAQDoZkZgAgAAAAAAAGkIMAEA\nAAAAAIA0BJgAAAAAAABAGgJMAAAAAAAAIA0BJgAAAAAAAJCGABOA0ujrWxtFUdR89fWtbXc1AQAA\nAABoQE+7KwAAS/X2229GRLVOmaI1lQEAAAAAoCmMwAQAAAAAAADSEGACAAAAAAAAaQgwAQAAAAAA\ngDQEmAAAAAAAAEAaAkwAAAAAAAAgDQEmPIS+vrVRFEXNV1/f2nZXEwAAAAAAoHR62l0BKKO3334z\nIqp1yhStqQwAAAAAAEAHMQITAAAAAAAASEOACQAAAAAAAKQhwAQAAAAAAADSEGACAAAAAAAAaQgw\nAQAAAAAAgDQEmAAAAAAAAEAaAkwAAAAAAAAgDQEmAAAAAAAAkIYAEwAAAAAAAEhDgAkAAAAAAACk\nIcAEAAAAAAAA0hBgAgAAAAAAAGkIMAEAAAAAAIA0BJgAAAAAAABAGgJMAAAAAAAAIA0BJgAAAAAA\nAJCGABMAAAAAAABIQ4AJAAAAAAAApCHABAAAAAAAANIQYAIAAAAAAABpCDABAAAAAACANASYAAAA\nAAAAQBoCTAAAAAAAACANASYAAAAAAACQhgATAAAAAAAASEOACQAAAAAAAKQhwAQAAAAAAADSEGAC\nAAAAAAAAaQgwAQAAAAAAgDQEmAAAAAAAAEAaAkwAAAAAAAAgDQEmAAAAAAAAkIYAEwAAAAAAAEhD\ngAkAAAAAAACkIcAEAAAAAAAA0hBgAgAAAAAAAGkIMAEAAAAAAIA0BJgAAAAAAABAGgJMAAAAAAAA\nIA0BJgAAAAAAAJCGABMAAAAAAABIQ4AJAAAAAAAApCHABAAAAAAAANIQYAIAAAAAAABpCDABAAAA\nAACANASYAAAAAAAAQBoCTAAAAAAAACANASYAAAAAAACQhgATAAAAAAAASEOACQAAAAAAAKQhwAQA\nAAAAAADSEGACAAAAAAAAaQgwAQAAAAAAgDQEmAAAAAAAAEAaAkwAAAAAAAAgDQEmAAAAAAAAkIYA\nEwAAAAAAAEhDgAkAAAAAAACkIcAEAAAAAAAA0hBgAgAAAAAAAGnUDTAnJydj48aNMTQ0FIcOHXpg\nmX379sXQ0FCMjIzEuXPnlrzsT37yk3jkkUfi+vXrDawCAAAAAAAA0ClqBpgLCwuxd+/emJycjPPn\nz8eJEyfiwoUL95SZmJiIS5cuxczMTBw9ejT27NmzpGVnZ2fj9OnT8cEPfrAJqwUAAAAAAACUUc0A\nc3p6OgYHB2P9+vXR29sb27dvj/Hx8XvKnDp1Knbs2BEREZs3b475+fm4du1a3WW/9rWvxQ9/+MMm\nrBIAAAAAAABQVj21/jg3Nxfr1q27875SqcTZs2frlpmbm4urV68uuuz4+HhUKpX4yEc+soQq7n/H\nf4/+7wUAAAAAAABkMDU1FVNTUyv2eTUDzKIolvQh1Wp1yf/gv/71r/j+978fp0+fXuLy+5f82QAA\nAAAAAEBrjY6Oxujo6J33Bw4caOjzagaYAwMDMTs7e+f97OxsVCqVmmWuXLkSlUolbt269cBl//rX\nv8bly5djZGTkTvmPfexjMT09HU888URDKwMAAAAAAACUW81nYG7atClmZmbi8uXLcfPmzTh58mSM\njY3dU2ZsbCyOHz8eERFnzpyJNWvWRH9//6LLPvnkk/HGG2/E66+/Hq+//npUKpX44x//KLwEAAAA\nAAAAao/A7OnpicOHD8fWrVtjYWEhdu3aFcPDw3HkyJGIiNi9e3ds27YtJiYmYnBwMFatWhXHjh2r\nuey7LXWaWgAAAAAAAKDzFdXlPMCyxW6Hm/WqV0S1Wi1d2YilrN9yyja/zhnKRmgLbXF/2Qj9SVs0\nVjZCW2iL+8tG6E/aorGyEdpCW9xfNkJ/0haNlY3QFtri/rIR+pO2aKxshLbQFveXjdCftEVjZSO0\nhba4v2zE8voT5VUUjW3HmlPIAgAAAAAAALSSABMAAAAAAABIQ4AJAAAAAAAApCHABAAAAAAAANIQ\nYAIAAAAAAABpCDABAAAAAACANASYAAAAAAAAQBoCTAAAAAAAACANASYAAAAAAACQhgATAAAAAAAA\nSEOACQAAAAAAAKQhwAQAAAAAAADSEGACAAAAAAAAaQgwAQAAAAAAgDQEmAAAAAAAAEAaAkwAAAAA\nAAAgDQEmAAAAAAAAkIYAEwAAAAAAAEhDgAkAAAAAAACkIcAEAAAAAAAA0hBgAgAAAAAAAGkIMAEA\nAAAAAIA0BJgAAAAAAABAGgJMAAAAAAAAIA0BJgAAAAAAAJCGABMAAAAAAABIQ4AJAAAAAAAApCHA\nBAAAAAAAANIQYAIAAAAAAABpCDABAAAAAACANASYAAAAAAAAQBoCTAAAAAAAACANASYAAAAAAACQ\nhgATAAAAAAAASEOACQAAAAAAAKQhwAQAAAAAAADSEGACAAAAAAAAaQgwAQAAAAAAgDQEmAAAAAAA\nAEAaAkwAAAAAAAAgDQEmAAAAAAAAkIYAEwAAAAAAAEhDgAkAAAAAAACkIcAEAAAAAAAA0hBgAgAA\nAAAAAGkIMAEAAAAAAIA0BJgAAAAAAABAGgJMAAAAAAAAIA0BJgAAAAAAAJCGABMAAAAAAABIQ4AJ\nAAAAAAAApCHABAAAAAAAANIQYAIAAAAAAABpCDABAAAAAACANASYAAAAAAAAQBoCTAAAAAAAACAN\nASYAAAAAAACQhgATAAAAAAAASEOACQAAAAAAAKQhwAQAAAAAAADSEGACAAAAAAAAaQgwAQAAAAAA\ngDQEmAAAAAAAAEAaAkwAAAAAAAAgDQEmAAAAAAAAkIYAEwAAAAAAAEhDgAkAAAAAAACkIcAEAAAA\nAAAA0hBgAgAAAAAAAGkIMAEAAAAAAIA0BJgAAAAAAABAGgJMAAAAAAAAIA0BJgAAAAAAAJCGABMA\nAAAAAABIQ4AJAAAAAAAApCHABAAAAAAAANIQYAIAAAAAAABpCDABAAAAAACANASYAAAAAAAAQBoC\nTAAAAAAAACCNJQWYk5OTsXHjxhgaGopDhw49sMy+fftiaGgoRkZG4ty5c3WX/cY3vhHDw8MxMjIS\nzz//fLz11lsNrgoAAAAAAABQdnUDzIWFhdi7d29MTk7G+fPn48SJE3HhwoV7ykxMTMSlS5diZmYm\njh49Gnv27Km77LPPPht/+ctf4k9/+lNs2LAhfvCDHzRh9QAAAAAAAIAyqRtgTk9Px+DgYKxfvz56\ne3tj+/btMT4+fk+ZU6dOxY4dOyIiYvPmzTE/Px/Xrl2rueyWLVvikUceubPMlStXVnrdAAAAAAAA\ngJKpG2DOzc3FunXr7ryvVCoxNze3pDJXr16tu2xExC9+8YvYtm3bQ60AAAAAAAAA0Dl66hUoimJJ\nH1StVh+qAt/73vfi0UcfjS9+8YuLlNj/jv8e/d8LAAAAAAAAyGBqaiqmpqZW7PPqBpgDAwMxOzt7\n5/3s7GxUKpWaZa5cuRKVSiVu3bpVc9lf/vKXMTExEb/5zW9q1GD/ElYDAAAAAAAAaIfR0dEYHR29\n8/7AgQMNfV7dKWQ3bdoUMzMzcfny5bh582acPHkyxsbG7ikzNjYWx48fj4iIM2fOxJo1a6K/v7/m\nspOTk/GjH/0oxsfH47HHHmtoJQAAAAAAAIDOUHcEZk9PTxw+fDi2bt0aCwsLsWvXrhgeHo4jR45E\nRMTu3btj27ZtMTExEYODg7Fq1ao4duxYzWUjIr761a/GzZs3Y8uWLRER8YlPfCJeeumlZq0nAAAA\nAAAAUAKi6CH3AAAN6ElEQVRF9WEfXtkCt5+/Wa96RVSr1dKVjVjK+i2nbPPrnKFshLbQFveXjdCf\ntEVjZSO0hba4v2yE/qQtGisboS20xf1lI/QnbdFY2QhtoS3uLxuhP2mLxspGaAttcX/ZCP1JWzRW\nNkJbaIv7y0Ysrz9RXkXR2HasO4UsAAAAAAAAQKsIMAEAAAAAAIA0BJgAAAAAAABAGgJMAAAAAAAA\nIA0BJgAAAAAAAJCGABMAAAAAAABIQ4AJAAAAAAAApCHABAAAAAAAANIQYAIAAAAAAABpCDABAAAA\nAACANASYAAAAAAAAQBoCTAAAAAAAACANASYAAAAAAACQhgATAAAAAAAASEOACQAAAAAAAKQhwAQA\nAAAAAADSEGACAAAAAAAAaQgwAQAAAAAAgDQEmAAAAAAAAEAaAkwAAAAAAAAgDQEmAAAAAAAAkIYA\nEwAAAAAAAEhDgAkAAAAAAACkIcAEAAAAAAAA0hBgAgAAAAAAAGkIMAEAAAAAAIA0BJgAAAAAAABA\nGgJMAAAAAAAAIA0BJgAAAAAAAJCGABMAAAAAAABIQ4AJAAAAAAAApCHABAAAAAAAANIQYAIAAAAA\nAABpCDABAAAAAACANASYAAAAAAAAQBoCTAAAAAAAACANASYAAAAAAACQhgATAAAAAAAASEOACQAA\nAAAAAKQhwAQAAAAAAADSEGACAAAAAAAAaQgwAQAAAAAAgDQEmAAAAAAAAEAaAkwAAAAAAAAgDQEm\nAAAAAAAAkIYAEwAAAAAAAEhDgAkAAAAAAACkIcAEAAAAAAAA0hBgAgAAAAAAAGkIMAEAAAAAAIA0\nBJgAAAAAAABAGgJMAAAAAAAAIA0BJgAAAAAAAJCGABMAAAAAAABIQ4AJAAAAAAAApCHABAAAAAAA\nANIQYAIAAAAAAABpCDABAAAAAACANASYAAAAAAAAQBoCTAAAAAAAACANASYAAAAAAACQhgATAAAA\nAAAASEOACQAAAAAAAKQhwAQAAAAAAADSEGACAAAAAAAAaQgwAQAAAAAAgDQEmAAAAAAAAEAaAkwA\nAAAAAAAgDQEmAAAAAAAAkIYAEwAAAAAAAEhDgAkAAAAAAACkIcAEAAAAAAAA0hBgAgAAAAAAAGkI\nMAEAAAAAAIA0BJgAAAAAAABAGgJMAAAAAAAAIA0BJgAAAAAAAJCGABMAAAAAAABIQ4AJAAAAAAAA\npCHABAAAAAAAANIQYAIAAAAAAABp1A0wJycnY+PGjTE0NBSHDh16YJl9+/bF0NBQjIyMxLlz5+ou\ne/369diyZUts2LAhnn322Zifn1+BVQEAAAAAAADKrmaAubCwEHv37o3Jyck4f/58nDhxIi5cuHBP\nmYmJibh06VLMzMzE0aNHY8+ePXWXPXjwYGzZsiUuXrwYzzzzTBw8eLBJqwcAAAAAAACUSc0Ac3p6\nOgYHB2P9+vXR29sb27dvj/Hx8XvKnDp1Knbs2BEREZs3b475+f9r725D2yr/MI5fKckQ5kOdrinr\nqXbYjSZdV9t1TgXBB7JhYaF0o9jJLG3VwRB8xu3lBJdpEZ2rE5E5Oye2e7WWUcIsExW1GyOy6Vps\nNxJI0jUwZ4ezw9iY/4uxYP23ibZ5avL9vMt9fqf5nZteND13z90JjY+Pxz337+e0tLTo6NGjqbg2\nAAAAAAAAAAAAAAuMOd7BYDCo0tLS2GvDMHTy5MmENcFgUGNjY7OeGwqFZLVaJUlWq1WhUChOF6aE\nF2Ey3ahZaLWJ6/9L7dz7WGi12dJHNtRmSx/ZUJu4Pn/mLXF9/swFeWIu5lubuD5/5i1xff7MBXli\nLuZbm7g+f+YtcX3+zAV5Yi7mW5u4Pn/mLXF9/swFeWIu5lubuD5/5i1xff7MBXliLuZbm7h+ei3y\nUdwFzH/7DRKNRv9VzUxfz2Qyzfo+/+brAgAAAAAAAAAAAMgdcbeQLSkpkd/vj732+/0yDCNuTSAQ\nkGEYM46XlJRIuv7U5fj4uCTp4sWLKioqmv+VAAAAAAAAAAAAAFjw4i5g1tXVaXR0VD6fT+FwWD09\nPXI6ndNqnE6nDh06JEkaHBxUYWGhrFZr3HOdTqe6urokSV1dXWpoaEjFtQEAAAAAAAAAAABYYOJu\nIWs2m9XZ2akNGzYoEomovb1dNptNH374oSRp27Ztqq+vV39/v8rLy7V48WIdPHgw7rmStGPHDjU1\nNenAgQMqKyvTkSNHUnyZAAAAAAAAAAAAABaCuE9gStLjjz+un3/+WefPn9fOnTslXV+43LZtW6ym\ns7NT58+f15kzZ1RbWxv3XElasmSJBgYGNDIyouPHj6uwsHDae7rdblVUVGjFihV68803532RAGbm\n9/v1yCOPqLKyUqtWrdJ7770nSbp8+bIcDodWrlyp9evXa2JiIsOdArkpEomopqZGGzdulET2gHSZ\nmJjQ5s2bZbPZZLfbdfLkSfIHpIHL5VJlZaWqqqq0ZcsW/fHHH2QPSIG2tjZZrVZVVVXFxuJlzeVy\nacWKFaqoqNDx48cz0TKQE2bK3quvviqbzabq6mo1NjbqypUrsWNkD0iOmbJ3w9tvv62CggJdvnw5\nNkb2gOSZLX/79u2TzWbTqlWr9Nprr8XG/2v+Ei5gplskEtFzzz0nt9utoaEhff755xoeHs50W0BO\nslgseuedd3Tu3DkNDg7q/fff1/DwsPbs2SOHw6GRkRE99thj2rNnT6ZbBXLS3r17ZbfbZTKZJIns\nAWny/PPPq76+XsPDwzp79qwqKirIH5BiPp9PH330kTwej3788UdFIhF1d3eTPSAFWltb5Xa7p43N\nlrWhoSH19PRoaGhIbrdb27dv119//ZWJtoEFb6bsrV+/XufOndOZM2e0cuVKuVwuSWQPSKaZsidd\nf3Djiy++0N133x0bI3tAcs2Uvy+//FJ9fX06e/asfvrpJ73yyiuS5pa/rFvAPHXqlMrLy1VWViaL\nxaInnnhCvb29mW4LyEnFxcW69957JUk333yzbDabgsGg+vr61NLSIklqaWnR0aNHM9kmkJMCgYD6\n+/v19NNPKxqNShLZA9LgypUr+uabb9TW1ibp+r89uO2228gfkGK33nqrLBaLJicnNTU1pcnJSS1b\ntozsASnw0EMP6fbbb582NlvWent71dzcLIvForKyMpWXl+vUqVNp7xnIBTNlz+FwqKDg+u3XdevW\nKRAISCJ7QDLNlD1Jeumll/TWW29NGyN7QHLNlL8PPvhAO3fulMVikSQtXbpU0tzyl3ULmMFgUKWl\npbHXhmEoGAxmsCMgP/h8Pv3www9at26dQqGQrFarJMlqtSoUCmW4OyD3vPjii+ro6Ij9MiuJ7AFp\n4PV6tXTpUrW2tqq2tlbPPPOMfv/9d/IHpNiSJUv08ssv66677tKyZctUWFgoh8NB9oA0mS1rY2Nj\nMgwjVsc9GCB1Pv74Y9XX10sie0Cq9fb2yjAMrV69eto42QNSb3R0VF9//bXuv/9+Pfzwwzp9+rSk\nueUv6xYwb2yjByB9rl69qk2bNmnv3r265ZZbph0zmUzkEkiyY8eOqaioSDU1NbGnL/+J7AGpMTU1\nJY/Ho+3bt8vj8Wjx4sX/t2Ul+QOS78KFC3r33Xfl8/k0Njamq1ev6vDhw9NqyB6QHomyRg6B5Hvj\njTe0aNEibdmyZdYasgckx+TkpHbv3q1du3bFxma79yKRPSDZpqam9Ouvv2pwcFAdHR1qamqatTZR\n/rJuAbOkpER+vz/22u/3T1uVBZBcf/75pzZt2qStW7eqoaFB0vW/yB0fH5ckXbx4UUVFRZlsEcg5\n3333nfr6+rR8+XI1NzfrxIkT2rp1K9kD0sAwDBmGobVr10qSNm/eLI/Ho+LiYvIHpNDp06f14IMP\n6o477pDZbFZjY6O+//57sgekyWyfM/95DyYQCKikpCQjPQK56pNPPlF/f78+++yz2BjZA1LnwoUL\n8vl8qq6u1vLlyxUIBLRmzRqFQiGyB6SBYRhqbGyUJK1du1YFBQW6dOnSnPKXdQuYdXV1Gh0dlc/n\nUzgcVk9Pj5xOZ6bbAnJSNBpVe3u77Ha7Xnjhhdi40+lUV1eXJKmrqyu2sAkgOXbv3i2/3y+v16vu\n7m49+uij+vTTT8kekAbFxcUqLS3VyMiIJGlgYECVlZXauHEj+QNSqKKiQoODg7p27Zqi0agGBgZk\nt9vJHpAms33OdDqd6u7uVjgcltfr1ejoqO67775MtgrkFLfbrY6ODvX29uqmm26KjZM9IHWqqqoU\nCoXk9Xrl9XplGIY8Ho+sVivZA9KgoaFBJ06ckCSNjIwoHA7rzjvvnFP+zOlo+L8wm83q7OzUhg0b\nFIlE1N7eLpvNlum2gJz07bff6vDhw1q9erVqamokSS6XSzt27FBTU5MOHDigsrIyHTlyJMOdArnt\nxnYJZA9Ij3379unJJ59UOBzWPffco4MHDyoSiZA/IIWqq6v11FNPqa6uTgUFBaqtrdWzzz6r3377\njewBSdbc3KyvvvpKly5dUmlpqV5//fVZP2fa7XY1NTXJbrfLbDZr//79bKUHzNE/s7dr1y65XC6F\nw2E5HA5J0gMPPKD9+/eTPSCJbmTvl19+if3ca21tjR3/e7bIHpBcM+Wvra1NbW1tqqqq0qJFi3To\n0CFJc8ufKRpvA2gAAAAAAAAAAAAASKOs20IWAAAAAAAAAAAAQP5iARMAAAAAAAAAAABA1mABEwAA\nAAAAAAAAAEDWYAETAAAAAAAAAAAAQNZgARMAAAAAAAAAAABA1vgfX2D2+0Kmm+cAAAAASUVORK5C\nYII=\n", "text": [ "" ] } ], "prompt_number": 66 }, { "cell_type": "code", "collapsed": false, "input": [], "language": "python", "metadata": {}, "outputs": [] } ], "metadata": {} } ] }