{ "metadata": { "name": "TEST_theanoml2_formula_optimize" }, "nbformat": 3, "nbformat_minor": 0, "worksheets": [ { "cells": [ { "cell_type": "code", "collapsed": false, "input": [ "import theanoml2\n", "reload(theanoml2)" ], "language": "python", "metadata": {}, "outputs": [ { "output_type": "pyout", "prompt_number": 1, "text": [ "" ] } ], "prompt_number": 1 }, { "cell_type": "code", "collapsed": false, "input": [ "reload(theanoml2.formula)\n", "reload(theanoml2.optimize)\n", "import cPickle\n", "import numpy as np\n", "from sklearn.cross_validation import train_test_split\n", "from functools import partial\n", "X, y = cPickle.load(open('data/digits.pkl', 'rb'))\n", "X = X / 255.\n", "classes = np.unique(y)\n", "print X.shape, y.shape\n", "train_X, validation_X, train_y, validation_y = train_test_split(X, y, test_size = 0.2)" ], "language": "python", "metadata": {}, "outputs": [ { "output_type": "stream", "stream": "stdout", "text": [ "(42000, 784) (42000,)\n" ] } ], "prompt_number": 9 }, { "cell_type": "raw", "metadata": {}, "source": [ "## Logistic Regression\n", "\n", "lgr = theanoml2.formula.FLogisticRegression(n_in = X.shape[1], n_out = len(classes))\n", "\n", "v_train_X = theanoml2.formula.share_data(train_X)\n", "v_validation_X = theanoml2.formula.share_data(validation_X)\n", "v_train_y, v_validation_y = map(partial(theanoml2.formula.share_data, dtype = 'int32'), [train_y, validation_y])\n", "\n", "lgr_infor = theanoml2.formula.build_batch_sgd_model_infor(lgr, \n", " v_train_X, v_train_y, v_validation_X, v_validation_y, batch_size = 500)\n", "\n", "theanoml2.optimize.batch_sgd_optimize(lgr_infor, n_epochs = 10)" ] }, { "cell_type": "raw", "metadata": {}, "source": [ "## auto denoising encoder\n", "reload(theanoml2)\n", "reload(theanoml2.formula)\n", "reload(theanoml2.optimize)\n", "\n", "ade = theanoml2.formula.FDAE(n_visible = X.shape[1], n_hidden = 500, corruption_level = 0.01)\n", "v_train_X = theanoml2.formula.share_data(train_X)\n", "ade_infor = theanoml2.formula.build_batch_fixed_iter_model_infor(ade, v_train_X, batch_size = 500)\n", "r = theanoml2.optimize.batch_fixed_iter_optimize(ade_infor, n_epochs = 5)\n", "print r[1].eval()" ] }, { "cell_type": "code", "collapsed": false, "input": [ "## test SdA classifier\n", "reload(theanoml2)\n", "reload(theanoml2.lenet)\n", "reload(theanoml2.formula)\n", "reload(theanoml2.auto_encoder)\n", "sda = theanoml2.auto_encoder.SdAClassifier(len(classes), pretrain_epochs = 15, \n", " finetune_epochs = 100)\n", "sda.formula_ = sda._create_formula(train_X, train_y)\n", "sda.pretrain(X) ## train on whole X\n", "sda.fit(train_X, train_y)\n", "print sda.score(train_X, train_y)\n", "print sda.score(validation_X, validation_y)" ], "language": "python", "metadata": {}, "outputs": [ { "output_type": "stream", "stream": "stdout", "text": [ "training epoch 0, train cost 122.080660\n", "training epoch 1, train cost 97.409468" ] }, { "output_type": "stream", "stream": "stdout", "text": [ "\n", "training epoch 2, train cost 91.860783" ] }, { "output_type": "stream", "stream": "stdout", "text": [ "\n", "training epoch 3, train cost 88.531213" ] }, { "output_type": "stream", "stream": "stdout", "text": [ "\n", "training epoch 4, train cost 86.063633" ] }, { "output_type": "stream", "stream": "stdout", "text": [ "\n", "training epoch 5, train cost 84.116139" ] }, { "output_type": "stream", "stream": "stdout", "text": [ "\n", "training epoch 6, train cost 82.559561" ] }, { "output_type": "stream", "stream": "stdout", "text": [ "\n", "training epoch 7, train cost 81.350771" ] }, { "output_type": "stream", "stream": "stdout", "text": [ "\n", "training epoch 8, train cost 80.438471" ] }, { "output_type": "stream", "stream": "stdout", "text": [ "\n", "training epoch 9, train cost 79.705019" ] }, { "output_type": "stream", "stream": "stdout", "text": [ "\n", "training epoch 10, train cost 79.149875" ] }, { "output_type": "stream", "stream": "stdout", "text": [ "\n", "training epoch 11, train cost 78.653986" ] }, { "output_type": "stream", "stream": "stdout", "text": [ "\n", "training epoch 12, train cost 78.222760" ] }, { "output_type": "stream", "stream": "stdout", "text": [ "\n", "training epoch 13, train cost 77.884829" ] }, { "output_type": "stream", "stream": "stdout", "text": [ "\n", "training epoch 14, train cost 77.607123" ] }, { "output_type": "stream", "stream": "stdout", "text": [ "\n", "optimization complete with best train cost: 77.607123\n", "training epoch 0, train cost 37.300914" ] }, { "output_type": "stream", "stream": "stdout", "text": [ "\n", "training epoch 1, train cost 32.143227" ] }, { "output_type": "stream", "stream": "stdout", "text": [ "\n", "training epoch 2, train cost 31.259929" ] }, { "output_type": "stream", "stream": "stdout", "text": [ "\n", "training epoch 3, train cost 30.858785" ] }, { "output_type": "stream", "stream": "stdout", "text": [ "\n", "training epoch 4, train cost 30.584925" ] }, { "output_type": "stream", "stream": "stdout", "text": [ "\n", "training epoch 5, train cost 30.392978" ] }, { "output_type": "stream", "stream": "stdout", "text": [ "\n", "training epoch 6, train cost 30.283089" ] }, { "output_type": "stream", "stream": "stdout", "text": [ "\n", "training epoch 7, train cost 30.123150" ] }, { "output_type": "stream", "stream": "stdout", "text": [ "\n", "training epoch 8, train cost 30.018182" ] }, { "output_type": "stream", "stream": "stdout", "text": [ "\n", "training epoch 9, train cost 29.913434" ] }, { "output_type": "stream", "stream": "stdout", "text": [ "\n", "training epoch 10, train cost 29.831760" ] }, { "output_type": "stream", "stream": "stdout", "text": [ "\n", "training epoch 11, train cost 29.768833" ] }, { "output_type": "stream", "stream": "stdout", "text": [ "\n", "training epoch 12, train cost 29.708539" ] }, { "output_type": "stream", "stream": "stdout", "text": [ "\n", "training epoch 13, train cost 29.649420" ] }, { "output_type": "stream", "stream": "stdout", "text": [ "\n", "training epoch 14, train cost 29.576363" ] }, { "output_type": "stream", "stream": "stdout", "text": [ "\n", "optimization complete with best train cost: 29.576363\n", "training epoch 0, train cost 44.067446" ] }, { "output_type": "stream", "stream": "stdout", "text": [ "\n", "training epoch 1, train cost 40.254193" ] }, { "output_type": "stream", "stream": "stdout", "text": [ "\n", "training epoch 2, train cost 39.663832" ] }, { "output_type": "stream", "stream": "stdout", "text": [ "\n", "training epoch 3, train cost 39.421751" ] }, { "output_type": "stream", "stream": "stdout", "text": [ "\n", "training epoch 4, train cost 39.224811" ] }, { "output_type": "stream", "stream": "stdout", "text": [ "\n", "training epoch 5, train cost 39.083494" ] }, { "output_type": "stream", "stream": "stdout", "text": [ "\n", "training epoch 6, train cost 38.955715" ] }, { "output_type": "stream", "stream": "stdout", "text": [ "\n", "training epoch 7, train cost 38.871046" ] }, { "output_type": "stream", "stream": "stdout", "text": [ "\n", "training epoch 8, train cost 38.763742" ] }, { "output_type": "stream", "stream": "stdout", "text": [ "\n", "training epoch 9, train cost 38.659808" ] }, { "output_type": "stream", "stream": "stdout", "text": [ "\n", "training epoch 10, train cost 38.616512" ] }, { "output_type": "stream", "stream": "stdout", "text": [ "\n", "training epoch 11, train cost 38.561521" ] }, { "output_type": "stream", "stream": "stdout", "text": [ "\n", "training epoch 12, train cost 38.500389" ] }, { "output_type": "stream", "stream": "stdout", "text": [ "\n", "training epoch 13, train cost 38.431741" ] }, { "output_type": "stream", "stream": "stdout", "text": [ "\n", "training epoch 14, train cost 38.374703" ] }, { "output_type": "stream", "stream": "stdout", "text": [ "\n", "optimization complete with best train cost: 38.374703\n", "epoch 1, minibatch 500/537, validation_error 21.746269 %" ] }, { "output_type": "stream", "stream": "stdout", "text": [ "\n", "epoch 2, minibatch 463/537, validation_error 12.880597 %" ] }, { "output_type": "stream", "stream": "stdout", "text": [ "\n", "epoch 3, minibatch 426/537, validation_error 10.328358 %" ] }, { "output_type": "stream", "stream": "stdout", "text": [ "\n", "epoch 4, minibatch 389/537, validation_error 9.552239 %" ] }, { "output_type": "stream", "stream": "stdout", "text": [ "\n", "epoch 5, minibatch 352/537, validation_error 8.567164 %" ] }, { "output_type": "stream", "stream": "stdout", "text": [ "\n", "epoch 6, minibatch 315/537, validation_error 8.432836 %" ] }, { "output_type": "stream", "stream": "stdout", "text": [ "\n", "epoch 7, minibatch 278/537, validation_error 7.686567 %" ] }, { "output_type": "stream", "stream": "stdout", "text": [ "\n", "epoch 8, minibatch 241/537, validation_error 7.000000 %" ] }, { "output_type": "stream", "stream": "stdout", "text": [ "\n", "epoch 9, minibatch 204/537, validation_error 6.567164 %" ] }, { "output_type": "stream", "stream": "stdout", "text": [ "\n", "epoch 10, minibatch 167/537, validation_error 6.432836 %" ] }, { "output_type": "stream", "stream": "stdout", "text": [ "\n", "epoch 11, minibatch 130/537, validation_error 6.313433 %" ] }, { "output_type": "stream", "stream": "stdout", "text": [ "\n", "epoch 12, minibatch 93/537, validation_error 5.880597 %" ] }, { "output_type": "stream", "stream": "stdout", "text": [ "\n", "epoch 13, minibatch 56/537, validation_error 5.611940 %" ] }, { "output_type": "stream", "stream": "stdout", "text": [ "\n", "epoch 14, minibatch 19/537, validation_error 5.492537 %" ] }, { "output_type": "stream", "stream": "stdout", "text": [ "\n", "epoch 14, minibatch 519/537, validation_error 5.223881 %" ] }, { "output_type": "stream", "stream": "stdout", "text": [ "\n", "epoch 15, minibatch 482/537, validation_error 5.328358 %" ] }, { "output_type": "stream", "stream": "stdout", "text": [ "\n", "epoch 16, minibatch 445/537, validation_error 5.149254 %" ] }, { "output_type": "stream", "stream": "stdout", "text": [ "\n", "epoch 17, minibatch 408/537, validation_error 5.119403 %" ] }, { "output_type": "stream", "stream": "stdout", "text": [ "\n", "epoch 18, minibatch 371/537, validation_error 5.000000 %" ] }, { "output_type": "stream", "stream": "stdout", "text": [ "\n", "epoch 19, minibatch 334/537, validation_error 4.820896 %" ] }, { "output_type": "stream", "stream": "stdout", "text": [ "\n", "epoch 20, minibatch 297/537, validation_error 4.880597 %" ] }, { "output_type": "stream", "stream": "stdout", "text": [ "\n", "epoch 21, minibatch 260/537, validation_error 4.791045 %" ] }, { "output_type": "stream", "stream": "stdout", "text": [ "\n", "epoch 22, minibatch 223/537, validation_error 4.522388 %" ] }, { "output_type": "stream", "stream": "stdout", "text": [ "\n", "epoch 23, minibatch 186/537, validation_error 4.611940 %" ] }, { "output_type": "stream", "stream": "stdout", "text": [ "\n", "epoch 24, minibatch 149/537, validation_error 4.641791 %" ] }, { "output_type": "stream", "stream": "stdout", "text": [ "\n", "epoch 25, minibatch 112/537, validation_error 4.462687 %" ] }, { "output_type": "stream", "stream": "stdout", "text": [ "\n", "epoch 26, minibatch 75/537, validation_error 4.492537 %" ] }, { "output_type": "stream", "stream": "stdout", "text": [ "\n", "epoch 27, minibatch 38/537, validation_error 4.313433 %" ] }, { "output_type": "stream", "stream": "stdout", "text": [ "\n", "epoch 28, minibatch 1/537, validation_error 4.134328 %" ] }, { "output_type": "stream", "stream": "stdout", "text": [ "\n", "epoch 28, minibatch 501/537, validation_error 4.179104 %" ] }, { "output_type": "stream", "stream": "stdout", "text": [ "\n", "epoch 29, minibatch 464/537, validation_error 4.194030 %" ] }, { "output_type": "stream", "stream": "stdout", "text": [ "\n", "epoch 30, minibatch 427/537, validation_error 4.164179 %" ] }, { "output_type": "stream", "stream": "stdout", "text": [ "\n", "epoch 31, minibatch 390/537, validation_error 4.149254 %" ] }, { "output_type": "stream", "stream": "stdout", "text": [ "\n", "epoch 32, minibatch 353/537, validation_error 3.985075 %" ] }, { "output_type": "stream", "stream": "stdout", "text": [ "\n", "epoch 33, minibatch 316/537, validation_error 4.029851 %" ] }, { "output_type": "stream", "stream": "stdout", "text": [ "\n", "epoch 34, minibatch 279/537, validation_error 3.985075 %" ] }, { "output_type": "stream", "stream": "stdout", "text": [ "\n", "epoch 35, minibatch 242/537, validation_error 3.791045 %" ] }, { "output_type": "stream", "stream": "stdout", "text": [ "\n", "epoch 36, minibatch 205/537, validation_error 3.955224 %" ] }, { "output_type": "stream", "stream": "stdout", "text": [ "\n", "epoch 37, minibatch 168/537, validation_error 3.865672 %" ] }, { "output_type": "stream", "stream": "stdout", "text": [ "\n", "epoch 38, minibatch 131/537, validation_error 4.238806 %" ] }, { "output_type": "stream", "stream": "stdout", "text": [ "\n", "epoch 39, minibatch 94/537, validation_error 3.776119 %" ] }, { "output_type": "stream", "stream": "stdout", "text": [ "\n", "epoch 40, minibatch 57/537, validation_error 3.865672 %" ] }, { "output_type": "stream", "stream": "stdout", "text": [ "\n", "epoch 41, minibatch 20/537, validation_error 3.985075 %" ] }, { "output_type": "stream", "stream": "stdout", "text": [ "\n", "epoch 41, minibatch 520/537, validation_error 3.761194 %" ] }, { "output_type": "stream", "stream": "stdout", "text": [ "\n", "epoch 42, minibatch 483/537, validation_error 3.910448 %" ] }, { "output_type": "stream", "stream": "stdout", "text": [ "\n", "epoch 43, minibatch 446/537, validation_error 3.597015 %" ] }, { "output_type": "stream", "stream": "stdout", "text": [ "\n", "epoch 44, minibatch 409/537, validation_error 3.626866 %" ] }, { "output_type": "stream", "stream": "stdout", "text": [ "\n", "epoch 45, minibatch 372/537, validation_error 3.671642 %" ] }, { "output_type": "stream", "stream": "stdout", "text": [ "\n", "epoch 46, minibatch 335/537, validation_error 3.671642 %" ] }, { "output_type": "stream", "stream": "stdout", "text": [ "\n", "epoch 47, minibatch 298/537, validation_error 3.850746 %" ] }, { "output_type": "stream", "stream": "stdout", "text": [ "\n", "epoch 48, minibatch 261/537, validation_error 3.656716 %" ] }, { "output_type": "stream", "stream": "stdout", "text": [ "\n", "epoch 49, minibatch 224/537, validation_error 3.626866 %" ] }, { "output_type": "stream", "stream": "stdout", "text": [ "\n", "epoch 50, minibatch 187/537, validation_error 3.626866 %" ] }, { "output_type": "stream", "stream": "stdout", "text": [ "\n", "epoch 51, minibatch 150/537, validation_error 3.656716 %" ] }, { "output_type": "stream", "stream": "stdout", "text": [ "\n", "epoch 52, minibatch 113/537, validation_error 3.492537 %" ] }, { "output_type": "stream", "stream": "stdout", "text": [ "\n", "epoch 53, minibatch 76/537, validation_error 3.641791 %" ] }, { "output_type": "stream", "stream": "stdout", "text": [ "\n", "epoch 54, minibatch 39/537, validation_error 3.776119 %" ] }, { "output_type": "stream", "stream": "stdout", "text": [ "\n", "epoch 55, minibatch 2/537, validation_error 3.507463 %" ] }, { "output_type": "stream", "stream": "stdout", "text": [ "\n", "epoch 55, minibatch 502/537, validation_error 3.626866 %" ] }, { "output_type": "stream", "stream": "stdout", "text": [ "\n", "epoch 56, minibatch 465/537, validation_error 3.582090 %" ] }, { "output_type": "stream", "stream": "stdout", "text": [ "\n", "epoch 57, minibatch 428/537, validation_error 3.641791 %" ] }, { "output_type": "stream", "stream": "stdout", "text": [ "\n", "epoch 58, minibatch 391/537, validation_error 3.522388 %" ] }, { "output_type": "stream", "stream": "stdout", "text": [ "\n", "epoch 59, minibatch 354/537, validation_error 3.507463 %" ] }, { "output_type": "stream", "stream": "stdout", "text": [ "\n", "epoch 60, minibatch 317/537, validation_error 3.626866 %" ] }, { "output_type": "stream", "stream": "stdout", "text": [ "\n", "epoch 61, minibatch 280/537, validation_error 3.462687 %" ] }, { "output_type": "stream", "stream": "stdout", "text": [ "\n", "epoch 62, minibatch 243/537, validation_error 3.582090 %" ] }, { "output_type": "stream", "stream": "stdout", "text": [ "\n", "epoch 63, minibatch 206/537, validation_error 3.582090 %" ] }, { "output_type": "stream", "stream": "stdout", "text": [ "\n", "epoch 64, minibatch 169/537, validation_error 3.552239 %" ] }, { "output_type": "stream", "stream": "stdout", "text": [ "\n", "epoch 65, minibatch 132/537, validation_error 3.761194 %" ] }, { "output_type": "stream", "stream": "stdout", "text": [ "\n", "epoch 66, minibatch 95/537, validation_error 3.492537 %" ] }, { "output_type": "stream", "stream": "stdout", "text": [ "\n", "epoch 67, minibatch 58/537, validation_error 3.671642 %" ] }, { "output_type": "stream", "stream": "stdout", "text": [ "\n", "epoch 68, minibatch 21/537, validation_error 3.611940 %" ] }, { "output_type": "stream", "stream": "stdout", "text": [ "\n", "epoch 68, minibatch 521/537, validation_error 3.552239 %" ] }, { "output_type": "stream", "stream": "stdout", "text": [ "\n", "epoch 69, minibatch 484/537, validation_error 3.582090 %" ] }, { "output_type": "stream", "stream": "stdout", "text": [ "\n", "epoch 70, minibatch 447/537, validation_error 3.462687 %" ] }, { "output_type": "stream", "stream": "stdout", "text": [ "\n", "epoch 71, minibatch 410/537, validation_error 3.507463 %" ] }, { "output_type": "stream", "stream": "stdout", "text": [ "\n", "epoch 72, minibatch 373/537, validation_error 3.567164 %" ] }, { "output_type": "stream", "stream": "stdout", "text": [ "\n", "epoch 73, minibatch 336/537, validation_error 3.552239 %" ] }, { "output_type": "stream", "stream": "stdout", "text": [ "\n", "epoch 74, minibatch 299/537, validation_error 3.582090 %" ] }, { "output_type": "stream", "stream": "stdout", "text": [ "\n", "epoch 75, minibatch 262/537, validation_error 3.402985 %" ] }, { "output_type": "stream", "stream": "stdout", "text": [ "\n", "epoch 76, minibatch 225/537, validation_error 3.477612 %" ] }, { "output_type": "stream", "stream": "stdout", "text": [ "\n", "epoch 77, minibatch 188/537, validation_error 3.402985 %" ] }, { "output_type": "stream", "stream": "stdout", "text": [ "\n", "epoch 78, minibatch 151/537, validation_error 3.597015 %" ] }, { "output_type": "stream", "stream": "stdout", "text": [ "\n", "epoch 79, minibatch 114/537, validation_error 3.507463 %" ] }, { "output_type": "stream", "stream": "stdout", "text": [ "\n", "epoch 80, minibatch 77/537, validation_error 3.492537 %" ] }, { "output_type": "stream", "stream": "stdout", "text": [ "\n", "epoch 81, minibatch 40/537, validation_error 3.597015 %" ] }, { "output_type": "stream", "stream": "stdout", "text": [ "\n", "epoch 82, minibatch 3/537, validation_error 3.507463 %" ] }, { "output_type": "stream", "stream": "stdout", "text": [ "\n", "epoch 82, minibatch 503/537, validation_error 3.552239 %" ] }, { "output_type": "stream", "stream": "stdout", "text": [ "\n", "epoch 83, minibatch 466/537, validation_error 3.582090 %" ] }, { "output_type": "stream", "stream": "stdout", "text": [ "\n", "epoch 84, minibatch 429/537, validation_error 3.477612 %" ] }, { "output_type": "stream", "stream": "stdout", "text": [ "\n", "epoch 85, minibatch 392/537, validation_error 3.507463 %" ] }, { "output_type": "stream", "stream": "stdout", "text": [ "\n", "epoch 86, minibatch 355/537, validation_error 3.522388 %" ] }, { "output_type": "stream", "stream": "stdout", "text": [ "\n", "epoch 87, minibatch 318/537, validation_error 3.492537 %" ] }, { "output_type": "stream", "stream": "stdout", "text": [ "\n", "epoch 88, minibatch 281/537, validation_error 3.477612 %" ] }, { "output_type": "stream", "stream": "stdout", "text": [ "\n", "epoch 89, minibatch 244/537, validation_error 3.507463 %" ] }, { "output_type": "stream", "stream": "stdout", "text": [ "\n", "epoch 90, minibatch 207/537, validation_error 3.477612 %" ] }, { "output_type": "stream", "stream": "stdout", "text": [ "\n", "epoch 91, minibatch 170/537, validation_error 3.417910 %" ] }, { "output_type": "stream", "stream": "stdout", "text": [ "\n", "epoch 92, minibatch 133/537, validation_error 3.537313 %" ] }, { "output_type": "stream", "stream": "stdout", "text": [ "\n", "epoch 93, minibatch 96/537, validation_error 3.432836 %" ] }, { "output_type": "stream", "stream": "stdout", "text": [ "\n", "epoch 94, minibatch 59/537, validation_error 3.507463 %" ] }, { "output_type": "stream", "stream": "stdout", "text": [ "\n", "epoch 95, minibatch 22/537, validation_error 3.537313 %" ] }, { "output_type": "stream", "stream": "stdout", "text": [ "\n", "epoch 95, minibatch 522/537, validation_error 3.507463 %" ] }, { "output_type": "stream", "stream": "stdout", "text": [ "\n", "epoch 96, minibatch 485/537, validation_error 3.507463 %" ] }, { "output_type": "stream", "stream": "stdout", "text": [ "\n", "epoch 97, minibatch 448/537, validation_error 3.328358 %" ] }, { "output_type": "stream", "stream": "stdout", "text": [ "\n", "epoch 98, minibatch 411/537, validation_error 3.507463 %" ] }, { "output_type": "stream", "stream": "stdout", "text": [ "\n", "epoch 99, minibatch 374/537, validation_error 3.522388 %" ] }, { "output_type": "stream", "stream": "stdout", "text": [ "\n", "epoch 100, minibatch 337/537, validation_error 3.507463 %" ] }, { "output_type": "stream", "stream": "stdout", "text": [ "\n", "optimization complete with best validation error: 3.328358 %" ] }, { "output_type": "stream", "stream": "stdout", "text": [ "\n", "0.992857142857" ] }, { "output_type": "stream", "stream": "stdout", "text": [ "\n", "0.968571428571" ] }, { "output_type": "stream", "stream": "stdout", "text": [ "\n" ] } ], "prompt_number": 10 }, { "cell_type": "code", "collapsed": false, "input": [], "language": "python", "metadata": {}, "outputs": [] }, { "cell_type": "code", "collapsed": false, "input": [ "## Denoising Auto Encoder\n", "reload(theanoml2)\n", "reload(theanoml2.lenet)\n", "reload(theanoml2.formula)\n", "reload(theanoml2.auto_encoder)\n", "d_auto_encoder = theanoml2.auto_encoder.DenoisingAutoEncoder(n_hidden = 500, corruption_level = 0.1, n_epochs=5)\n", "d_auto_encoder.partial_fit(X)\n", "print d_auto_encoder.transform(X).shape" ], "language": "python", "metadata": {}, "outputs": [ { "output_type": "stream", "stream": "stdout", "text": [ "training epoch 0, train cost -4661960859.348659\n", "training epoch 1, train cost -14022858680.161507" ] }, { "output_type": "stream", "stream": "stdout", "text": [ "\n", "training epoch 2, train cost -23384465235.135036" ] }, { "output_type": "stream", "stream": "stdout", "text": [ "\n", "training epoch 3, train cost -32746071790.105610" ] }, { "output_type": "stream", "stream": "stdout", "text": [ "\n", "training epoch 4, train cost -42107678345.075310" ] }, { "output_type": "stream", "stream": "stdout", "text": [ "\n", "optimization complete with best train cost: -42107678345.075310\n", "(42000, 500)" ] }, { "output_type": "stream", "stream": "stdout", "text": [ "\n" ] } ], "prompt_number": 4 }, { "cell_type": "code", "collapsed": false, "input": [ "reload(theanoml2)\n", "reload(theanoml2.lenet)\n", "reload(theanoml2.formula)\n", "reload(theanoml2.auto_encoder)\n", "c_auto_encoder = theanoml2.auto_encoder.ContractiveAutoEncoder(n_hidden = 500, \n", " contraction_level = 0.1, n_epochs=5)\n", "c_auto_encoder.partial_fit(X[:1000])\n", "print c_auto_encoder.transform(X[:1000]).shape" ], "language": "python", "metadata": {}, "outputs": [ { "output_type": "stream", "stream": "stdout", "text": [ "training epoch 0, train cost -103757114.201847\n", "training epoch 1, train cost -334109909.187471" ] }, { "output_type": "stream", "stream": "stdout", "text": [ "\n", "training epoch 2, train cost -564847972.077478" ] }, { "output_type": "stream", "stream": "stdout", "text": [ "\n", "training epoch 3, train cost -795586036.173563" ] }, { "output_type": "stream", "stream": "stdout", "text": [ "\n", "training epoch 4, train cost -1026324100.134109" ] }, { "output_type": "stream", "stream": "stdout", "text": [ "\n", "optimization complete with best train cost: -1026324100.134109\n", "(1000, 500)" ] }, { "output_type": "stream", "stream": "stdout", "text": [ "\n" ] } ], "prompt_number": 5 }, { "cell_type": "code", "collapsed": false, "input": [ "## LeNet classification\n", "import theanoml2\n", "reload(theanoml2)\n", "reload(theanoml2.lenet)\n", "reload(theanoml2.formula)\n", "lenet = theanoml2.lenet.LeNetClassifier(n_classes = len(classes), image_size = (28, 28), \n", " n_hidden = 500, n_epochs = 5,)\n", "lenet.partial_fit(train_X, train_y)\n", "print lenet.score(train_X[:50], train_y[:50])\n", "print np.mean([lenet.score(validation_X[i*50:(i+1)*50], validation_y[i*50:(i+1)*50])\n", " for i in xrange(validation_X.shape[0] / 50)])" ], "language": "python", "metadata": {}, "outputs": [ { "output_type": "stream", "stream": "stdout", "text": [ "epoch 1, minibatch 500/537, validation_error 7.686567 %\n", "epoch 2, minibatch 463/537, validation_error 5.298507 %" ] }, { "output_type": "stream", "stream": "stdout", "text": [ "\n", "epoch 3, minibatch 426/537, validation_error 4.223881 %" ] }, { "output_type": "stream", "stream": "stdout", "text": [ "\n", "epoch 4, minibatch 389/537, validation_error 3.746269 %" ] }, { "output_type": "stream", "stream": "stdout", "text": [ "\n", "epoch 5, minibatch 352/537, validation_error 3.238806 %" ] }, { "output_type": "stream", "stream": "stdout", "text": [ "\n", "optimization complete with best validation error: 3.238806 %" ] }, { "output_type": "stream", "stream": "stdout", "text": [ "\n", "0.94" ] }, { "output_type": "stream", "stream": "stdout", "text": [ "\n", "0.966071428571" ] }, { "output_type": "stream", "stream": "stdout", "text": [ "\n" ] } ], "prompt_number": 4 }, { "cell_type": "code", "collapsed": false, "input": [ "v_valid_X = theanoml2.formula.share_data(validation_X)\n", "print v_valid_X.eval().shape\n", "print lenet.batch_size, lenet.image_size\n", "batch_size = 8400\n", "layer0_input = v_valid_X.reshape((batch_size, 1, lenet.image_size[0], lenet.image_size[1]))\n", "layer0_input.eval()" ], "language": "python", "metadata": {}, "outputs": [] }, { "cell_type": "code", "collapsed": false, "input": [ "## MLP classification\n", "reload(theanoml2.mlp)\n", "reload(theanoml2.formula)\n", "mlp = theanoml2.mlp.MLPClassifier(n_classes = len(classes), n_hidden = 500, n_epochs=5)\n", "mlp.partial_fit(train_X, train_y)\n", "print mlp.score(train_X, train_y)\n", "mlp.partial_fit(train_X, train_y)\n", "print mlp.score(train_X, train_y)\n", "print mlp.score(validation_X, validation_y)" ], "language": "python", "metadata": {}, "outputs": [ { "output_type": "stream", "stream": "stdout", "text": [ "epoch 1, minibatch 500/537, validation_error 10.194030 %\n", "epoch 2, minibatch 463/537, validation_error 8.820896 %" ] }, { "output_type": "stream", "stream": "stdout", "text": [ "\n", "epoch 3, minibatch 426/537, validation_error 8.000000 %" ] }, { "output_type": "stream", "stream": "stdout", "text": [ "\n", "epoch 4, minibatch 389/537, validation_error 7.955224 %" ] }, { "output_type": "stream", "stream": "stdout", "text": [ "\n", "epoch 5, minibatch 352/537, validation_error 7.925373 %" ] }, { "output_type": "stream", "stream": "stdout", "text": [ "\n", "optimization complete with best validation error: 7.925373 %" ] }, { "output_type": "stream", "stream": "stdout", "text": [ "\n", "0.92994047619" ] }, { "output_type": "stream", "stream": "stdout", "text": [ "\n", "epoch 1, minibatch 500/537, validation_error 7.432836 %" ] }, { "output_type": "stream", "stream": "stdout", "text": [ "\n", "epoch 2, minibatch 463/537, validation_error 7.000000 %" ] }, { "output_type": "stream", "stream": "stdout", "text": [ "\n", "epoch 3, minibatch 426/537, validation_error 6.716418 %" ] }, { "output_type": "stream", "stream": "stdout", "text": [ "\n", "epoch 4, minibatch 389/537, validation_error 6.626866 %" ] }, { "output_type": "stream", "stream": "stdout", "text": [ "\n", "epoch 5, minibatch 352/537, validation_error 6.567164 %" ] }, { "output_type": "stream", "stream": "stdout", "text": [ "\n", "optimization complete with best validation error: 6.567164 %" ] }, { "output_type": "stream", "stream": "stdout", "text": [ "\n", "0.942708333333" ] }, { "output_type": "stream", "stream": "stdout", "text": [ "\n", "0.932857142857" ] }, { "output_type": "stream", "stream": "stdout", "text": [ "\n" ] } ], "prompt_number": 18 }, { "cell_type": "code", "collapsed": false, "input": [ "## Use Logistic Regression Wrapper \n", "reload(theanoml2)\n", "reload(theanoml2.formula)\n", "reload(theanoml2.linear_model)\n", "reload(theanoml2.optimize)\n", "lgr_model = theanoml2.linear_model.LogisticRegression(n_classes = len(classes), n_epochs = 50)\n", "lgr_model.partial_fit(train_X, train_y)\n", "#print lgr_model.predict(train_X)\n", "print lgr_model.score(train_X, train_y)\n", "print lgr_model.score(validation_X, validation_y)" ], "language": "python", "metadata": {}, "outputs": [ { "output_type": "stream", "stream": "stdout", "text": [ "epoch 1, minibatch 500/537, validation_error 16.298507 %\n", "epoch 2, minibatch 463/537, validation_error 13.059701 %" ] }, { "output_type": "stream", "stream": "stdout", "text": [ "\n", "epoch 3, minibatch 426/537, validation_error 10.000000 %" ] }, { "output_type": "stream", "stream": "stdout", "text": [ "\n", "epoch 4, minibatch 389/537, validation_error 12.985075 %" ] }, { "output_type": "stream", "stream": "stdout", "text": [ "\n", "epoch 5, minibatch 352/537, validation_error 13.208955 %" ] }, { "output_type": "stream", "stream": "stdout", "text": [ "\n", "epoch 6, minibatch 315/537, validation_error 10.850746 %" ] }, { "output_type": "stream", "stream": "stdout", "text": [ "\n", "epoch 7, minibatch 278/537, validation_error 14.119403 %" ] }, { "output_type": "stream", "stream": "stdout", "text": [ "\n", "epoch 8, minibatch 241/537, validation_error 13.223881 %" ] }, { "output_type": "stream", "stream": "stdout", "text": [ "\n", "epoch 9, minibatch 204/537, validation_error 10.522388 %" ] }, { "output_type": "stream", "stream": "stdout", "text": [ "\n", "epoch 10, minibatch 167/537, validation_error 10.104478 %" ] }, { "output_type": "stream", "stream": "stdout", "text": [ "\n", "epoch 11, minibatch 130/537, validation_error 10.417910 %" ] }, { "output_type": "stream", "stream": "stdout", "text": [ "\n", "epoch 12, minibatch 93/537, validation_error 10.477612 %" ] }, { "output_type": "stream", "stream": "stdout", "text": [ "\n", "epoch 13, minibatch 56/537, validation_error 11.000000 %" ] }, { "output_type": "stream", "stream": "stdout", "text": [ "\n", "epoch 14, minibatch 19/537, validation_error 10.835821 %" ] }, { "output_type": "stream", "stream": "stdout", "text": [ "\n", "epoch 14, minibatch 519/537, validation_error 13.791045 %" ] }, { "output_type": "stream", "stream": "stdout", "text": [ "\n", "epoch 15, minibatch 482/537, validation_error 11.746269 %" ] }, { "output_type": "stream", "stream": "stdout", "text": [ "\n", "epoch 16, minibatch 445/537, validation_error 11.970149 %" ] }, { "output_type": "stream", "stream": "stdout", "text": [ "\n", "epoch 17, minibatch 408/537, validation_error 12.910448 %" ] }, { "output_type": "stream", "stream": "stdout", "text": [ "\n", "epoch 18, minibatch 371/537, validation_error 21.910448 %" ] }, { "output_type": "stream", "stream": "stdout", "text": [ "\n", "epoch 19, minibatch 334/537, validation_error 12.791045 %" ] }, { "output_type": "stream", "stream": "stdout", "text": [ "\n", "epoch 20, minibatch 297/537, validation_error 11.029851 %" ] }, { "output_type": "stream", "stream": "stdout", "text": [ "\n", "epoch 21, minibatch 260/537, validation_error 11.537313 %" ] }, { "output_type": "stream", "stream": "stdout", "text": [ "\n", "epoch 22, minibatch 223/537, validation_error 12.268657 %" ] }, { "output_type": "stream", "stream": "stdout", "text": [ "\n", "epoch 23, minibatch 186/537, validation_error 14.567164 %" ] }, { "output_type": "stream", "stream": "stdout", "text": [ "\n", "epoch 24, minibatch 149/537, validation_error 10.014925 %" ] }, { "output_type": "stream", "stream": "stdout", "text": [ "\n", "epoch 25, minibatch 112/537, validation_error 12.343284 %" ] }, { "output_type": "stream", "stream": "stdout", "text": [ "\n", "epoch 26, minibatch 75/537, validation_error 11.477612 %" ] }, { "output_type": "stream", "stream": "stdout", "text": [ "\n", "epoch 27, minibatch 38/537, validation_error 11.029851 %" ] }, { "output_type": "stream", "stream": "stdout", "text": [ "\n", "epoch 28, minibatch 1/537, validation_error 11.567164 %" ] }, { "output_type": "stream", "stream": "stdout", "text": [ "\n", "epoch 28, minibatch 501/537, validation_error 14.253731 %" ] }, { "output_type": "stream", "stream": "stdout", "text": [ "\n", "epoch 29, minibatch 464/537, validation_error 13.194030 %" ] }, { "output_type": "stream", "stream": "stdout", "text": [ "\n", "epoch 30, minibatch 427/537, validation_error 11.238806 %" ] }, { "output_type": "stream", "stream": "stdout", "text": [ "\n", "epoch 31, minibatch 390/537, validation_error 11.388060 %" ] }, { "output_type": "stream", "stream": "stdout", "text": [ "\n", "epoch 32, minibatch 353/537, validation_error 13.432836 %" ] }, { "output_type": "stream", "stream": "stdout", "text": [ "\n", "epoch 33, minibatch 316/537, validation_error 11.880597 %" ] }, { "output_type": "stream", "stream": "stdout", "text": [ "\n", "epoch 34, minibatch 279/537, validation_error 10.940299 %" ] }, { "output_type": "stream", "stream": "stdout", "text": [ "\n", "epoch 35, minibatch 242/537, validation_error 12.537313 %" ] }, { "output_type": "stream", "stream": "stdout", "text": [ "\n", "epoch 36, minibatch 205/537, validation_error 10.925373 %" ] }, { "output_type": "stream", "stream": "stdout", "text": [ "\n", "epoch 37, minibatch 168/537, validation_error 10.089552 %" ] }, { "output_type": "stream", "stream": "stdout", "text": [ "\n", "epoch 38, minibatch 131/537, validation_error 11.477612 %" ] }, { "output_type": "stream", "stream": "stdout", "text": [ "\n", "epoch 39, minibatch 94/537, validation_error 12.283582 %" ] }, { "output_type": "stream", "stream": "stdout", "text": [ "\n", "epoch 40, minibatch 57/537, validation_error 11.522388 %" ] }, { "output_type": "stream", "stream": "stdout", "text": [ "\n", "epoch 41, minibatch 20/537, validation_error 11.253731 %" ] }, { "output_type": "stream", "stream": "stdout", "text": [ "\n", "epoch 41, minibatch 520/537, validation_error 12.268657 %" ] }, { "output_type": "stream", "stream": "stdout", "text": [ "\n", "epoch 42, minibatch 483/537, validation_error 14.388060 %" ] }, { "output_type": "stream", "stream": "stdout", "text": [ "\n", "epoch 43, minibatch 446/537, validation_error 13.597015 %" ] }, { "output_type": "stream", "stream": "stdout", "text": [ "\n", "epoch 44, minibatch 409/537, validation_error 12.119403 %" ] }, { "output_type": "stream", "stream": "stdout", "text": [ "\n", "epoch 45, minibatch 372/537, validation_error 12.238806 %" ] }, { "output_type": "stream", "stream": "stdout", "text": [ "\n", "epoch 46, minibatch 335/537, validation_error 12.865672 %" ] }, { "output_type": "stream", "stream": "stdout", "text": [ "\n", "epoch 47, minibatch 298/537, validation_error 13.761194 %" ] }, { "output_type": "stream", "stream": "stdout", "text": [ "\n", "epoch 48, minibatch 261/537, validation_error 17.865672 %" ] }, { "output_type": "stream", "stream": "stdout", "text": [ "\n", "epoch 49, minibatch 224/537, validation_error 12.865672 %" ] }, { "output_type": "stream", "stream": "stdout", "text": [ "\n", "epoch 50, minibatch 187/537, validation_error 17.283582 %" ] }, { "output_type": "stream", "stream": "stdout", "text": [ "\n", "optimization complete with best validation error: 10.000000 %" ] }, { "output_type": "stream", "stream": "stdout", "text": [ "\n", "0.903125" ] }, { "output_type": "stream", "stream": "stdout", "text": [ "\n", "0.88880952381" ] }, { "output_type": "stream", "stream": "stdout", "text": [ "\n" ] } ], "prompt_number": 5 }, { "cell_type": "code", "collapsed": false, "input": [ "## Linear Regression Model\n", "import theanoml2\n", "reload(theanoml2)\n", "reload(theanoml2.linear_model)\n", "reload(theanoml2.formula)\n", "from sklearn.datasets import load_diabetes\n", "from sklearn.cross_validation import train_test_split\n", "diabetes = load_diabetes()\n", "X, y = diabetes.data, diabetes.target\n", "X = np.random.uniform(size = (1000, 3))\n", "y = X[:,0]*2. + X[:, 1] * 5. + 1.\n", "print X.shape, y.shape\n", "train_X, validation_X, train_y, validation_y = train_test_split(X, y, test_size = 0.2)\n", "lr_model = theanoml2.linear_model.LinearRegression(n_epochs = 10, batch_size=10)\n", "lr_model.partial_fit(train_X, train_y)\n", "print lr_model.score(train_X, train_y)\n", "print lr_model.score(validation_X, validation_y)\n", "print map(lambda v: v.eval(), lr_model.formula_.params)" ], "language": "python", "metadata": {}, "outputs": [ { "output_type": "stream", "stream": "stdout", "text": [ "(1000, 3) (1000,)\n", "epoch 1, minibatch 64/64, validation_error 91.202448 %" ] }, { "output_type": "stream", "stream": "stdout", "text": [ "\n", "epoch 2, minibatch 64/64, validation_error 76.554709 %\n", "epoch 3, minibatch 64/64, validation_error 68.960607 %\n", "epoch 4, minibatch 64/64, validation_error 62.231556 %\n", "epoch 5, minibatch 64/64, validation_error 56.194271 %\n", "epoch 6, minibatch 64/64, validation_error 50.775172 %\n", "epoch 7, minibatch 64/64, validation_error 45.913820 %\n", "epoch 8, minibatch 64/64, validation_error 41.565633 %\n", "epoch 9, minibatch 64/64, validation_error 37.656044 %" ] }, { "output_type": "stream", "stream": "stdout", "text": [ "\n", "epoch 10, minibatch 64/64, validation_error 34.142120 %\n", "optimization complete with best validation error: 34.142120 %\n", "0.928087113353" ] }, { "output_type": "stream", "stream": "stdout", "text": [ "\n", "0.93059764046\n", "[array([ 1.56063239, 3.64438407, 0.17407062]), array(1.8625258492463739)]" ] }, { "output_type": "stream", "stream": "stdout", "text": [ "\n" ] } ], "prompt_number": 5 }, { "cell_type": "code", "collapsed": false, "input": [ "import theanoml2\n", "reload(theanoml2.formula)\n", "reload(theanoml2.optimize)\n", "import cPickle\n", "import numpy as np\n", "from sklearn.cross_validation import train_test_split\n", "from sklearn.preprocessing import MinMaxScaler\n", "from functools import partial\n", "X, y = cPickle.load(open('data/blackbox.pkl', 'rb'))\n", "X = MinMaxScaler().fit_transform(X)\n", "import features\n", "from IPython.parallel import Client\n", "n_features = X.shape[1]\n", "tri = features.TriKmeansFeatures(n_clusters = 20, sparse_result=False,\n", " feat_patches = features.bootstrap_seqs(range(n_features), \n", " n_iter = 500, subsize=20), \n", " client = Client(), )\n", "X = tri.fit_transform(X)\n", "#print np.min(X), np.max(X)\n", "y = y - 1\n", "classes = np.unique(y)\n", "print X.shape, y.shape\n", "train_X, validation_X, train_y, validation_y = train_test_split(X, y, test_size = 0.2)" ], "language": "python", "metadata": {}, "outputs": [ { "output_type": "stream", "stream": "stdout", "text": [ " 4/4 tasks finished after 47 s" ] }, { "output_type": "stream", "stream": "stdout", "text": [ "\n", "done\n", "(1000, 10000)" ] }, { "output_type": "stream", "stream": "stdout", "text": [ " (1000,)\n" ] } ], "prompt_number": 33 }, { "cell_type": "code", "collapsed": false, "input": [ "## stacked auto encoders on blackbox data\n", "reload(theanoml2)\n", "reload(theanoml2.lenet)\n", "reload(theanoml2.formula)\n", "reload(theanoml2.auto_encoder)\n", "sda = theanoml2.auto_encoder.SdAClassifier(len(classes), pretrain_epochs = 30, pretrain_batch_size=1,\n", "finetune_learning_rate=0.01,\n", " finetune_epochs = 100, hidden_layer_sizes = (3000, 1000, 500, 100), \n", " corruption_levels = (0.1, 0.2, 0.3, 0.3))\n", "sda.formula_ = sda._create_formula(train_X, train_y)\n", "sda.pretrain(X) ## train on whole X\n", "sda.fit(train_X, train_y)\n", "print sda.score(train_X, train_y)\n", "print sda.score(validation_X, validation_y)\n", "print sda.transform(X).shape" ], "language": "python", "metadata": {}, "outputs": [ { "output_type": "stream", "stream": "stdout", "text": [ "training epoch 0, train cost 3445.677260\n", "training epoch 1, train cost 3084.838276" ] }, { "output_type": "stream", "stream": "stdout", "text": [ "\n", "training epoch 2, train cost 2952.708848" ] }, { "output_type": "stream", "stream": "stdout", "text": [ "\n", "training epoch 3, train cost 2875.915584" ] }, { "output_type": "stream", "stream": "stdout", "text": [ "\n", "training epoch 4, train cost 2826.343480" ] }, { "output_type": "stream", "stream": "stdout", "text": [ "\n", "training epoch 5, train cost 2792.545099" ] }, { "output_type": "stream", "stream": "stdout", "text": [ "\n", "training epoch 6, train cost 2768.712079" ] }, { "output_type": "stream", "stream": "stdout", "text": [ "\n", "training epoch 7, train cost 2750.969216" ] }, { "output_type": "stream", "stream": "stdout", "text": [ "\n", "training epoch 8, train cost 2737.301677" ] }, { "output_type": "stream", "stream": "stdout", "text": [ "\n", "training epoch 9, train cost 2726.588094" ] }, { "output_type": "stream", "stream": "stdout", "text": [ "\n", "training epoch 10, train cost 2717.791846" ] }, { "output_type": "stream", "stream": "stdout", "text": [ "\n", "training epoch 11, train cost 2710.277212" ] }, { "output_type": "stream", "stream": "stdout", "text": [ "\n", "training epoch 12, train cost 2703.995309" ] }, { "output_type": "stream", "stream": "stdout", "text": [ "\n", "training epoch 13, train cost 2698.469449" ] }, { "output_type": "stream", "stream": "stdout", "text": [ "\n", "training epoch 14, train cost 2693.468839" ] }, { "output_type": "stream", "stream": "stdout", "text": [ "\n", "training epoch 15, train cost 2689.044113" ] }, { "output_type": "stream", "stream": "stdout", "text": [ "\n", "training epoch 16, train cost 2684.916942" ] }, { "output_type": "stream", "stream": "stdout", "text": [ "\n", "training epoch 17, train cost 2681.083811" ] }, { "output_type": "stream", "stream": "stdout", "text": [ "\n", "training epoch 18, train cost 2677.633932" ] }, { "output_type": "stream", "stream": "stdout", "text": [ "\n", "training epoch 19, train cost 2674.365541" ] }, { "output_type": "stream", "stream": "stdout", "text": [ "\n", "training epoch 20, train cost 2671.309701" ] }, { "output_type": "stream", "stream": "stdout", "text": [ "\n", "training epoch 21, train cost 2668.376969" ] }, { "output_type": "stream", "stream": "stdout", "text": [ "\n", "training epoch 22, train cost 2665.713755" ] }, { "output_type": "stream", "stream": "stdout", "text": [ "\n", "training epoch 23, train cost 2663.069842" ] }, { "output_type": "stream", "stream": "stdout", "text": [ "\n", "training epoch 24, train cost 2660.519091" ] }, { "output_type": "stream", "stream": "stdout", "text": [ "\n", "training epoch 25, train cost 2658.184363" ] }, { "output_type": "stream", "stream": "stdout", "text": [ "\n", "training epoch 26, train cost 2655.978615" ] }, { "output_type": "stream", "stream": "stdout", "text": [ "\n", "training epoch 27, train cost 2653.744477" ] }, { "output_type": "stream", "stream": "stdout", "text": [ "\n", "training epoch 28, train cost 2651.533174" ] }, { "output_type": "stream", "stream": "stdout", "text": [ "\n", "training epoch 29, train cost 2649.620674" ] }, { "output_type": "stream", "stream": "stdout", "text": [ "\n", "optimization complete with best train cost: 2649.620674\n", "training epoch 0, train cost 1491.487965" ] }, { "output_type": "stream", "stream": "stdout", "text": [ "\n", "training epoch 1, train cost 1374.793743" ] }, { "output_type": "stream", "stream": "stdout", "text": [ "\n", "training epoch 2, train cost 1343.583601" ] }, { "output_type": "stream", "stream": "stdout", "text": [ "\n", "training epoch 3, train cost 1326.428579" ] }, { "output_type": "stream", "stream": "stdout", "text": [ "\n", "training epoch 4, train cost 1314.783143" ] }, { "output_type": "stream", "stream": "stdout", "text": [ "\n", "training epoch 5, train cost 1306.003855" ] }, { "output_type": "stream", "stream": "stdout", "text": [ "\n", "training epoch 6, train cost 1299.006819" ] }, { "output_type": "stream", "stream": "stdout", "text": [ "\n", "training epoch 7, train cost 1292.962288" ] }, { "output_type": "stream", "stream": "stdout", "text": [ "\n", "training epoch 8, train cost 1288.015381" ] }, { "output_type": "stream", "stream": "stdout", "text": [ "\n", "training epoch 9, train cost 1283.396309" ] }, { "output_type": "stream", "stream": "stdout", "text": [ "\n", "training epoch 10, train cost 1279.571486" ] }, { "output_type": "stream", "stream": "stdout", "text": [ "\n", "training epoch 11, train cost 1275.909117" ] }, { "output_type": "stream", "stream": "stdout", "text": [ "\n", "training epoch 12, train cost 1272.799425" ] }, { "output_type": "stream", "stream": "stdout", "text": [ "\n", "training epoch 13, train cost 1269.893933" ] }, { "output_type": "stream", "stream": "stdout", "text": [ "\n", "training epoch 14, train cost 1267.298578" ] }, { "output_type": "stream", "stream": "stdout", "text": [ "\n", "training epoch 15, train cost 1264.789091" ] }, { "output_type": "stream", "stream": "stdout", "text": [ "\n", "training epoch 16, train cost 1262.331509" ] }, { "output_type": "stream", "stream": "stdout", "text": [ "\n", "training epoch 17, train cost 1260.316512" ] }, { "output_type": "stream", "stream": "stdout", "text": [ "\n", "training epoch 18, train cost 1258.039868" ] }, { "output_type": "stream", "stream": "stdout", "text": [ "\n", "training epoch 19, train cost 1256.336931" ] }, { "output_type": "stream", "stream": "stdout", "text": [ "\n", "training epoch 20, train cost 1254.433314" ] }, { "output_type": "stream", "stream": "stdout", "text": [ "\n", "training epoch 21, train cost 1252.699207" ] }, { "output_type": "stream", "stream": "stdout", "text": [ "\n", "training epoch 22, train cost 1251.015170" ] }, { "output_type": "stream", "stream": "stdout", "text": [ "\n", "training epoch 23, train cost 1249.360617" ] }, { "output_type": "stream", "stream": "stdout", "text": [ "\n", "training epoch 24, train cost 1247.789795" ] }, { "output_type": "stream", "stream": "stdout", "text": [ "\n", "training epoch 25, train cost 1246.483422" ] }, { "output_type": "stream", "stream": "stdout", "text": [ "\n", "training epoch 26, train cost 1244.994625" ] }, { "output_type": "stream", "stream": "stdout", "text": [ "\n", "training epoch 27, train cost 1243.640561" ] }, { "output_type": "stream", "stream": "stdout", "text": [ "\n", "training epoch 28, train cost 1242.429469" ] }, { "output_type": "stream", "stream": "stdout", "text": [ "\n", "training epoch 29, train cost 1241.212573" ] }, { "output_type": "stream", "stream": "stdout", "text": [ "\n", "optimization complete with best train cost: 1241.212573\n", "training epoch 0, train cost 316.818868" ] }, { "output_type": "stream", "stream": "stdout", "text": [ "\n", "training epoch 1, train cost 249.854899" ] }, { "output_type": "stream", "stream": "stdout", "text": [ "\n", "training epoch 2, train cost 230.941835" ] }, { "output_type": "stream", "stream": "stdout", "text": [ "\n", "training epoch 3, train cost 219.916137" ] }, { "output_type": "stream", "stream": "stdout", "text": [ "\n", "training epoch 4, train cost 212.443484" ] }, { "output_type": "stream", "stream": "stdout", "text": [ "\n", "training epoch 5, train cost 207.302483" ] }, { "output_type": "stream", "stream": "stdout", "text": [ "\n", "training epoch 6, train cost 203.410199" ] }, { "output_type": "stream", "stream": "stdout", "text": [ "\n", "training epoch 7, train cost 199.959555" ] }, { "output_type": "stream", "stream": "stdout", "text": [ "\n", "training epoch 8, train cost 197.292084" ] }, { "output_type": "stream", "stream": "stdout", "text": [ "\n", "training epoch 9, train cost 195.270577" ] }, { "output_type": "stream", "stream": "stdout", "text": [ "\n", "training epoch 10, train cost 193.038864" ] }, { "output_type": "stream", "stream": "stdout", "text": [ "\n", "training epoch 11, train cost 191.157723" ] }, { "output_type": "stream", "stream": "stdout", "text": [ "\n", "training epoch 12, train cost 189.551793" ] }, { "output_type": "stream", "stream": "stdout", "text": [ "\n", "training epoch 13, train cost 187.935762" ] }, { "output_type": "stream", "stream": "stdout", "text": [ "\n", "training epoch 14, train cost 186.899146" ] }, { "output_type": "stream", "stream": "stdout", "text": [ "\n", "training epoch 15, train cost 185.333712" ] }, { "output_type": "stream", "stream": "stdout", "text": [ "\n", "training epoch 16, train cost 184.362900" ] }, { "output_type": "stream", "stream": "stdout", "text": [ "\n", "training epoch 17, train cost 183.259189" ] }, { "output_type": "stream", "stream": "stdout", "text": [ "\n", "training epoch 18, train cost 182.278092" ] }, { "output_type": "stream", "stream": "stdout", "text": [ "\n", "training epoch 19, train cost 181.393068" ] }, { "output_type": "stream", "stream": "stdout", "text": [ "\n", "training epoch 20, train cost 180.430494" ] }, { "output_type": "stream", "stream": "stdout", "text": [ "\n", "training epoch 21, train cost 179.944222" ] }, { "output_type": "stream", "stream": "stdout", "text": [ "\n", "training epoch 22, train cost 179.113612" ] }, { "output_type": "stream", "stream": "stdout", "text": [ "\n", "training epoch 23, train cost 178.383777" ] }, { "output_type": "stream", "stream": "stdout", "text": [ "\n", "training epoch 24, train cost 177.698206" ] }, { "output_type": "stream", "stream": "stdout", "text": [ "\n", "training epoch 25, train cost 177.067117" ] }, { "output_type": "stream", "stream": "stdout", "text": [ "\n", "training epoch 26, train cost 176.356915" ] }, { "output_type": "stream", "stream": "stdout", "text": [ "\n", "training epoch 27, train cost 175.855538" ] }, { "output_type": "stream", "stream": "stdout", "text": [ "\n", "training epoch 28, train cost 175.364912" ] }, { "output_type": "stream", "stream": "stdout", "text": [ "\n", "training epoch 29, train cost 174.895925" ] }, { "output_type": "stream", "stream": "stdout", "text": [ "\n", "optimization complete with best train cost: 174.895925\n", "training epoch 0, train cost 259.094671" ] }, { "output_type": "stream", "stream": "stdout", "text": [ "\n", "training epoch 1, train cost 205.865195" ] }, { "output_type": "stream", "stream": "stdout", "text": [ "\n", "training epoch 2, train cost 189.986593" ] }, { "output_type": "stream", "stream": "stdout", "text": [ "\n", "training epoch 3, train cost 179.742673" ] }, { "output_type": "stream", "stream": "stdout", "text": [ "\n", "training epoch 4, train cost 172.105566" ] }, { "output_type": "stream", "stream": "stdout", "text": [ "\n", "training epoch 5, train cost 166.146700" ] }, { "output_type": "stream", "stream": "stdout", "text": [ "\n", "training epoch 6, train cost 161.447090" ] }, { "output_type": "stream", "stream": "stdout", "text": [ "\n", "training epoch 7, train cost 157.553290" ] }, { "output_type": "stream", "stream": "stdout", "text": [ "\n", "training epoch 8, train cost 154.230319" ] }, { "output_type": "stream", "stream": "stdout", "text": [ "\n", "training epoch 9, train cost 151.560145" ] }, { "output_type": "stream", "stream": "stdout", "text": [ "\n", "training epoch 10, train cost 149.113159" ] }, { "output_type": "stream", "stream": "stdout", "text": [ "\n", "training epoch 11, train cost 147.074910" ] }, { "output_type": "stream", "stream": "stdout", "text": [ "\n", "training epoch 12, train cost 145.172795" ] }, { "output_type": "stream", "stream": "stdout", "text": [ "\n", "training epoch 13, train cost 143.722784" ] }, { "output_type": "stream", "stream": "stdout", "text": [ "\n", "training epoch 14, train cost 142.139697" ] }, { "output_type": "stream", "stream": "stdout", "text": [ "\n", "training epoch 15, train cost 140.969293" ] }, { "output_type": "stream", "stream": "stdout", "text": [ "\n", "training epoch 16, train cost 139.805747" ] }, { "output_type": "stream", "stream": "stdout", "text": [ "\n", "training epoch 17, train cost 138.560454" ] }, { "output_type": "stream", "stream": "stdout", "text": [ "\n", "training epoch 18, train cost 137.723139" ] }, { "output_type": "stream", "stream": "stdout", "text": [ "\n", "training epoch 19, train cost 136.705694" ] }, { "output_type": "stream", "stream": "stdout", "text": [ "\n", "training epoch 20, train cost 135.815412" ] }, { "output_type": "stream", "stream": "stdout", "text": [ "\n", "training epoch 21, train cost 135.184513" ] }, { "output_type": "stream", "stream": "stdout", "text": [ "\n", "training epoch 22, train cost 134.283806" ] }, { "output_type": "stream", "stream": "stdout", "text": [ "\n", "training epoch 23, train cost 133.508631" ] }, { "output_type": "stream", "stream": "stdout", "text": [ "\n", "training epoch 24, train cost 132.809381" ] }, { "output_type": "stream", "stream": "stdout", "text": [ "\n", "training epoch 25, train cost 132.420527" ] }, { "output_type": "stream", "stream": "stdout", "text": [ "\n", "training epoch 26, train cost 131.712802" ] }, { "output_type": "stream", "stream": "stdout", "text": [ "\n", "training epoch 27, train cost 131.015996" ] }, { "output_type": "stream", "stream": "stdout", "text": [ "\n", "training epoch 28, train cost 130.467072" ] }, { "output_type": "stream", "stream": "stdout", "text": [ "\n", "training epoch 29, train cost 130.076170" ] }, { "output_type": "stream", "stream": "stdout", "text": [ "\n", "optimization complete with best train cost: 130.076170\n", "epoch 1, minibatch 12/12, validation_error 74.000000 %" ] }, { "output_type": "stream", "stream": "stdout", "text": [ "\n", "epoch 2, minibatch 12/12, validation_error 74.000000 %" ] }, { "output_type": "stream", "stream": "stdout", "text": [ "\n", "epoch 3, minibatch 12/12, validation_error 74.000000 %" ] }, { "output_type": "stream", "stream": "stdout", "text": [ "\n", "epoch 4, minibatch 12/12, validation_error 74.000000 %" ] }, { "output_type": "stream", "stream": "stdout", "text": [ "\n", "epoch 5, minibatch 12/12, validation_error 74.000000 %" ] }, { "output_type": "stream", "stream": "stdout", "text": [ "\n", "epoch 6, minibatch 12/12, validation_error 74.000000 %" ] }, { "output_type": "stream", "stream": "stdout", "text": [ "\n", "epoch 7, minibatch 12/12, validation_error 74.000000 %" ] }, { "output_type": "stream", "stream": "stdout", "text": [ "\n", "epoch 8, minibatch 12/12, validation_error 74.000000 %" ] }, { "output_type": "stream", "stream": "stdout", "text": [ "\n", "epoch 9, minibatch 12/12, validation_error 74.000000 %" ] }, { "output_type": "stream", "stream": "stdout", "text": [ "\n", "epoch 10, minibatch 12/12, validation_error 74.000000 %" ] }, { "output_type": "stream", "stream": "stdout", "text": [ "\n", "epoch 11, minibatch 12/12, validation_error 74.000000 %" ] }, { "output_type": "stream", "stream": "stdout", "text": [ "\n", "epoch 12, minibatch 12/12, validation_error 74.000000 %" ] }, { "output_type": "stream", "stream": "stdout", "text": [ "\n", "epoch 13, minibatch 12/12, validation_error 74.000000 %" ] }, { "output_type": "stream", "stream": "stdout", "text": [ "\n", "epoch 14, minibatch 12/12, validation_error 74.000000 %" ] }, { "output_type": "stream", "stream": "stdout", "text": [ "\n", "epoch 15, minibatch 12/12, validation_error 74.000000 %" ] }, { "output_type": "stream", "stream": "stdout", "text": [ "\n", "epoch 16, minibatch 12/12, validation_error 74.000000 %" ] }, { "output_type": "stream", "stream": "stdout", "text": [ "\n", "epoch 17, minibatch 12/12, validation_error 74.000000 %" ] }, { "output_type": "stream", "stream": "stdout", "text": [ "\n", "epoch 18, minibatch 12/12, validation_error 74.000000 %" ] }, { "output_type": "stream", "stream": "stdout", "text": [ "\n", "epoch 19, minibatch 12/12, validation_error 74.000000 %" ] }, { "output_type": "stream", "stream": "stdout", "text": [ "\n", "epoch 20, minibatch 12/12, validation_error 74.000000 %" ] }, { "output_type": "stream", "stream": "stdout", "text": [ "\n", "epoch 21, minibatch 12/12, validation_error 74.000000 %" ] }, { "output_type": "stream", "stream": "stdout", "text": [ "\n", "epoch 22, minibatch 12/12, validation_error 74.000000 %" ] }, { "output_type": "stream", "stream": "stdout", "text": [ "\n", "epoch 23, minibatch 12/12, validation_error 74.000000 %" ] }, { "output_type": "stream", "stream": "stdout", "text": [ "\n", "epoch 24, minibatch 12/12, validation_error 74.000000 %" ] }, { "output_type": "stream", "stream": "stdout", "text": [ "\n", "epoch 25, minibatch 12/12, validation_error 74.000000 %" ] }, { "output_type": "stream", "stream": "stdout", "text": [ "\n", "epoch 26, minibatch 12/12, validation_error 74.000000 %" ] }, { "output_type": "stream", "stream": "stdout", "text": [ "\n", "epoch 27, minibatch 12/12, validation_error 74.000000 %" ] }, { "output_type": "stream", "stream": "stdout", "text": [ "\n", "epoch 28, minibatch 12/12, validation_error 74.000000 %" ] }, { "output_type": "stream", "stream": "stdout", "text": [ "\n", "epoch 29, minibatch 12/12, validation_error 74.000000 %" ] }, { "output_type": "stream", "stream": "stdout", "text": [ "\n", "epoch 30, minibatch 12/12, validation_error 74.000000 %" ] }, { "output_type": "stream", "stream": "stdout", "text": [ "\n", "epoch 31, minibatch 12/12, validation_error 74.000000 %" ] }, { "output_type": "stream", "stream": "stdout", "text": [ "\n", "epoch 32, minibatch 12/12, validation_error 74.000000 %" ] }, { "output_type": "stream", "stream": "stdout", "text": [ "\n", "epoch 33, minibatch 12/12, validation_error 74.000000 %" ] }, { "output_type": "stream", "stream": "stdout", "text": [ "\n", "epoch 34, minibatch 12/12, validation_error 74.000000 %" ] }, { "output_type": "stream", "stream": "stdout", "text": [ "\n", "epoch 35, minibatch 12/12, validation_error 74.000000 %" ] }, { "output_type": "stream", "stream": "stdout", "text": [ "\n", "epoch 36, minibatch 12/12, validation_error 74.000000 %" ] }, { "output_type": "stream", "stream": "stdout", "text": [ "\n", "epoch 37, minibatch 12/12, validation_error 74.000000 %" ] }, { "output_type": "stream", "stream": "stdout", "text": [ "\n", "epoch 38, minibatch 12/12, validation_error 74.000000 %" ] }, { "output_type": "stream", "stream": "stdout", "text": [ "\n", "epoch 39, minibatch 12/12, validation_error 74.000000 %" ] }, { "output_type": "stream", "stream": "stdout", "text": [ "\n", "epoch 40, minibatch 12/12, validation_error 73.333333 %" ] }, { "output_type": "stream", "stream": "stdout", "text": [ "\n", "epoch 41, minibatch 12/12, validation_error 73.333333 %" ] }, { "output_type": "stream", "stream": "stdout", "text": [ "\n", "epoch 42, minibatch 12/12, validation_error 73.333333 %" ] }, { "output_type": "stream", "stream": "stdout", "text": [ "\n", "epoch 43, minibatch 12/12, validation_error 72.000000 %" ] }, { "output_type": "stream", "stream": "stdout", "text": [ "\n", "epoch 44, minibatch 12/12, validation_error 73.333333 %" ] }, { "output_type": "stream", "stream": "stdout", "text": [ "\n", "epoch 45, minibatch 12/12, validation_error 73.333333 %" ] }, { "output_type": "stream", "stream": "stdout", "text": [ "\n", "epoch 46, minibatch 12/12, validation_error 73.333333 %" ] }, { "output_type": "stream", "stream": "stdout", "text": [ "\n", "epoch 47, minibatch 12/12, validation_error 74.000000 %" ] }, { "output_type": "stream", "stream": "stdout", "text": [ "\n", "epoch 48, minibatch 12/12, validation_error 75.333333 %" ] }, { "output_type": "stream", "stream": "stdout", "text": [ "\n", "epoch 49, minibatch 12/12, validation_error 76.000000 %" ] }, { "output_type": "stream", "stream": "stdout", "text": [ "\n", "epoch 50, minibatch 12/12, validation_error 76.000000 %" ] }, { "output_type": "stream", "stream": "stdout", "text": [ "\n", "epoch 51, minibatch 12/12, validation_error 76.000000 %" ] }, { "output_type": "stream", "stream": "stdout", "text": [ "\n", "epoch 52, minibatch 12/12, validation_error 76.000000 %" ] }, { "output_type": "stream", "stream": "stdout", "text": [ "\n", "epoch 53, minibatch 12/12, validation_error 76.666667 %" ] }, { "output_type": "stream", "stream": "stdout", "text": [ "\n", "epoch 54, minibatch 12/12, validation_error 76.666667 %" ] }, { "output_type": "stream", "stream": "stdout", "text": [ "\n", "epoch 55, minibatch 12/12, validation_error 77.333333 %" ] }, { "output_type": "stream", "stream": "stdout", "text": [ "\n", "epoch 56, minibatch 12/12, validation_error 77.333333 %" ] }, { "output_type": "stream", "stream": "stdout", "text": [ "\n", "epoch 57, minibatch 12/12, validation_error 76.666667 %" ] }, { "output_type": "stream", "stream": "stdout", "text": [ "\n", "epoch 58, minibatch 12/12, validation_error 76.666667 %" ] }, { "output_type": "stream", "stream": "stdout", "text": [ "\n", "epoch 59, minibatch 12/12, validation_error 76.666667 %" ] }, { "output_type": "stream", "stream": "stdout", "text": [ "\n", "epoch 60, minibatch 12/12, validation_error 76.666667 %" ] }, { "output_type": "stream", "stream": "stdout", "text": [ "\n", "epoch 61, minibatch 12/12, validation_error 77.333333 %" ] }, { "output_type": "stream", "stream": "stdout", "text": [ "\n", "epoch 62, minibatch 12/12, validation_error 77.333333 %" ] }, { "output_type": "stream", "stream": "stdout", "text": [ "\n", "epoch 63, minibatch 12/12, validation_error 77.333333 %" ] }, { "output_type": "stream", "stream": "stdout", "text": [ "\n", "epoch 64, minibatch 12/12, validation_error 77.333333 %" ] }, { "output_type": "stream", "stream": "stdout", "text": [ "\n", "epoch 65, minibatch 12/12, validation_error 77.333333 %" ] }, { "output_type": "stream", "stream": "stdout", "text": [ "\n", "epoch 66, minibatch 12/12, validation_error 77.333333 %" ] }, { "output_type": "stream", "stream": "stdout", "text": [ "\n", "epoch 67, minibatch 12/12, validation_error 78.000000 %" ] }, { "output_type": "stream", "stream": "stdout", "text": [ "\n", "epoch 68, minibatch 12/12, validation_error 78.666667 %" ] }, { "output_type": "stream", "stream": "stdout", "text": [ "\n", "epoch 69, minibatch 12/12, validation_error 78.666667 %" ] }, { "output_type": "stream", "stream": "stdout", "text": [ "\n", "epoch 70, minibatch 12/12, validation_error 78.666667 %" ] }, { "output_type": "stream", "stream": "stdout", "text": [ "\n", "epoch 71, minibatch 12/12, validation_error 78.666667 %" ] }, { "output_type": "stream", "stream": "stdout", "text": [ "\n", "epoch 72, minibatch 12/12, validation_error 78.666667 %" ] }, { "output_type": "stream", "stream": "stdout", "text": [ "\n", "epoch 73, minibatch 12/12, validation_error 78.666667 %" ] }, { "output_type": "stream", "stream": "stdout", "text": [ "\n", "epoch 74, minibatch 12/12, validation_error 78.666667 %" ] }, { "output_type": "stream", "stream": "stdout", "text": [ "\n", "epoch 75, minibatch 12/12, validation_error 78.666667 %" ] }, { "output_type": "stream", "stream": "stdout", "text": [ "\n", "epoch 76, minibatch 12/12, validation_error 78.666667 %" ] }, { "output_type": "stream", "stream": "stdout", "text": [ "\n", "epoch 77, minibatch 12/12, validation_error 78.666667 %" ] }, { "output_type": "stream", "stream": "stdout", "text": [ "\n", "epoch 78, minibatch 12/12, validation_error 78.666667 %" ] }, { "output_type": "stream", "stream": "stdout", "text": [ "\n", "epoch 79, minibatch 12/12, validation_error 78.666667 %" ] }, { "output_type": "stream", "stream": "stdout", "text": [ "\n", "epoch 80, minibatch 12/12, validation_error 78.666667 %" ] }, { "output_type": "stream", "stream": "stdout", "text": [ "\n", "epoch 81, minibatch 12/12, validation_error 78.666667 %" ] }, { "output_type": "stream", "stream": "stdout", "text": [ "\n", "epoch 82, minibatch 12/12, validation_error 78.666667 %" ] }, { "output_type": "stream", "stream": "stdout", "text": [ "\n", "epoch 83, minibatch 12/12, validation_error 78.666667 %" ] }, { "output_type": "stream", "stream": "stdout", "text": [ "\n", "epoch 84, minibatch 12/12, validation_error 78.666667 %" ] }, { "output_type": "stream", "stream": "stdout", "text": [ "\n", "epoch 85, minibatch 12/12, validation_error 78.666667 %" ] }, { "output_type": "stream", "stream": "stdout", "text": [ "\n", "epoch 86, minibatch 12/12, validation_error 78.666667 %" ] }, { "output_type": "stream", "stream": "stdout", "text": [ "\n", "epoch 87, minibatch 12/12, validation_error 78.000000 %" ] }, { "output_type": "stream", "stream": "stdout", "text": [ "\n", "epoch 88, minibatch 12/12, validation_error 78.666667 %" ] }, { "output_type": "stream", "stream": "stdout", "text": [ "\n", "epoch 89, minibatch 12/12, validation_error 78.666667 %" ] }, { "output_type": "stream", "stream": "stdout", "text": [ "\n", "epoch 90, minibatch 12/12, validation_error 78.666667 %" ] }, { "output_type": "stream", "stream": "stdout", "text": [ "\n", "epoch 91, minibatch 12/12, validation_error 78.666667 %" ] }, { "output_type": "stream", "stream": "stdout", "text": [ "\n", "epoch 92, minibatch 12/12, validation_error 80.000000 %" ] }, { "output_type": "stream", "stream": "stdout", "text": [ "\n", "epoch 93, minibatch 12/12, validation_error 80.000000 %" ] }, { "output_type": "stream", "stream": "stdout", "text": [ "\n", "epoch 94, minibatch 12/12, validation_error 80.000000 %" ] }, { "output_type": "stream", "stream": "stdout", "text": [ "\n", "epoch 95, minibatch 12/12, validation_error 80.000000 %" ] }, { "output_type": "stream", "stream": "stdout", "text": [ "\n", "epoch 96, minibatch 12/12, validation_error 80.000000 %" ] }, { "output_type": "stream", "stream": "stdout", "text": [ "\n", "epoch 97, minibatch 12/12, validation_error 80.000000 %" ] }, { "output_type": "stream", "stream": "stdout", "text": [ "\n", "epoch 98, minibatch 12/12, validation_error 80.000000 %" ] }, { "output_type": "stream", "stream": "stdout", "text": [ "\n", "epoch 99, minibatch 12/12, validation_error 80.000000 %" ] }, { "output_type": "stream", "stream": "stdout", "text": [ "\n", "epoch 100, minibatch 12/12, validation_error 80.000000 %" ] }, { "output_type": "stream", "stream": "stdout", "text": [ "\n", "optimization complete with best validation error: 72.000000 %\n", "0.21875" ] }, { "output_type": "stream", "stream": "stdout", "text": [ "\n", "0.17" ] }, { "output_type": "stream", "stream": "stdout", "text": [ "\n", "(1000, 100)" ] }, { "output_type": "stream", "stream": "stdout", "text": [ "\n" ] } ], "prompt_number": 34 }, { "cell_type": "raw", "metadata": {}, "source": [ "import joblib\n", "joblib.dump(sda, 'sda.pkl')" ] }, { "cell_type": "code", "collapsed": false, "input": [ "## trying deep learning tutorial mnist data\n", "train_set, validation_set, test_set = cPickle.load(open('data/mnist.pkl', 'rb'))\n", "train_X, train_y = train_set\n", "validation_X, validation_y = validation_set\n", "X = np.vstack([train_X, validation_X])\n", "print train_X.shape, train_y.shape\n", "print validation_X.shape, validation_y.shape\n", "print X.shape\n", "classes = range(10)" ], "language": "python", "metadata": {}, "outputs": [ { "output_type": "stream", "stream": "stdout", "text": [ "(50000, 784) (50000,)\n", "(10000, 784) (10000,)\n", "(60000, 784)\n" ] } ], "prompt_number": 7 }, { "cell_type": "code", "collapsed": false, "input": [ "## Denoising Auto Encoder\n", "reload(theanoml2)\n", "reload(theanoml2.lenet)\n", "reload(theanoml2.formula)\n", "reload(theanoml2.auto_encoder)\n", "d_auto_encoder = theanoml2.auto_encoder.DenoisingAutoEncoder(n_hidden = 500, corruption_level = 0.1, n_epochs=5)\n", "d_auto_encoder.partial_fit(X)\n", "print d_auto_encoder.transform(X).shape" ], "language": "python", "metadata": {}, "outputs": [ { "output_type": "stream", "stream": "stdout", "text": [ "training epoch 0, train cost 99.151071\n", "training epoch 1, train cost 79.053245" ] }, { "output_type": "stream", "stream": "stdout", "text": [ "\n", "training epoch 2, train cost 74.209851" ] }, { "output_type": "stream", "stream": "stdout", "text": [ "\n", "training epoch 3, train cost 71.477951" ] }, { "ename": "KeyboardInterrupt", "evalue": "", "output_type": "pyerr", "traceback": [ "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m\n\u001b[0;31mKeyboardInterrupt\u001b[0m Traceback (most recent call last)", "\u001b[0;32m\u001b[0m in \u001b[0;36m\u001b[0;34m()\u001b[0m\n\u001b[1;32m 5\u001b[0m \u001b[0mreload\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mtheanoml2\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mauto_encoder\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 6\u001b[0m \u001b[0md_auto_encoder\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mtheanoml2\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mauto_encoder\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mDenoisingAutoEncoder\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mn_hidden\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;36m500\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mcorruption_level\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;36m0.1\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mn_epochs\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;36m5\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m----> 7\u001b[0;31m \u001b[0md_auto_encoder\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mpartial_fit\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mX\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 8\u001b[0m \u001b[0;32mprint\u001b[0m \u001b[0md_auto_encoder\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mtransform\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mX\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mshape\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n", "\u001b[0;32m/Users/lima/workspace/tutorials/ml-tutorials/theanoml2/formula.pyc\u001b[0m in \u001b[0;36mpartial_fit\u001b[0;34m(self, X)\u001b[0m\n\u001b[1;32m 104\u001b[0m \u001b[0;32mdef\u001b[0m \u001b[0mpartial_fit\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mX\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 105\u001b[0m \u001b[0;32mif\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mformula_\u001b[0m \u001b[0;32mis\u001b[0m \u001b[0mNone\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 106\u001b[0;31m \u001b[0;32mreturn\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mfit\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mX\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 107\u001b[0m \u001b[0;32melse\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 108\u001b[0m \u001b[0;32mreturn\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_optimize\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mX\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n", "\u001b[0;32m/Users/lima/workspace/tutorials/ml-tutorials/theanoml2/formula.pyc\u001b[0m in \u001b[0;36mfit\u001b[0;34m(self, X)\u001b[0m\n\u001b[1;32m 101\u001b[0m \u001b[0;32mdef\u001b[0m \u001b[0mfit\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mX\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 102\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mformula_\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_create_formula\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mX\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 103\u001b[0;31m \u001b[0;32mreturn\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_optimize\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mX\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 104\u001b[0m \u001b[0;32mdef\u001b[0m \u001b[0mpartial_fit\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mX\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 105\u001b[0m \u001b[0;32mif\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mformula_\u001b[0m \u001b[0;32mis\u001b[0m \u001b[0mNone\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n", "\u001b[0;32m/Users/lima/workspace/tutorials/ml-tutorials/theanoml2/formula.pyc\u001b[0m in \u001b[0;36m_optimize\u001b[0;34m(self, X)\u001b[0m\n\u001b[1;32m 120\u001b[0m \t\tmodel_infor = build_batch_fixed_iter_model_infor(self.formula_, v_X, \n\u001b[1;32m 121\u001b[0m \t\t\tbatch_size = self.batch_size, learning_rate = self.learning_rate)\n\u001b[0;32m--> 122\u001b[0;31m \u001b[0mbest_params\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mbatch_fixed_iter_optimize\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mmodel_infor\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mn_epochs\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mn_epochs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 123\u001b[0m \u001b[0;32mfor\u001b[0m \u001b[0mi\u001b[0m \u001b[0;32min\u001b[0m \u001b[0mxrange\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mlen\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mbest_params\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 124\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mformula_\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mparams\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0mi\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mset_value\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mbest_params\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0mi\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n", "\u001b[0;32m/Users/lima/workspace/tutorials/ml-tutorials/theanoml2/optimize.pyc\u001b[0m in \u001b[0;36mbatch_fixed_iter_optimize\u001b[0;34m(model_infor, n_epochs, verbose)\u001b[0m\n\u001b[1;32m 67\u001b[0m \u001b[0;31m## iterative optimization for a fixed number of iterations\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 68\u001b[0m \u001b[0;32mfor\u001b[0m \u001b[0mepoch\u001b[0m \u001b[0;32min\u001b[0m \u001b[0mxrange\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mn_epochs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 69\u001b[0;31m \u001b[0mtrain_cost\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mnp\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mmean\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0mtrain_model\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mi\u001b[0m\u001b[0;34m)\u001b[0m \u001b[0;32mfor\u001b[0m \u001b[0mi\u001b[0m \u001b[0;32min\u001b[0m \u001b[0mxrange\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mn_train_batches\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 70\u001b[0m \u001b[0;32mif\u001b[0m \u001b[0mverbose\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 71\u001b[0m \u001b[0;32mprint\u001b[0m \u001b[0;34m'training epoch %i, train cost %f'\u001b[0m \u001b[0;34m%\u001b[0m \u001b[0;34m(\u001b[0m\u001b[0mepoch\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mtrain_cost\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n", "\u001b[0;32m/Library/Python/2.7/site-packages/Theano-0.6.0rc3-py2.7.egg/theano/compile/function_module.pyc\u001b[0m in \u001b[0;36m__call__\u001b[0;34m(self, *args, **kwargs)\u001b[0m\n\u001b[1;32m 578\u001b[0m \u001b[0mt0_fn\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mtime\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mtime\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 579\u001b[0m \u001b[0;32mtry\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 580\u001b[0;31m \u001b[0moutputs\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mfn\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 581\u001b[0m \u001b[0;32mexcept\u001b[0m \u001b[0mException\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 582\u001b[0m \u001b[0;32mif\u001b[0m \u001b[0mhasattr\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mfn\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m'position_of_error'\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n", "\u001b[0;31mKeyboardInterrupt\u001b[0m: " ] }, { "output_type": "stream", "stream": "stdout", "text": [ "\n" ] } ], "prompt_number": 8 }, { "cell_type": "code", "collapsed": false, "input": [], "language": "python", "metadata": {}, "outputs": [] } ], "metadata": {} } ] }