{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "https://blog.keras.io/building-autoencoders-in-keras.html"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Using Theano backend.\n"
     ]
    }
   ],
   "source": [
    "import matplotlib.pyplot as plt\n",
    "import numpy as np\n",
    "from keras.layers import Input,Dense, Convolution2D, MaxPooling2D, UpSampling2D\n",
    "from keras.models import Model\n",
    "from keras.datasets import mnist\n",
    "from keras import regularizers"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "(x_train,_),(x_test,_) = mnist.load_data()\n",
    "\n",
    "x_train = x_train.astype('float32') / 255\n",
    "x_test = x_test.astype('float32') / 255\n",
    "x_train = x_train.reshape((len(x_train), np.prod(x_train.shape[1:])))\n",
    "x_test = x_test.reshape((len(x_test), np.prod(x_test.shape[1:])))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "%matplotlib inline"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "def plot_imgs(decoded_images,n=20):\n",
    "\n",
    "    plt.figure(figsize=(20,4))\n",
    "    for i in range(n):\n",
    "        # original digit\n",
    "        ax = plt.subplot(2,n,i+1)\n",
    "        plt.imshow(x_test[i].reshape(28,28))\n",
    "        plt.gray()\n",
    "        ax.get_xaxis().set_visible(False)\n",
    "        ax.get_yaxis().set_visible(False)\n",
    "\n",
    "        # reconstruction\n",
    "        ax = plt.subplot(2,n,i+1+n)\n",
    "        plt.imshow(decoded_images[i].reshape(28,28))\n",
    "        plt.gray()\n",
    "        ax.get_xaxis().set_visible(False)\n",
    "        ax.get_yaxis().set_visible(False)\n",
    "    \n",
    "    plt.show()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "collapsed": true
   },
   "source": [
    "# relu activation"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "%%time\n",
    "\n",
    "encoded = None\n",
    "decoded = None\n",
    "\n",
    "encoding_dim = 32\n",
    "input_img = Input(shape=(784,))\n",
    "encoded = Dense(encoding_dim, activation='relu')(input_img)\n",
    "decoded = Dense(784,activation='sigmoid')(encoded)\n",
    "autoencoder = Model(input=input_img, output=decoded)\n",
    "encoder = Model(input=input_img, output = encoded)\n",
    "encoded_input = Input(shape=(encoding_dim,))\n",
    "decoder_layer = autoencoder.layers[-1]\n",
    "decoder = Model(input=encoded_input, output=decoder_layer(encoded_input))\n",
    "autoencoder.compile(optimizer='adadelta',loss='binary_crossentropy')\n",
    "\n",
    "\n",
    "autoencoder.fit(x_train, x_train,\n",
    "               verbose=2,  \n",
    "               nb_epoch = 50,\n",
    "               batch_size = 256,\n",
    "               shuffle=True,\n",
    "               validation_data=(x_test,x_test))\n",
    "\n",
    "encoded_imgs = encoder.predict(x_test)\n",
    "decoded_imgs = decoder.predict(encoded_imgs)\n",
    "\n",
    "plot_imgs(decoded_imgs)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# linear activation"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false,
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "encoded = None\n",
    "decoded = None\n",
    "\n",
    "encoding_dim = 32\n",
    "input_img = Input(shape=(784,))\n",
    "\n",
    "encoded = Dense(encoding_dim, activation='linear')(input_img)\n",
    "decoded = Dense(784,activation='sigmoid')(encoded)\n",
    "autoencoder = Model(input=input_img, output=decoded)\n",
    "\n",
    "encoder = Model(input=input_img, output = encoded)\n",
    "encoded_input = Input(shape=(encoding_dim,))\n",
    "\n",
    "\n",
    "decoder_layer = autoencoder.layers[-1]\n",
    "decoder = Model(input=encoded_input, output=decoder_layer(encoded_input))\n",
    "\n",
    "autoencoder.compile(optimizer='adadelta',loss='binary_crossentropy')\n",
    "\n",
    "\n",
    "autoencoder.fit(x_train, x_train,\n",
    "               verbose=2,  \n",
    "               nb_epoch = 50,\n",
    "               batch_size = 256,\n",
    "               shuffle=True,\n",
    "               validation_data=(x_test,x_test))\n",
    "\n",
    "encoded_imgs = encoder.predict(x_test)\n",
    "decoded_imgs = decoder.predict(encoded_imgs)\n",
    "\n",
    "plot_imgs(decoded_imgs)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "collapsed": true
   },
   "source": [
    "# using regularization"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "%%time\n",
    "\n",
    "encoded = None\n",
    "decoded = None\n",
    "\n",
    "encoding_dim = 32\n",
    "\n",
    "input_img = Input(shape=(784,))\n",
    "# add a Dense layer with a L1 activity regularizer\n",
    "encoded = Dense(encoding_dim, activation='relu',\n",
    "                activity_regularizer=regularizers.activity_l1(10e-5))(input_img)\n",
    "decoded = Dense(784, activation='sigmoid')(encoded)\n",
    "\n",
    "autoencoder = Model(input=input_img, output=decoded)\n",
    "\n",
    "encoder = Model(input=input_img, output = encoded)\n",
    "encoded_input = Input(shape=(encoding_dim,))\n",
    "\n",
    "decoder_layer = autoencoder.layers[-1]\n",
    "decoder = Model(input=encoded_input, output=decoder_layer(encoded_input))\n",
    "autoencoder.compile(optimizer='adadelta',loss='binary_crossentropy')\n",
    "\n",
    "\n",
    "autoencoder.fit(x_train, x_train,\n",
    "               verbose=2,  \n",
    "               nb_epoch = 50,\n",
    "               batch_size = 256,\n",
    "               shuffle=True,\n",
    "               validation_data=(x_test,x_test))\n",
    "\n",
    "encoded_imgs = encoder.predict(x_test)\n",
    "decoded_imgs = decoder.predict(encoded_imgs)\n",
    "\n",
    "plot_imgs(decoded_imgs)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# deep autoencoder"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Train on 60000 samples, validate on 10000 samples\n",
      "Epoch 1/100\n",
      "13s - loss: 0.3602 - val_loss: 0.2642\n",
      "Epoch 2/100\n",
      "13s - loss: 0.2599 - val_loss: 0.2558\n",
      "Epoch 3/100\n",
      "13s - loss: 0.2490 - val_loss: 0.2410\n",
      "Epoch 4/100\n",
      "13s - loss: 0.2341 - val_loss: 0.2223\n",
      "Epoch 5/100\n"
     ]
    },
    {
     "ename": "KeyboardInterrupt",
     "evalue": "",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[0;31mKeyboardInterrupt\u001b[0m                         Traceback (most recent call last)",
      "\u001b[0;32m<ipython-input-5-8f401d38a93a>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m()\u001b[0m\n\u001b[0;32m----> 1\u001b[0;31m \u001b[0mget_ipython\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mrun_cell_magic\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34mu'time'\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34mu''\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34mu\"\\nencoded = None\\ndecoded = None\\n\\ninput_img = Input(shape=(784,))\\n\\nencoded = Dense(128,activation='relu')(input_img)\\nencoded = Dense(64, activation='relu')(encoded)\\nencoded = Dense(32,activation='relu')(encoded)\\n\\ndecoded = Dense(64,activation='relu')(encoded)\\ndecoded = Dense(128,activation='relu')(decoded)\\ndecoded = Dense(784,activation='sigmoid')(decoded)\\n\\nautoencoder = Model(input=input_img, output=decoded)\\n\\n\\nencoder = Model(input=input_img, output = encoded)\\nencoded_input = Input(shape=(32,))\\n\\ndecoder_layer1 = autoencoder.layers[-3]\\ndecoder_layer2 = autoencoder.layers[-2]\\ndecoder_layer3 = autoencoder.layers[-1]\\n\\ndecoder = Model(input=encoded_input, output= decoder_layer3(decoder_layer2(decoder_layer1(encoded_input))))\\n\\nautoencoder.compile(optimizer='adadelta',loss='binary_crossentropy')\\n\\nautoencoder.fit(x_train, x_train, nb_epoch=100, verbose=2,\\n               batch_size=256,shuffle=True,\\n               validation_data=(x_test, x_test))\\n\\nencoded_imgs = encoder.predict(x_test)\\ndecoded_imgs = decoder.predict(encoded_imgs)\\n\\nplot_imgs(decoded_imgs)\"\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m",
      "\u001b[0;32m/home/felipe/venv2/local/lib/python2.7/site-packages/IPython/core/interactiveshell.pyc\u001b[0m in \u001b[0;36mrun_cell_magic\u001b[0;34m(self, magic_name, line, cell)\u001b[0m\n\u001b[1;32m   2113\u001b[0m             \u001b[0mmagic_arg_s\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mvar_expand\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mline\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mstack_depth\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   2114\u001b[0m             \u001b[0;32mwith\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mbuiltin_trap\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m-> 2115\u001b[0;31m                 \u001b[0mresult\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mfn\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mmagic_arg_s\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mcell\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m   2116\u001b[0m             \u001b[0;32mreturn\u001b[0m \u001b[0mresult\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   2117\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m<decorator-gen-59>\u001b[0m in \u001b[0;36mtime\u001b[0;34m(self, line, cell, local_ns)\u001b[0m\n",
      "\u001b[0;32m/home/felipe/venv2/local/lib/python2.7/site-packages/IPython/core/magic.pyc\u001b[0m in \u001b[0;36m<lambda>\u001b[0;34m(f, *a, **k)\u001b[0m\n\u001b[1;32m    186\u001b[0m     \u001b[0;31m# but it's overkill for just that one bit of state.\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    187\u001b[0m     \u001b[0;32mdef\u001b[0m \u001b[0mmagic_deco\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0marg\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 188\u001b[0;31m         \u001b[0mcall\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;32mlambda\u001b[0m \u001b[0mf\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m*\u001b[0m\u001b[0ma\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m**\u001b[0m\u001b[0mk\u001b[0m\u001b[0;34m:\u001b[0m \u001b[0mf\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m*\u001b[0m\u001b[0ma\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m**\u001b[0m\u001b[0mk\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    189\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    190\u001b[0m         \u001b[0;32mif\u001b[0m \u001b[0mcallable\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0marg\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/home/felipe/venv2/local/lib/python2.7/site-packages/IPython/core/magics/execution.pyc\u001b[0m in \u001b[0;36mtime\u001b[0;34m(self, line, cell, local_ns)\u001b[0m\n\u001b[1;32m   1178\u001b[0m         \u001b[0;32melse\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1179\u001b[0m             \u001b[0mst\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mclock2\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m-> 1180\u001b[0;31m             \u001b[0;32mexec\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mcode\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mglob\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mlocal_ns\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m   1181\u001b[0m             \u001b[0mend\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mclock2\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1182\u001b[0m             \u001b[0mout\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mNone\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m<timed exec>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m()\u001b[0m\n",
      "\u001b[0;32m/home/felipe/venv2/local/lib/python2.7/site-packages/keras/engine/training.pyc\u001b[0m in \u001b[0;36mfit\u001b[0;34m(self, x, y, batch_size, nb_epoch, verbose, callbacks, validation_split, validation_data, shuffle, class_weight, sample_weight)\u001b[0m\n\u001b[1;32m   1102\u001b[0m                               \u001b[0mverbose\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mverbose\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mcallbacks\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mcallbacks\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1103\u001b[0m                               \u001b[0mval_f\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mval_f\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mval_ins\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mval_ins\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mshuffle\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mshuffle\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m-> 1104\u001b[0;31m                               callback_metrics=callback_metrics)\n\u001b[0m\u001b[1;32m   1105\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1106\u001b[0m     \u001b[0;32mdef\u001b[0m \u001b[0mevaluate\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mx\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0my\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mbatch_size\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;36m32\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mverbose\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;36m1\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0msample_weight\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mNone\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/home/felipe/venv2/local/lib/python2.7/site-packages/keras/engine/training.pyc\u001b[0m in \u001b[0;36m_fit_loop\u001b[0;34m(self, f, ins, out_labels, batch_size, nb_epoch, verbose, callbacks, val_f, val_ins, shuffle, callback_metrics)\u001b[0m\n\u001b[1;32m    820\u001b[0m                 \u001b[0mbatch_logs\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;34m'size'\u001b[0m\u001b[0;34m]\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mlen\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mbatch_ids\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    821\u001b[0m                 \u001b[0mcallbacks\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mon_batch_begin\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mbatch_index\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mbatch_logs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 822\u001b[0;31m                 \u001b[0mouts\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mf\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mins_batch\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    823\u001b[0m                 \u001b[0;32mif\u001b[0m \u001b[0mtype\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mouts\u001b[0m\u001b[0;34m)\u001b[0m \u001b[0;34m!=\u001b[0m \u001b[0mlist\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    824\u001b[0m                     \u001b[0mouts\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;34m[\u001b[0m\u001b[0mouts\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/home/felipe/venv2/local/lib/python2.7/site-packages/keras/backend/theano_backend.pyc\u001b[0m in \u001b[0;36m__call__\u001b[0;34m(self, inputs)\u001b[0m\n\u001b[1;32m    670\u001b[0m     \u001b[0;32mdef\u001b[0m \u001b[0m__call__\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0minputs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    671\u001b[0m         \u001b[0;32massert\u001b[0m \u001b[0mtype\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0minputs\u001b[0m\u001b[0;34m)\u001b[0m \u001b[0;32min\u001b[0m \u001b[0;34m{\u001b[0m\u001b[0mlist\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mtuple\u001b[0m\u001b[0;34m}\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 672\u001b[0;31m         \u001b[0;32mreturn\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mfunction\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m*\u001b[0m\u001b[0minputs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    673\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    674\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/home/felipe/venv2/local/lib/python2.7/site-packages/theano/compile/function_module.pyc\u001b[0m in \u001b[0;36m__call__\u001b[0;34m(self, *args, **kwargs)\u001b[0m\n\u001b[1;32m    857\u001b[0m         \u001b[0mt0_fn\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mtime\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mtime\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    858\u001b[0m         \u001b[0;32mtry\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 859\u001b[0;31m             \u001b[0moutputs\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mfn\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    860\u001b[0m         \u001b[0;32mexcept\u001b[0m \u001b[0mException\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    861\u001b[0m             \u001b[0;32mif\u001b[0m \u001b[0mhasattr\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mfn\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m'position_of_error'\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;31mKeyboardInterrupt\u001b[0m: "
     ]
    }
   ],
   "source": [
    "%%time\n",
    "\n",
    "encoded = None\n",
    "decoded = None\n",
    "\n",
    "input_img = Input(shape=(784,))\n",
    "\n",
    "encoded = Dense(128,activation='relu')(input_img)\n",
    "encoded = Dense(64, activation='relu')(encoded)\n",
    "encoded = Dense(32,activation='relu')(encoded)\n",
    "\n",
    "decoded = Dense(64,activation='relu')(encoded)\n",
    "decoded = Dense(128,activation='relu')(decoded)\n",
    "decoded = Dense(784,activation='sigmoid')(decoded)\n",
    "\n",
    "autoencoder = Model(input=input_img, output=decoded)\n",
    "\n",
    "\n",
    "encoder = Model(input=input_img, output = encoded)\n",
    "encoded_input = Input(shape=(32,))\n",
    "\n",
    "decoder_layer1 = autoencoder.layers[-3]\n",
    "decoder_layer2 = autoencoder.layers[-2]\n",
    "decoder_layer3 = autoencoder.layers[-1]\n",
    "\n",
    "decoder = Model(input=encoded_input, output= decoder_layer3(decoder_layer2(decoder_layer1(encoded_input))))\n",
    "\n",
    "autoencoder.compile(optimizer='adadelta',loss='binary_crossentropy')\n",
    "\n",
    "autoencoder.fit(x_train, x_train, nb_epoch=100, verbose=2,\n",
    "               batch_size=256,shuffle=True,\n",
    "               validation_data=(x_test, x_test))\n",
    "\n",
    "encoded_imgs = encoder.predict(x_test)\n",
    "decoded_imgs = decoder.predict(encoded_imgs)\n",
    "\n",
    "plot_imgs(decoded_imgs)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "encoded = None\n",
    "decoded = None\n",
    "\n",
    "input_img = Input(shape=(1,28,28))\n",
    "x = Convolution2D(16,3,3, activation='relu', border_mode='same')(input_img)\n",
    "x = MaxPooling2D((2,2), border_mode='same')(x)\n",
    "x = Convolution2D(8,3,3, activation='relu', border_mode='same')(x)\n",
    "x = MaxPooling2D((2,2), border_mode='same')(x)\n",
    "x = Convolution2D(8,3,3, activation='relu', border_mode='same')(x)\n",
    "\n",
    "encoded = MaxPooling2D((2,2),border_mode='same')(x)\n",
    "\n",
    "# at this point the representation has 8 x 4 x 4 = 128 dimensions\n",
    "x = Convolution2D(8,3,3,activation='relu',border_mode='same')(encoded)\n",
    "x = UpSampling2D((2,2))(x)\n",
    "x = Convolution2D(8,3,3, activation='relu', border_mode='same')(x)\n",
    "x = UpSampling2D((2,2))(x)\n",
    "x = Convolution2D(16,3,3, activation='relu')(x)\n",
    "x = UpSampling2D((2,2))(x)\n",
    "\n",
    "decoded = Convolution2D(1,3,3, activation='sigmoid', border_mode='same')(x)\n",
    "\n",
    "autoencoder = Model(input_img,decoded)\n",
    "autoencoder.compile(optimizer='adadelta',loss='binary_crossentropy')\n",
    "\n",
    "(x_train, _), (x_test, _) = mnist.load_data()\n",
    "\n",
    "x_train = x_train.astype('float32') / 255.\n",
    "x_test = x_test.astype('float32') / 255.\n",
    "x_train = np.reshape(x_train, (len(x_train), 1, 28, 28))\n",
    "x_test = np.reshape(x_test, (len(x_test), 1, 28, 28))\n",
    "\n",
    "# the tutorial suggests we use tensorflow but I'll use theano\n",
    "\n",
    "autoencoder.fit(x_train,x_train,\n",
    "            verbose=2,\n",
    "            nb_epoch=50,\n",
    "            batch_size=128,\n",
    "            shuffle=True,\n",
    "            validation_data=(x_test,x_test))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "decoded_imgs = autoencoder.predict(x_test)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "plot_imgs(decoded_imgs)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## denoising"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "(x_train, _), (x_test, _) = mnist.load_data()\n",
    "\n",
    "x_train = x_train.astype('float32') / 255.\n",
    "x_test = x_test.astype('float32') / 255.\n",
    "x_train = np.reshape(x_train, (len(x_train), 1, 28, 28))\n",
    "x_test = np.reshape(x_test, (len(x_test), 1, 28, 28))\n",
    "\n",
    "noise_factor= 0.5\n",
    "\n",
    "x_train_noisy = x_train + noise_factor * np.random.normal(loc=0.0,scale=1.0,size=x_train.shape)\n",
    "x_test_noisy = x_test + noise_factor * np.random.normal(loc=0.0,scale=1.0,size=x_test.shape)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "x_train_noisy = np.clip(x_train_noisy,0.,1.)\n",
    "x_test_noisy = np.clip(x_test_noisy, 0.,1.)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "plot_imgs(x_test_noisy)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "input_img = Input(shape=(1,28,28))\n",
    "\n",
    "x = Convolution2D(32,3,3,activation='relu',border_mode='same')(input_img)\n",
    "x = MaxPooling2D((2,2), border_mode='same')(x)\n",
    "x = Convolution2D(32,3,3,activation='relu',border_mode='same')(x)\n",
    "encoded = MaxPooling2D((2,2),border_mode='same')(x)\n",
    "\n",
    "# at this point the representation is (32,7,7)\n",
    "x = Convolution2D(32,3,3,activation='relu',border_mode='same')(encoded)\n",
    "x = UpSampling2D((2,2))(x)\n",
    "x = Convolution2D(32,3,3,activation='relu',border_mode='same')(x)\n",
    "x = UpSampling2D((2,2))(x)\n",
    "\n",
    "decoded = Convolution2D(1,3,3,activation='sigmoid',border_mode='same')(x)\n",
    "\n",
    "autoencoder = Model(input_img,decoded)\n",
    "\n",
    "# decoder_layer1 = autoencoder.layers[-4]\n",
    "# decoder_layer2 = autoencoder.layers[-3]\n",
    "# decoder_layer3 = autoencoder.layers[-2]\n",
    "# decoder_layer4 = autoencoder.layers[-1]\n",
    "\n",
    "# encoded_input = Input(shape=(32,7,7))\n",
    "\n",
    "# decoder = Model(\n",
    "#     input=encoded_input, \n",
    "#     output= decoder_layer4(decoder_layer3(decoder_layer2(decoder_layer1(encoded_input)))))\n",
    "\n",
    "\n",
    "autoencoder.compile(optimizer='adadelta',loss='binary_crossentropy')\n",
    "\n",
    "# the source is noisy, but the target isn't?\n",
    "autoencoder.fit(x_train_noisy,x_train,\n",
    "               nb_epoch=100,\n",
    "               batch_size=128,\n",
    "               shuffle=True,\n",
    "               validation_data=(x_test_noisy,x_test))\n",
    "\n",
    "\n",
    "decoded_imgs = autoencoder.predict(x_test)\n",
    "plot_imgs(decoded_imgs)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 2",
   "language": "python",
   "name": "python2"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 2
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython2",
   "version": "2.7.12"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 0
}