{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "import numpy as np\n",
    "import matplotlib.pyplot as plt\n",
    "%matplotlib inline\n",
    "from PIL import Image\n",
    "import random\n",
    "import caffe\n",
    "\n",
    "from lib import run_net\n",
    "from lib import score_util\n",
    "\n",
    "from datasets.cityscapes import cityscapes\n",
    "\n",
    "random.seed(0xCAFFE)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Configure caffe and load net"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "caffe.set_device(0)\n",
    "caffe.set_mode_gpu()\n",
    "\n",
    "net = caffe.Net('../nets/stage-cityscapes-fcn8s.prototxt',\n",
    "                '../nets/cityscapes-fcn8s-heavy.caffemodel',\n",
    "                caffe.TEST)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Dataset details"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "CS = cityscapes('/x/cityscapes')\n",
    "n_cl = len(CS.classes)\n",
    "split = 'val'\n",
    "label_frames = CS.list_label_frames(split)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Oracle per frame"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "running 0/500\n",
      "running 100/500\n",
      "running 200/500\n",
      "running 300/500\n",
      "running 400/500\n",
      "Oracle: Per frame\n",
      "acc\t\t cl acc\t\t mIU\t\t fwIU\n",
      "94.179351\t 76.361439\t 66.388069\t 89.528448\t\n"
     ]
    }
   ],
   "source": [
    "hist_perframe = np.zeros((n_cl, n_cl))\n",
    "for i, idx in enumerate(label_frames):\n",
    "    if i % 100 == 0:\n",
    "        print 'running {}/{}'.format(i, len(label_frames))\n",
    "    city = idx.split('_')[0]\n",
    "    # idx is city_shot_frame\n",
    "    im = CS.load_image(split, city, idx)       \n",
    "    out = run_net.segrun(net, CS.preprocess(im))\n",
    "    label = CS.load_label(split, city, idx)\n",
    "    hist_perframe += score_util.fast_hist(label.flatten(), out.flatten(), n_cl)\n",
    "\n",
    "accP, cl_accP, mean_iuP, fw_iuP = score_util.get_scores(hist_perframe)\n",
    "print 'Oracle: Per frame'\n",
    "print 'acc\\t\\t cl acc\\t\\t mIU\\t\\t fwIU'\n",
    "print '{:f}\\t {:f}\\t {:f}\\t {:f}\\t'.format(100*accP, 100*cl_accP, 100*mean_iuP, 100*fw_iuP)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Clockwork baseline every other frame"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Each iteration, choose with probabillity 1/2 whether to run the previous frame or the current frame (this is most similar to adaptive, which samples based on the network, here we do unconditional sampling)."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "running 0/500\n",
      "running 100/500\n",
      "running 200/500\n",
      "running 300/500\n",
      "running 400/500\n",
      "Baseline: Full FCN every other frame\n",
      "acc\t\t cl acc\t\t mIU\t\t fwIU\n",
      "92.814862\t 72.967450\t 61.987991\t 87.382582\t\n"
     ]
    }
   ],
   "source": [
    "hist_baseline = np.zeros((n_cl, n_cl))\n",
    "for i, idx in enumerate(label_frames):\n",
    "    if i % 100 == 0:\n",
    "        print 'running {}/{}'.format(i, len(label_frames))\n",
    "    city = idx.split('_')[0]\n",
    "    all_frames = CS.collect_frame_sequence(split, idx, 19) # list of Images including labeled frame\n",
    "    label = CS.load_label(split, city, idx) # label for CURRENT frame\n",
    "    choice = random.random() # in [0,1)\n",
    "    if choice < 0.5:\n",
    "        preceding_frame = all_frames[-2] # do previous frame\n",
    "        out = run_net.segrun(net, CS.preprocess(preceding_frame))\n",
    "        hist_baseline += score_util.fast_hist(label.flatten(), out.flatten(), n_cl)\n",
    "    else:\n",
    "        curr_frame = all_frames[-1]     \n",
    "        out = run_net.segrun(net, CS.preprocess(curr_frame))\n",
    "        hist_baseline += score_util.fast_hist(label.flatten(), out.flatten(), n_cl)\n",
    "\n",
    "acc, cl_acc, mean_iu, fw_iu = score_util.get_scores(hist_baseline)\n",
    "print 'Baseline: Full FCN every other frame'\n",
    "print 'acc\\t\\t cl acc\\t\\t mIU\\t\\t fwIU'\n",
    "print '{:f}\\t {:f}\\t {:f}\\t {:f}\\t'.format(100*acc, 100*cl_acc, 100*mean_iu, 100*fw_iu)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Alternating Clockwork"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "running 0/500\n",
      "running 100/500\n",
      "running 200/500\n",
      "running 300/500\n",
      "running 400/500\n",
      "Alternating Clockwork\n",
      "acc\t\t cl acc\t\t mIU\t\t fwIU\n",
      "93.643650\t 74.957217\t 64.594754\t 88.641113\t\n"
     ]
    }
   ],
   "source": [
    "hist_altern = np.zeros((n_cl, n_cl))\n",
    "for i, idx in enumerate(label_frames):\n",
    "    if i % 100 == 0:\n",
    "        print 'running {}/{}'.format(i, len(label_frames))\n",
    "    city = idx.split('_')[0]\n",
    "    all_frames = CS.collect_frame_sequence(split, idx, 19) # list of Images including labeled frame\n",
    "    label = CS.load_label(split, city, idx)\n",
    "    curr_frame = all_frames[-1]\n",
    "    choice = random.random() # in [0,1)\n",
    "            \n",
    "    if choice < 0.5:\n",
    "        # Push previous frame through the net\n",
    "        preceding_frame = all_frames[-2] # do previous frame            \n",
    "        _ = run_net.segrun(net, CS.preprocess(preceding_frame))\n",
    "        # Update lower layers on current frame and get prediction\n",
    "        out = run_net.clockwork_forward(net, CS.preprocess(curr_frame))\n",
    "        hist_altern += score_util.fast_hist(label.flatten(), out.flatten(), n_cl)\n",
    "    else:\n",
    "        out = run_net.segrun(net, CS.preprocess(curr_frame))\n",
    "        hist_altern += score_util.fast_hist(label.flatten(), out.flatten(), n_cl)\n",
    "\n",
    "acc, cl_acc, mean_iu, fw_iu = score_util.get_scores(hist_altern)\n",
    "print 'Alternating Clockwork'\n",
    "print 'acc\\t\\t cl acc\\t\\t mIU\\t\\t fwIU'\n",
    "print '{:f}\\t {:f}\\t {:f}\\t {:f}\\t'.format(100*acc, 100*cl_acc, 100*mean_iu, 100*fw_iu)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Adaptive Clockwork"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Adaptive Clockwork: Threshold 0.25  Updated 4252/4500 frames (94.5%)\n",
      "acc\t cl acc\t mIU\t fwIU\n",
      "94.1\t 75.8\t 65.8\t 89.4\n",
      "Adaptive Clockwork: Threshold 0.375  Updated 3739/4500 frames (83.1%)\n",
      "acc\t cl acc\t mIU\t fwIU\n",
      "93.8\t 74.3\t 64.2\t 88.8\n",
      "Adaptive Clockwork: Threshold 0.5  Updated 1872/4500 frames (41.6%)\n",
      "acc\t cl acc\t mIU\t fwIU\n",
      "92.6\t 70.7\t 60.1\t 86.9\n",
      "Adaptive Clockwork: Threshold 0.47  Updated 2250/4500 frames (50.0%)\n",
      "acc\t cl acc\t mIU\t fwIU\n",
      "93.0\t 72.2\t 61.8\t 87.6\n"
     ]
    }
   ],
   "source": [
    "# collect all preceding frames in the Cityscapes sequence surrounding each annotated frame\n",
    "SEQ_LEN = 19  \n",
    "\n",
    "def scoremap_diff(prev_scores, scores):\n",
    "    prev_seg = prev_scores.argmax(axis=0).astype(np.uint8).copy()\n",
    "    curr_seg = scores.argmax(axis=0).astype(np.uint8).copy()\n",
    "    diff = np.array(prev_seg != curr_seg).mean()\n",
    "    return diff\n",
    "\n",
    "def adaptive_clockwork_cityscapes(thresh):\n",
    "    hist = np.zeros((n_cl, n_cl))\n",
    "    num_frames = 0  # number of frames in total\n",
    "    num_update_frames = 0  # number of frames when clock fires\n",
    "    for idx in CS.list_label_frames('val'):\n",
    "        city = idx.split('_')[0]\n",
    "        # run on sequence of preceding frames, fully processing the first frame\n",
    "        frames = CS.collect_frame_sequence('val', idx, SEQ_LEN)\n",
    "        first_frame, frames = frames[0], frames[1:]\n",
    "        _ = run_net.segrun(net, CS.preprocess(first_frame))\n",
    "        prev_score = net.blobs['score_pool4'].data[0].copy() \n",
    "        num_frames += 1\n",
    "        for f in frames:\n",
    "            num_frames += 1\n",
    "            # Run to pool4 on current frame\n",
    "            run_net.feed_net(net, CS.preprocess(f))\n",
    "            net.forward(start='conv1_1', end='score_pool4')\n",
    "            curr_score = net.blobs['score_pool4'].data[0].copy()\n",
    "\n",
    "            # Decide whether or not to update to fc7\n",
    "            if scoremap_diff(prev_score, curr_score) >= thresh:\n",
    "                net.forward(start='conv5_1', end='upscore2') \n",
    "                prev_score = net.blobs['score_pool4'].data[0].copy()\n",
    "                num_update_frames += 1\n",
    "\n",
    "        # Compute full merge score on the annotated frame (the last frame)\n",
    "        net.forward(start='score_pool4c') \n",
    "        out = net.blobs['score'].data[0].argmax(axis=0).astype(np.uint8)\n",
    "        label = CS.load_label('val', city, idx)\n",
    "        hist += score_util.score_out_gt(out, label, n_cl=n_cl)\n",
    "\n",
    "    acc, cl_acc, mean_iu, fw_iu = score_util.get_scores(hist)\n",
    "    print 'Adaptive Clockwork: Threshold', thresh, ' Updated {:d}/{:d} frames ({:2.1f}%)'.format(num_update_frames, num_frames, 100.0*num_update_frames/num_frames)\n",
    "    print 'acc\\t cl acc\\t mIU\\t fwIU'\n",
    "    print '{:2.1f}\\t {:2.1f}\\t {:2.1f}\\t {:2.1f}\\t'.format(100*acc, 100*cl_acc, 100*mean_iu, 100*fw_iu)\n",
    "    return acc, cl_acc, mean_iu, fw_iu\n",
    "\n",
    "for thresh in (0.25, 0.35, 0.47):\n",
    "    adaptive_clockwork_cityscapes(thresh) "
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 2",
   "language": "python",
   "name": "python2"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 2
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython2",
   "version": "2.7.11+"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 0
}