{ "cells": [ { "cell_type": "markdown", "metadata": {}, "source": [ "# Tensorflow Basic concepts" ] }, { "cell_type": "code", "execution_count": 1, "metadata": {}, "outputs": [ { "data": { "text/html": [ "
" ], "text/plain": [ "" ] }, "execution_count": 1, "metadata": {}, "output_type": "execute_result" } ], "source": [ "import addutils.toc ; addutils.toc.js(ipy_notebook=True)" ] }, { "cell_type": "code", "execution_count": 2, "metadata": {}, "outputs": [ { "data": { "text/html": [ "\n", "\n" ], "text/plain": [ "" ] }, "execution_count": 2, "metadata": {}, "output_type": "execute_result" } ], "source": [ "import scipy.io\n", "import numpy as np\n", "import pandas as pd\n", "from time import time\n", "from sklearn.metrics import confusion_matrix\n", "from IPython.display import Image\n", "from addutils import css_notebook\n", "css_notebook()" ] }, { "cell_type": "code", "execution_count": 3, "metadata": {}, "outputs": [ { "data": { "text/html": [ "\n", "
\n", " \n", " Loading BokehJS ...\n", "
" ] }, "metadata": {}, "output_type": "display_data" }, { "data": { "application/javascript": [ "\n", "(function(root) {\n", " function now() {\n", " return new Date();\n", " }\n", "\n", " var force = true;\n", "\n", " if (typeof (root._bokeh_onload_callbacks) === \"undefined\" || force === true) {\n", " root._bokeh_onload_callbacks = [];\n", " root._bokeh_is_loading = undefined;\n", " }\n", "\n", " var JS_MIME_TYPE = 'application/javascript';\n", " var HTML_MIME_TYPE = 'text/html';\n", " var EXEC_MIME_TYPE = 'application/vnd.bokehjs_exec.v0+json';\n", " var CLASS_NAME = 'output_bokeh rendered_html';\n", "\n", " /**\n", " * Render data to the DOM node\n", " */\n", " function render(props, node) {\n", " var script = document.createElement(\"script\");\n", " node.appendChild(script);\n", " }\n", "\n", " /**\n", " * Handle when an output is cleared or removed\n", " */\n", " function handleClearOutput(event, handle) {\n", " var cell = handle.cell;\n", "\n", " var id = cell.output_area._bokeh_element_id;\n", " var server_id = cell.output_area._bokeh_server_id;\n", " // Clean up Bokeh references\n", " if (id !== undefined) {\n", " Bokeh.index[id].model.document.clear();\n", " delete Bokeh.index[id];\n", " }\n", "\n", " if (server_id !== undefined) {\n", " // Clean up Bokeh references\n", " var cmd = \"from bokeh.io.state import curstate; print(curstate().uuid_to_server['\" + server_id + \"'].get_sessions()[0].document.roots[0]._id)\";\n", " cell.notebook.kernel.execute(cmd, {\n", " iopub: {\n", " output: function(msg) {\n", " var element_id = msg.content.text.trim();\n", " Bokeh.index[element_id].model.document.clear();\n", " delete Bokeh.index[element_id];\n", " }\n", " }\n", " });\n", " // Destroy server and session\n", " var cmd = \"import bokeh.io.notebook as ion; ion.destroy_server('\" + server_id + \"')\";\n", " cell.notebook.kernel.execute(cmd);\n", " }\n", " }\n", "\n", " /**\n", " * Handle when a new output is added\n", " */\n", " function handleAddOutput(event, handle) {\n", " var output_area = handle.output_area;\n", " var output = handle.output;\n", "\n", " // limit handleAddOutput to display_data with EXEC_MIME_TYPE content only\n", " if ((output.output_type != \"display_data\") || (!output.data.hasOwnProperty(EXEC_MIME_TYPE))) {\n", " return\n", " }\n", "\n", " var toinsert = output_area.element.find(\".\" + CLASS_NAME.split(' ')[0]);\n", "\n", " if (output.metadata[EXEC_MIME_TYPE][\"id\"] !== undefined) {\n", " toinsert[0].firstChild.textContent = output.data[JS_MIME_TYPE];\n", " // store reference to embed id on output_area\n", " output_area._bokeh_element_id = output.metadata[EXEC_MIME_TYPE][\"id\"];\n", " }\n", " if (output.metadata[EXEC_MIME_TYPE][\"server_id\"] !== undefined) {\n", " var bk_div = document.createElement(\"div\");\n", " bk_div.innerHTML = output.data[HTML_MIME_TYPE];\n", " var script_attrs = bk_div.children[0].attributes;\n", " for (var i = 0; i < script_attrs.length; i++) {\n", " toinsert[0].firstChild.setAttribute(script_attrs[i].name, script_attrs[i].value);\n", " }\n", " // store reference to server id on output_area\n", " output_area._bokeh_server_id = output.metadata[EXEC_MIME_TYPE][\"server_id\"];\n", " }\n", " }\n", "\n", " function register_renderer(events, OutputArea) {\n", "\n", " function append_mime(data, metadata, element) {\n", " // create a DOM node to render to\n", " var toinsert = this.create_output_subarea(\n", " metadata,\n", " CLASS_NAME,\n", " EXEC_MIME_TYPE\n", " );\n", " this.keyboard_manager.register_events(toinsert);\n", " // Render to node\n", " var props = {data: data, metadata: metadata[EXEC_MIME_TYPE]};\n", " render(props, toinsert[0]);\n", " element.append(toinsert);\n", " return toinsert\n", " }\n", "\n", " /* Handle when an output is cleared or removed */\n", " events.on('clear_output.CodeCell', handleClearOutput);\n", " events.on('delete.Cell', handleClearOutput);\n", "\n", " /* Handle when a new output is added */\n", " events.on('output_added.OutputArea', handleAddOutput);\n", "\n", " /**\n", " * Register the mime type and append_mime function with output_area\n", " */\n", " OutputArea.prototype.register_mime_type(EXEC_MIME_TYPE, append_mime, {\n", " /* Is output safe? */\n", " safe: true,\n", " /* Index of renderer in `output_area.display_order` */\n", " index: 0\n", " });\n", " }\n", "\n", " // register the mime type if in Jupyter Notebook environment and previously unregistered\n", " if (root.Jupyter !== undefined) {\n", " var events = require('base/js/events');\n", " var OutputArea = require('notebook/js/outputarea').OutputArea;\n", "\n", " if (OutputArea.prototype.mime_types().indexOf(EXEC_MIME_TYPE) == -1) {\n", " register_renderer(events, OutputArea);\n", " }\n", " }\n", "\n", " \n", " if (typeof (root._bokeh_timeout) === \"undefined\" || force === true) {\n", " root._bokeh_timeout = Date.now() + 5000;\n", " root._bokeh_failed_load = false;\n", " }\n", "\n", " var NB_LOAD_WARNING = {'data': {'text/html':\n", " \"
\\n\"+\n", " \"

\\n\"+\n", " \"BokehJS does not appear to have successfully loaded. If loading BokehJS from CDN, this \\n\"+\n", " \"may be due to a slow or bad network connection. Possible fixes:\\n\"+\n", " \"

\\n\"+\n", " \"
    \\n\"+\n", " \"
  • re-rerun `output_notebook()` to attempt to load from CDN again, or
    • \\n\"+\n", " \"
  • use INLINE resources instead, as so:
    • \\n\"+\n", " \"
\\n\"+\n", " \"\\n\"+\n", " \"from bokeh.resources import INLINE\\n\"+\n", " \"output_notebook(resources=INLINE)\\n\"+\n", " \"\\n\"+\n", " \"
\"}};\n", "\n", " function display_loaded() {\n", " var el = document.getElementById(\"17b633e5-4553-47ce-aad4-efdc66fc83d3\");\n", " if (el != null) {\n", " el.textContent = \"BokehJS is loading...\";\n", " }\n", " if (root.Bokeh !== undefined) {\n", " if (el != null) {\n", " el.textContent = \"BokehJS \" + root.Bokeh.version + \" successfully loaded.\";\n", " }\n", " } else if (Date.now() < root._bokeh_timeout) {\n", " setTimeout(display_loaded, 100)\n", " }\n", " }\n", "\n", "\n", " function run_callbacks() {\n", " try {\n", " root._bokeh_onload_callbacks.forEach(function(callback) { callback() });\n", " }\n", " finally {\n", " delete root._bokeh_onload_callbacks\n", " }\n", " console.info(\"Bokeh: all callbacks have finished\");\n", " }\n", "\n", " function load_libs(js_urls, callback) {\n", " root._bokeh_onload_callbacks.push(callback);\n", " if (root._bokeh_is_loading > 0) {\n", " console.log(\"Bokeh: BokehJS is being loaded, scheduling callback at\", now());\n", " return null;\n", " }\n", " if (js_urls == null || js_urls.length === 0) {\n", " run_callbacks();\n", " return null;\n", " }\n", " console.log(\"Bokeh: BokehJS not loaded, scheduling load and callback at\", now());\n", " root._bokeh_is_loading = js_urls.length;\n", " for (var i = 0; i < js_urls.length; i++) {\n", " var url = js_urls[i];\n", " var s = document.createElement('script');\n", " s.src = url;\n", " s.async = false;\n", " s.onreadystatechange = s.onload = function() {\n", " root._bokeh_is_loading--;\n", " if (root._bokeh_is_loading === 0) {\n", " console.log(\"Bokeh: all BokehJS libraries loaded\");\n", " run_callbacks()\n", " }\n", " };\n", " s.onerror = function() {\n", " console.warn(\"failed to load library \" + url);\n", " };\n", " console.log(\"Bokeh: injecting script tag for BokehJS library: \", url);\n", " document.getElementsByTagName(\"head\")[0].appendChild(s);\n", " }\n", " };var element = document.getElementById(\"17b633e5-4553-47ce-aad4-efdc66fc83d3\");\n", " if (element == null) {\n", " console.log(\"Bokeh: ERROR: autoload.js configured with elementid '17b633e5-4553-47ce-aad4-efdc66fc83d3' but no matching script tag was found. \")\n", " return false;\n", " }\n", "\n", " var js_urls = [\"https://cdn.pydata.org/bokeh/release/bokeh-0.12.13.min.js\", \"https://cdn.pydata.org/bokeh/release/bokeh-widgets-0.12.13.min.js\", \"https://cdn.pydata.org/bokeh/release/bokeh-tables-0.12.13.min.js\", \"https://cdn.pydata.org/bokeh/release/bokeh-gl-0.12.13.min.js\"];\n", "\n", " var inline_js = [\n", " function(Bokeh) {\n", " Bokeh.set_log_level(\"info\");\n", " },\n", " \n", " function(Bokeh) {\n", " \n", " },\n", " function(Bokeh) {\n", " console.log(\"Bokeh: injecting CSS: https://cdn.pydata.org/bokeh/release/bokeh-0.12.13.min.css\");\n", " Bokeh.embed.inject_css(\"https://cdn.pydata.org/bokeh/release/bokeh-0.12.13.min.css\");\n", " console.log(\"Bokeh: injecting CSS: https://cdn.pydata.org/bokeh/release/bokeh-widgets-0.12.13.min.css\");\n", " Bokeh.embed.inject_css(\"https://cdn.pydata.org/bokeh/release/bokeh-widgets-0.12.13.min.css\");\n", " console.log(\"Bokeh: injecting CSS: https://cdn.pydata.org/bokeh/release/bokeh-tables-0.12.13.min.css\");\n", " Bokeh.embed.inject_css(\"https://cdn.pydata.org/bokeh/release/bokeh-tables-0.12.13.min.css\");\n", " }\n", " ];\n", "\n", " function run_inline_js() {\n", " \n", " if ((root.Bokeh !== undefined) || (force === true)) {\n", " for (var i = 0; i < inline_js.length; i++) {\n", " inline_js[i].call(root, root.Bokeh);\n", " }if (force === true) {\n", " display_loaded();\n", " }} else if (Date.now() < root._bokeh_timeout) {\n", " setTimeout(run_inline_js, 100);\n", " } else if (!root._bokeh_failed_load) {\n", " console.log(\"Bokeh: BokehJS failed to load within specified timeout.\");\n", " root._bokeh_failed_load = true;\n", " } else if (force !== true) {\n", " var cell = $(document.getElementById(\"17b633e5-4553-47ce-aad4-efdc66fc83d3\")).parents('.cell').data().cell;\n", " cell.output_area.append_execute_result(NB_LOAD_WARNING)\n", " }\n", "\n", " }\n", "\n", " if (root._bokeh_is_loading === 0) {\n", " console.log(\"Bokeh: BokehJS loaded, going straight to plotting\");\n", " run_inline_js();\n", " } else {\n", " load_libs(js_urls, function() {\n", " console.log(\"Bokeh: BokehJS plotting callback run at\", now());\n", " run_inline_js();\n", " });\n", " }\n", "}(window));" ], "application/vnd.bokehjs_load.v0+json": "\n(function(root) {\n function now() {\n return new Date();\n }\n\n var force = true;\n\n if (typeof (root._bokeh_onload_callbacks) === \"undefined\" || force === true) {\n root._bokeh_onload_callbacks = [];\n root._bokeh_is_loading = undefined;\n }\n\n \n\n \n if (typeof (root._bokeh_timeout) === \"undefined\" || force === true) {\n root._bokeh_timeout = Date.now() + 5000;\n root._bokeh_failed_load = false;\n }\n\n var NB_LOAD_WARNING = {'data': {'text/html':\n \"
\\n\"+\n \"

\\n\"+\n \"BokehJS does not appear to have successfully loaded. If loading BokehJS from CDN, this \\n\"+\n \"may be due to a slow or bad network connection. Possible fixes:\\n\"+\n \"

\\n\"+\n \"
    \\n\"+\n \"
  • re-rerun `output_notebook()` to attempt to load from CDN again, or
    • \\n\"+\n \"
  • use INLINE resources instead, as so:
    • \\n\"+\n \"
\\n\"+\n \"\\n\"+\n \"from bokeh.resources import INLINE\\n\"+\n \"output_notebook(resources=INLINE)\\n\"+\n \"\\n\"+\n \"
\"}};\n\n function display_loaded() {\n var el = document.getElementById(\"17b633e5-4553-47ce-aad4-efdc66fc83d3\");\n if (el != null) {\n el.textContent = \"BokehJS is loading...\";\n }\n if (root.Bokeh !== undefined) {\n if (el != null) {\n el.textContent = \"BokehJS \" + root.Bokeh.version + \" successfully loaded.\";\n }\n } else if (Date.now() < root._bokeh_timeout) {\n setTimeout(display_loaded, 100)\n }\n }\n\n\n function run_callbacks() {\n try {\n root._bokeh_onload_callbacks.forEach(function(callback) { callback() });\n }\n finally {\n delete root._bokeh_onload_callbacks\n }\n console.info(\"Bokeh: all callbacks have finished\");\n }\n\n function load_libs(js_urls, callback) {\n root._bokeh_onload_callbacks.push(callback);\n if (root._bokeh_is_loading > 0) {\n console.log(\"Bokeh: BokehJS is being loaded, scheduling callback at\", now());\n return null;\n }\n if (js_urls == null || js_urls.length === 0) {\n run_callbacks();\n return null;\n }\n console.log(\"Bokeh: BokehJS not loaded, scheduling load and callback at\", now());\n root._bokeh_is_loading = js_urls.length;\n for (var i = 0; i < js_urls.length; i++) {\n var url = js_urls[i];\n var s = document.createElement('script');\n s.src = url;\n s.async = false;\n s.onreadystatechange = s.onload = function() {\n root._bokeh_is_loading--;\n if (root._bokeh_is_loading === 0) {\n console.log(\"Bokeh: all BokehJS libraries loaded\");\n run_callbacks()\n }\n };\n s.onerror = function() {\n console.warn(\"failed to load library \" + url);\n };\n console.log(\"Bokeh: injecting script tag for BokehJS library: \", url);\n document.getElementsByTagName(\"head\")[0].appendChild(s);\n }\n };var element = document.getElementById(\"17b633e5-4553-47ce-aad4-efdc66fc83d3\");\n if (element == null) {\n console.log(\"Bokeh: ERROR: autoload.js configured with elementid '17b633e5-4553-47ce-aad4-efdc66fc83d3' but no matching script tag was found. \")\n return false;\n }\n\n var js_urls = [\"https://cdn.pydata.org/bokeh/release/bokeh-0.12.13.min.js\", \"https://cdn.pydata.org/bokeh/release/bokeh-widgets-0.12.13.min.js\", \"https://cdn.pydata.org/bokeh/release/bokeh-tables-0.12.13.min.js\", \"https://cdn.pydata.org/bokeh/release/bokeh-gl-0.12.13.min.js\"];\n\n var inline_js = [\n function(Bokeh) {\n Bokeh.set_log_level(\"info\");\n },\n \n function(Bokeh) {\n \n },\n function(Bokeh) {\n console.log(\"Bokeh: injecting CSS: https://cdn.pydata.org/bokeh/release/bokeh-0.12.13.min.css\");\n Bokeh.embed.inject_css(\"https://cdn.pydata.org/bokeh/release/bokeh-0.12.13.min.css\");\n console.log(\"Bokeh: injecting CSS: https://cdn.pydata.org/bokeh/release/bokeh-widgets-0.12.13.min.css\");\n Bokeh.embed.inject_css(\"https://cdn.pydata.org/bokeh/release/bokeh-widgets-0.12.13.min.css\");\n console.log(\"Bokeh: injecting CSS: https://cdn.pydata.org/bokeh/release/bokeh-tables-0.12.13.min.css\");\n Bokeh.embed.inject_css(\"https://cdn.pydata.org/bokeh/release/bokeh-tables-0.12.13.min.css\");\n }\n ];\n\n function run_inline_js() {\n \n if ((root.Bokeh !== undefined) || (force === true)) {\n for (var i = 0; i < inline_js.length; i++) {\n inline_js[i].call(root, root.Bokeh);\n }if (force === true) {\n display_loaded();\n }} else if (Date.now() < root._bokeh_timeout) {\n setTimeout(run_inline_js, 100);\n } else if (!root._bokeh_failed_load) {\n console.log(\"Bokeh: BokehJS failed to load within specified timeout.\");\n root._bokeh_failed_load = true;\n } else if (force !== true) {\n var cell = $(document.getElementById(\"17b633e5-4553-47ce-aad4-efdc66fc83d3\")).parents('.cell').data().cell;\n cell.output_area.append_execute_result(NB_LOAD_WARNING)\n }\n\n }\n\n if (root._bokeh_is_loading === 0) {\n console.log(\"Bokeh: BokehJS loaded, going straight to plotting\");\n run_inline_js();\n } else {\n load_libs(js_urls, function() {\n console.log(\"Bokeh: BokehJS plotting callback run at\", now());\n run_inline_js();\n });\n }\n}(window));" }, "metadata": {}, "output_type": "display_data" } ], "source": [ "import bokeh.plotting as bk\n", "bk.output_notebook()" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "# 1 Introduction" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "The main argument we treat in this notebook is TensorFlow, a library for machine learning and deep learning open sourced by Google in November 2015. For more information please visit [TensorFlow](http://www.tensorflow.org).\n", "\n", ">\"TensorFlow™ is an open source software library for numerical computation using data flow graphs.\"\n", "\n", "In TensorFlow the computation is formally a graph, with nodes representing operations while edges representing tensors (multidimensional data) communicated between operations. According to TensorFlow web site, the flow of tensors through the graph is where TensorFlow gets its name. It is not intended to be only a neural network library but to perform any computation that can be expressed as a graph. TensorFlow automatic differentiation is especially suited for gradient based machine learning algorithms. The library is written in C++ and it has nice Python bindings. Moreover it can run both on CPU and GPU. \n", "\n", "Great, but why TensorFlow? There a lot of framework for Deep Learning, each with its own features and use cases. For example PyTorch is suitable for reasearch because it is flexible enough to create new models but it is not suitable for production because it does note scale well. Other frameworks such as Caffe and MXNet are scalable but lack flexibility. Tensorflow is both **flexible** and **scalable** and allows users to go easility from research into production." ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "## 1.1 Installation" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "The notebooks on TensroFlow requires few additional packages. Please be sure to install them properly. We will outline how to do it in the following paragraphs." ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "#### 1.1.1 GPU support" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "To use TensorFlow to its full extent, you should take advantage of the parallelization provided by GPUs. Currently TensorFlow support Nvidia GPU via **CUDA Toolkit**. Download and install [CUDA Toolkit](https://developer.nvidia.com/cuda-downloads) for your operating system before continuing. At the time of writing the current relase of CUDA Toolkit is 9.1. \n", "\n", "**IMPORTANT**: TensorFlow build uses prior version of CUDA, that is **CUDA Toolkit 9.0**. We highly recommend this version if you are going to install TensorFlow via pip. Remember to install **cuDNN 7** with support for CUDA Toolkit 9.0 as well.\n", "\n", "> The NVIDIA® CUDA® Toolkit provides a development environment for creating high performance GPU-accelerated applications. With the CUDA Toolkit, you can develop, optimize and deploy your applications on GPU-accelerated embedded systems, desktop workstations, enterprise data centers, cloud-based platforms and HPC supercomputers. The toolkit includes GPU-accelerated libraries, debugging and optimization tools, a C/C++ compiler and a runtime library to deploy your application.\n", "\n", ">GPU-accelerated CUDA libraries enable drop-in acceleration across multiple domains such as linear algebra, image and video processing, deep learning and graph analytics. For developing custom algorithms, you can use available integrations with commonly used languages and numerical packages as well as well-published development APIs. Your CUDA applications can be deployed across all NVIDIA GPU families available on premise and on GPU instances in the cloud. Using built-in capabilities for distributing computations across multi-GPU configurations, scientists and researchers can develop applications that scale from single GPU workstations to cloud installations with thousands of GPUs.\n", "\n", "Nvidia provides also a library of primitives designed specifically for Deep Learning called **cuDNN**. Installing [cuDNN](https://developer.nvidia.com/cudnn) further improves performance and it is highly advisable (note that you need to register as a developer to download it). \n", "\n", "> The NVIDIA CUDA® Deep Neural Network library (cuDNN) is a GPU-accelerated library of primitives for deep neural networks. cuDNN provides highly tuned implementations for standard routines such as forward and backward convolution, pooling, normalization, and activation layers. cuDNN is part of the NVIDIA Deep Learning SDK.\n", "\n", "After this steps you can install TensorFlow via pip package or directly from source. Even if you can benefit from an installation from source, as generally TensorFlow runs faster when built from source, we recommend to install via pip package for the purpose of these tutorials. Activate your favourite environment (addfor_tutorials) and type: \n", "\n", "`pip install tensorflow-gpu`\n", "\n", "It should take few minutes to download and install. If you want to build from source, clone TensorFlow repository and follow this [guide](https://www.tensorflow.org/install/install_sources)" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "#### 1.1.2 CPU only" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "If you want to use this notebook on your laptop (*without* GPU support) type this command: \n", "\n", "`pip install tensorflow`\n", "\n", "This allows you to run the notebooks, although with poorer performance." ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "#### 1.1.3 Documentation" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "If above commands fails try with: \n", "\n", "- `pip install --ignore-installed --upgrade https://storage.googleapis.com/tensorflow/linux/gpu/tensorflow_gpu-1.5.0-cp36-cp36m-linux_x86_64.whl` \n", " \n", " (for GPU support)\n", "\n", "\n", "- `pip install --ignore-installed --upgrade https://storage.googleapis.com/tensorflow/linux/cpu/tensorflow-1.5.0-cp36-cp36m-linux_x86_64.whl` \n", " \n", " (for CPU only)\n", "\n", "For further information about installing TensorFlow read the official documentation: [https://www.tensorflow.org/install/](https://www.tensorflow.org/install/). " ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "## 2 Tensorflow Basics" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "First import tensorflow and show current version. At the moment of writing we are using tensorflow version 1.5." ] }, { "cell_type": "code", "execution_count": 4, "metadata": {}, "outputs": [ { "data": { "text/plain": [ "'1.5.0'" ] }, "execution_count": 4, "metadata": {}, "output_type": "execute_result" } ], "source": [ "import tensorflow as tf\n", "tf.__version__" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "TensorFlow represent computations as **graphs**. Nodes in the graph are called **ops** (operations). An op takes zero or more **Tensors** and produces zero or more Tensors as output. A Tensor is a multidimensional array with a specified type. The graph is a description of a computation, in order to actually execute the computation a graph must be launched in a **session**. A session exectue a specific graph on one of the available **devices** (that can be either CPUs or GPUs).\n", "\n", "In following paragraphs we will clarify these concepts." ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "### 2.1 Computation Graphs" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "TensorFlow programs are usually structured into a construction phase, that assembles a graph, and an execution phase that uses a session to execute ops in the graph.\n", "\n", "For example, it is possible to represent and train a neural network in the construction phase, and then repeatedly execute a set of training ops in the graph in the execution phase.\n", "\n", "It is possibile to build a graph by starting with nodes that do not need any input, such as constant nodes. Then it is possible to use the output of the constant node as input to other operations. TensorFlow uses a default graph to which operations are added, the default graph is created empty as soon as tensorflow is imported. It is sufficient for most operations but it is also possible to manage multiple graphs with the `Graph` class." ] }, { "cell_type": "code", "execution_count": 5, "metadata": { "collapsed": true }, "outputs": [], "source": [ "x = tf.constant(4.)\n", "y = tf.constant(3.)\n", "product = tf.multiply(x, y)" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "The code creates three nodes: two constant and an operation (multiplication) that takes two inputs (the two constants) and produces an output (product). To actually procude an output is is necessary to run the graph in a session. \n", "\n", "What is the difference between this expression and a corresponding plain python code that multiply two constants? Try to print the result:" ] }, { "cell_type": "code", "execution_count": 6, "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "Tensor(\"Mul:0\", shape=(), dtype=float32)\n" ] } ], "source": [ "print(product)" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "The key idea is that `product` does not compute the product of x and y but rather add the product operator to a graph of computation that will be executed later. \n", "\n", "**Computation Graphs** allow us to implement ML algorithms by creating and executing operations that interacts with each others. The interaction between operations constitutes the graph. \n", "\n", "What is a graph in the end? It is a set of *nodes* (or vertices) interconnected by (directed) *edges*. Edges allow data to **flow** from one node to another. Each node represent an operation, that produces an ouput that is passed on in the graph. Operations can be of any kind, from math to logging operation (we will se more on this when we will use TensorBoard).\n", "\n", "The graph connectivity defines a set of node dependencies, TensorFlow is able to optimize the computation based on this dependencies. Being able to identify connectivities allows to distribute computation and avoid performing redundant computations on irrelevant portion of the graph.\n", "\n", "We can add other operations to the graph. For example we can do a sum after the product." ] }, { "cell_type": "code", "execution_count": 7, "metadata": { "collapsed": true }, "outputs": [], "source": [ "z = tf.constant(8.)\n", "addition = tf.add(product, z)" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "The next image represent the default graph for the operations defined before" ] }, { "cell_type": "code", "execution_count": 8, "metadata": {}, "outputs": [ { "data": { "image/jpeg": "/9j/4AAQSkZJRgABAQEAbABsAAD/2wBDAAgGBgcGBQgHBwcJCQgKDBQNDAsLDBkSEw8UHRofHh0aHBwgJC4nICIsIxwcKDcpLDAxNDQ0Hyc5PTgyPC4zNDL/2wBDAQkJCQwLDBgNDRgyIRwhMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjL/wgARCAJ1A30DASIAAhEBAxEB/8QAGwABAAIDAQEAAAAAAAAAAAAAAAQFAgMGAQf/xAAaAQEAAwEBAQAAAAAAAAAAAAAAAgQFAQMG/9oADAMBAAIQAxAAAAHvwAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAADR3m9C8l5zkaRGfo5IAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAABBiwbNLbqLNQO8ZYnbKz5rfXs37DOpeB0AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAABVzqCxU8FuiAAABJveZuK1ucKt4AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAACqrt+jQygn5gAAAJEdyXTeeV2bryZXC9wZAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA5zDLHSxw7wCtYRvL2vh6+IAF/ETc3X5bpq655IAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAACg0WFfoZQT8wNDe50O8AG/nbqDZs3X0b4mh2yAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAABooOmqbFSvFuiAAAAtoF9Xt+ipeeeislSYxJVlkegAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAeeikidNXWqNUzws1QcHp5ulWnhZx2FO+DoACtshDmaIZZsMwAAARYdlKfRVsX7n9vF2gTrXh6PSIAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAHkaU7GvWCfnDlZIzCMwAAAAAK/KdqNqsmm5BnDX5ztP33wzE0whIBu0u86GXyl9s500aFUAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAABwXe4Hxj7HTxYSkRD5vYCEgAAGWLvOo2Ut19FklLYe/lKc/LLUAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAACjvOZz7WkYukANbmxCmyBHoGfU8n02pRge17Wo2tZTdOWgAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAHLdTzGZc1DI0AEWU7yotyXAhIBd0l5o1LBWWGznZgAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAEckc/vgU7GsYOoAAAAB71ND0Gvnq+wadOFN0wiza9gAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAIEoT49LrsVbjyoT87yTzWUZ9IqrOvayEPQAAAAAQCf8AN+1knyT61nI45fXf0Hz2sFf2AAAFl6Qnyz6PICcQIHs7UbVZNNwAAAAAAAAAAAAAAAAAAAAAAAAAAAAABVS89cIv5gSiAA36HO9HnRXtHSFb5e0O+4buegDyuLKuWBXWPoAAQJ7zlyuPT1WPoVrZrpWRlxim2lvwgXRsZ4e/mAAAhTRX2GFeWbRvABoj3dX1ejJvy4+ChZ92anJWFlzq14dYpbnXz/R7+YAAAAAAAAAAAAAAAAAAAGmgsK67nB7VwAAAFvUb/P1v6m2UNTiOyzryxrs55X2AAAAAAAMM3GvYAdAAAAAAAQ9VjXkC/wDg30k6vnJEPC0wpWQAAE+A9IdYr7D6LID0iU1Eds5XUdeAAAAAAAAAAAAAAAACg0Z4aWQHYgFT7GdqJQAA6CHlYZuvAnnJAAAAAAAAAAAAAAAAAAAYUPQ1Ff1qh87rgAAAAb+l5PpdWjvGrRroe6MR1vrLUAAAAAAAAAAAAAAAAHN479GlkB2IFToky/P2kj08QAL/AH45ZuuHJAAAAAAAAAAAAAAAAAAAKC/53PtRRi6QAjd5lv5rofWGwePoAvaKzu1rlXt3MsFeLBXiwV4sFeLBXiwV4sHCc4fXnLdSAAAAAAAAAAAVVd0FBdzvB7VwAAAG/Rb+frPFDUAAAAAAAAAAAAAAAAAAAAUF/U0rFSMLUAAjyDvA50BfUPTaFTcNrOAAAAAApo3RDHIAAAAAAAAAAAFVa+T8+aTId7MCUQABu53K+17KOkHn7AAAAAAAAAAAAAAAAAAACMSdHH6I9sk6D83sB5zAAAAk9HCm72UFzwAAAAAAAAAAAAAAAAAAAAV9glDndfSx7FSiW/k4VOVzJjOss8le0EPQAAAAAAAAAAAAAAAAAAAeHsDXLPkf1ab6Yc502up78ulxMPTCEgB6eWey11KIa1EAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAQ+xmKXR7V+hc5s7y/VM/z9t48/UCrrNlXx23uvZ3gOnlaSY8qQAAAIcxCVJG6RTsczs6JHtRZbVrwD38wAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAGv2h9fDZGLueHYgPfBYW3Mzq1u4FW9SQOqcatrzvPYkeW7FsgAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAGnvKyEaGSEogAAAXE7n+gpaJ5W+NiRolSAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAABW2VN6+EIXs0AR+dkKm2dDsQFpV23hZ8sSnoAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAKS7pvatCF3PAee1sZQ76nt+S9E/MBbVNx42JwpaIAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAACtstM/PnxoZQAAAAC/pugq3ArXQAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAKaF0dFcz9I96wAAAnRnLmmfqByQAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAADXsOUUbpodqnSpuj3raWzPvNHs+w8vaBalS6EfQAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAD/8QAMBAAAgIBAQgBAgYCAwEAAAAAAgMBBAAwBRESExQgUGAQIzEVISIyM0AGJDQ1kKD/2gAIAQEAAQUC/wDR+XLHOqDOqDIeuciYn2plmBwmEfbBSOLtZExMezOfx6K2yuRKDH2Sy3TSzll7G0+BepWPePsVovz1ElwtyZ3QFlDC9fdO9uqM7xu1OsDaC6RKHfw+vF+7te80N6gzudyf4bd0Kc7VTUVTRJFX9eL93bfOIRQ/1z7lvEMYoHAvZNFTPX2xub28kOcaVsPu5YkrltrYpwOj1+0P6tRQ8TfhtcWT1BJn15oca9SqG4fndvjksr4p4N9esK3TpKXzC+3c1AOznMr5ExMetzG+HJleiC5ZIBADoTXJUqsCwvXGVt+EMj2/fF1pnBGBjSakHDxtrYJiY6B2FhhXJzqm51Tci4eBbAsid/psxvya65zpBzpIyKy4yBgdcq/CS7G8+0zFYtsEzvBhLlLxb7CxQND61bAati6ttNxeMOFixksLQid0138yPYdv2BTNC8dGwpoPVYbzD0hKRJZwwPYCAWQ7YtF2Kpzs0NSme4su7RCnlZ3UVj2kZPp3ht+vWy3t1ALhPNoCMUNm/wDWsOvRTs5LCd666d7u0jEMTYFsdyp3p2jaLlbNslyucZ361zqT9dP9/bYDmJq1OmZ3VXAQ+ws/JurCAbW5ja2CQmPr9kdz9T75Ebhwq8gS7EEfrhvSrD2tRDCupu6tYOJ3yxQOHe6tgGLA9El64zqgzqgyLC5yJif6W31rNmztnMvtVWUmuYyB6ddfLX2nX/WFj9foLLMRhGR9sTMYuzORMFGrNqCnpjdja621kpXXVlhPMHSrJ4p72LBobnVsW0HB55z+PRWyVyBwY/L9pgraHdNrjzpibkRER2vr8zJiRnvTWktNleCMbEiXfM7oO2MYVlpZJlOb5yGnGDbOMW8GeOst3zpJZyy+L1rpa7KvTM+ftHVSzIrSzPtomsWQdMowgIfmImcCqwsXXBeqQiY8Dq2KcDh7GNFQsaTZ702pHPv4tp8C9Sse8ct0AtsubMOHrDlrmYGOpJuRV49fhGc4B/pNriwosEqT2she0cYcLAzlhaNd/BPirRfq1Elwt+CIQHqTdkVYKfCXSYNRi2KLYu0+oCw3mM06reIPm9WdaK1VXs19iOv2qSo2Xf8ABOne3Vlgirnsdg1R4vDkInFjZ1MNVR8Dfm5bGoqrTZLlfT/yHaf67vgi/d2m41bQsNZ1XcpCy8XdL89VJcSfi3s5dtsbKiJtUlW8rbPVWZ4Iv3du0VyValPUP7k/w+KtT9fuFok3uqT9HxDo3N7bTWRFdMIR3DG4fFWf+R2u5+5XV9WHHwdqHCpXWpzrU51qc61OdanOtTnWpzrU51qc61OdanOtTnWpzrU5t6yJ1a+2rqMo7Zi3P9e0P6tRI8TfF2o+v3CkQb3VI+jrbRofiCq+xKSMiIGP67Q416lYNw+LujrKHhV4mwrdOkpfMLxjg5itRAcbfFffHJkNFa5ZIBAD4p711lO/yRI5TvX9qWLKuAtOsrlh4xlaJwgIO2ImcXWmciIGPF2WK4GbLsDfqVQp1yGDFq5UWjWRv8f98lC5zpRzpByK64yBgfGTMRHNZYxSQTG6N/wYQwW1yX3xG+U1fWL1zo1Ot36YRPFHa14qmEE6e86yzwqZxkobGclmRWbODTwFgv047IDhWTnOYc5xFkNZGDaLAcB9u2ODp70X+mCRJfzMxEc1ljFJBMetGcALHEzQVY3dm0ajLEWPxC8lQQpXw2wKpiuTZ9bIoATOWFoodwT2zMDHNZZxSATHrlhnEenWZxD8tsCuYrk2fXWlwL1FlwMyZgY5jLOKSCY9etzrdSIhFcmz7BZn6va1y0B+JVO+pu4PYbH83b9oD/ff3VP2ew2f5e26lr0iN4Bjfu7asfT9htxrJjhV7C0eNeoseNnsdhfCenXXwj7GQwYsCVlooTxT7KYQcMSS9BVff7SdcCwqxxkqOM4CyEsnBqlgKAP/AKz/AP/EAC0RAAIABAUCBgICAwAAAAAAAAECAAMEERASIDFQIUATMDJBQlEUIlJhM3CA/9oACAEDAQE/Af8AV5IG8eMn3AYHblZtR7JBN98ZdQR0aN+SqZlhlGqmmfE8lNbM5OoGxvA5E42GS+hPSORmizkY3OIFzbkqmXf9hqppfXMeTm0/uuiXTk9WgC3lKpbaBSzDBpJkMjLvw5UHePAl/UBFXYeXJkeJ19oVQosMSAehifT5f2XbklXMbQqhRYapyZHtyNILvfQCDtjWDYwYP1x9HucTCqFFhjWegcjTNlma6xuoXhGcLvDVX8RH5LwtUfcQkxX28qTN8RdLuEFzDMWNzwcxwgvDMWNzoBINxEqZnHko5Q3ES6lW36QDfB6hFiZNaYbnyQCTYRLpB84CKNhFgYemRv6iZKaWevbVD3a2qS+V/Mv5dPKyC530ugcWMOhRrHtWNzfHIdCm47WSuZwNdYuzYHs2FjbG9pdtCiw7Wk9eJhHzY1X+PtahLPfVJTM/bUptM0WtjVn9LdrMQOLQylTY6ACegiTKyDtgbG8IwYXGqomZ36ds6B94alPxMfjzIWlPvCS1TbuJM7wz/UKwYXGifUfFOGZgu8GqX2j8ofUJORttA1qxXaBVuN4NY31DzXfc8NOm5B/cFixudEmf8W5ImwvDNmNzqp3zLbkag2TEC8FSN8aU/vbkan0YqLnrD403r5GauZCNdKu7clPlZTmGlFLGwhFCiw5Ii8TKb+MGWw3EZTCU7neElhB0/wC3/wD/xAAvEQACAQIEBAYBAwUAAAAAAAABAgMAEQQSIDEFEEFQEyEiMDJAUTNCcRRSYXCA/9oACAECAQE/Af8AV4BO1eG/4oqRv3WOHq1WtzeEHzXucCX9WqdP3dyjFlGoi4t3Ic7+q2hvl3GM3Uc7DmTYX7lA/wC3VO/7e5xzdG0PMBtR8/aZ1QXY02PhFDiEJpJUk+J7OGI2rxX/ADRYnf28VihCLDendnN25hipuKwmM8T0Pv3KRwiljTuXbMdINqw0vixhu48Ra0dvzoZSpsefDW82WhQ/Pb+JfFeYNjencubtz4b+of47jjkzRfxr4cllL9kVS21DD/mvASjhx0pkK7+yResTAYXt00xRGVsopECKFHY0TMbUqhRYaCL71ImQ8umuSNZFytU2BkT4+YogjflFg5ZP8VDAsIsvsswUXNTcQY+UdNI7bmgxG1RY2VN/OoZ0mF1+tCtlvqkXMvt2vQAG3t4zEeI2UbDTHI0bZlqKQSKGHMfSHkOebQd/q4qTJETr4bJunIUOn0htzA9d9B3+rxH9IfzzAubVLHk2PPAfrfVha621SNlX62PW8Ogknfnw5byE/VR8pvSsGFxoJtUj5z9ZlDDKakjMbFTqwcPhx+e5+srFdqGIHWvHSjiB0pnLb/YxWGEw8t6dGQ2bRhMEb55OzBS21DDtX9OfzTRsvtvGrizCm4fEdqHDo+pqPDxx/Edmjjz0ABtolh6r3IC/lSrlFtUy5W7jCPXzvagQduc49PcYPlzY2HlSc5/h3GNsrX14huncoZLix0swUXNM2Y37mk/91B1PWswpplG1O5bf/t//xABAEAABAQQGBgcGBAYDAQAAAAABAgADERIEITAxMlETICJBYHEQQ1BhgZHRI0JSU2KTFECx4SQzNHKhogWQoMH/2gAIAQEABj8C/wCx/E29rlNiariqCay1Z1ajBoL82iOJ4Jw2Pdk0RxLIPGz7t/EhNrLlxGE2o6IlpUP3alZBQPECrYFkuy8KUTRUB7zJd0JKPxQUJdDeObCN/D51nZIGhVUTkW0LsCVONR10s60gMq1Szbgxfuwh2+vQpFRJZ2V4ykR4fOtooTLe7KQy6KvHiB+LXS7XFEcKjcWKHiQpJ3Fp0UcTDMk8QK562ll24QiyVqTtJuOvIoAiECC3sovHfyyaxyLRQbrwbxxAFZ2oHTOCUPBctN7S0kAZPRhPpw+RvtZs9SBaNGrR8pV3hkxAiFi9Crxw9OLrOG7friNShcoXhoUitHzU/wD0NEGI4cg0RhsYBoCxmoxCc3Zwn0aQgoeC9CuHYo8mrENbbqDQAs4LHI7w3tIvXXxgbQ5hgpCgpJ3ixvie5tlPm1/+Gxf4asAtXstVwbW10OTYi2ItvLVCFuXlHVo1m8e6rm2jep0b3I7+WtFTQuTr7JbJWXEMrxIUG9586/3Hq06VApzYrcqiAYdExaJsYhoHFxEHbqKXrwbZG8MHia0nEnMMl67MUqub6RdZgi9gocQwWkKGRDfypDmgwZSXb8qdL91QutSjPolCS8ewmkTl3s7fQlnTGDLdUSiqfyVKM0oZaZFO3rupSFbuHoZWoVl0UlUBEuzEtR/7AynhCXaYxMBeWf016mQvsKMhw8vnrCZQEc2rglRJATHXTyZ9Rk0SkrJTAKSiKWc0ZVFpKClMCpSIJZT+lUKlrkMHSQ7qHexT+GpDqAjF4iHDx56ygEJUrdMxqSRCpW/X0cdsbjxErnbICx3gi8N7X2jv5gFY5hgpJBSbiOID32wGXQV0cyKN6fdU2jWNG9+E7+WfDvtHqE/3Kg1dJR4VtFwFqCL1S1WoyFepKtMQ21F86z94erBSFBSTvHAuJri1xa+DVGP5IP3RmI2XkLmgNl2MSm0DtMEMUm0rvOsXjlWjeb8lcw2jfJ0bzdkrkeAoIrbaOrUWgvzaItpXCdKruwjxb+JeRHy0VJ/dlOCmDsiEAwdOkypHREYhZzqu3WEi0hSci1UXzrL3k+rTO1RHb8BhsarsmiNR3REonKjBRjh14UdGlOdyR4tGkvJ/oFSf3aAEBrTJxfq0CK7CZd2VnpHatG9+Ib+ebB3SE6NZuPuqsIltkRa+HJq1HoqWW2q2vgcj2dIPGz7t/TMBF4rZQnMt/wAfMZnq30zxWZ1IloUZGk+u5Pm0aSvSfTcny9bLaDbBi1aSOmoRavZ5tG852pSoApN4Lezi9dfATtDkWig8xvGrE+TV3ZWEF1jPswm1ly6Ha1PXqFO7i7VBqLK/pjwFdaiuMnf3MlEylQF6jWWiTANCjO5/rVUn92jSF6U5e6PC3whsI8vyU6SUPR76WkpICcngwn0YUQ+K8jl0TFpjZSqw/p2WE2oPSVKIAG8t/DO4j5i6k/u0z9WlV33Dw7FeaJ1pFwgEtB4hSD9Qg34d8fapuPxBu4WkpvGo6dpeSOOsgay1FXQ5kKW8CSiaMwb8IpSg4domUAcRaj6AqDh8ZFIjER7DVbaRaglMLy3sHez8x5UPLe0z0l6v6rhyHZEFJBGRYPkugh4DUUVWoOpMQVLVUhAvUW/F0wzP/dSLnbPgrrHQKWoDkYtJN4DsM6yEKV7J4Ku4s5cuzCNauWul6pMyt0a4cuy0p8bZJ7ulDxT16hSLtGqDf1tN+6yZ5gpOFaTAhi9it49NU7xUT2GdadON2Zgz2lQv2U66eyzrrdi9F+v49kq1i6dOVqUoYobIZDsbtcDstWsNDo4752fy6CeqaMYMNJCbfLdrbU1Z91BP6N1v2Vejdb9lXo3W/ZV6N1v2Vejdb9lXo3W/ZV6N1v2Vejdb9lXo3W/ZV6N1v2Vejdb9lXo3W/ZV6N1v2Vejdb9lXozrR6QKDyNaCnc0NJpE5LraU0Z6DmkTD8wFWoHZnPXW8EYrv1+Zt0O55AFRuaJRpFZraCRAd35gi1mz7MSrwtkju7KnF2+z7t/ZpFqBuv7MiMNjU0B2WXr0wQN8G9i5UvvVU0oUHTlOMoDRGE2kTiPZsUVdzVjVqaK6u5oDswuVp0hWP5YvLCiy7SqxyZLl3uvOZaUtA2U6rt3aF3k15bEWui1Qh2ZE3NCj7KPmkfoxlvN6jeWjCvpgpo3pz16mmeeXDCSETrWqVCe9tNSXbhTqO1oyYhgRcdYJrU8NyE3tNSSDk7GEethke5qiC2AtgV5NhbbV5Nsjg6qtqqmxFsRbEW2hFqjXlqu07WmKxopfiZP43RmjgjSaHF/lklGEirUiTAN/D7Lv5pH6BjLeb1G88NxLZDKwgu7PUdPHJGlcqmTG4saOaKlwlWJZeAsh2LkiHSEQK3huQm9pqSQcnYwj14ciWibKU4daJMAGg42HfzSL+QbZFZvUbzw7AXC0lN41JACt4bkJvaekkKydjCPXh4m1B6IkwAb2Ow7+aRfyDbIvvJvPD6U2yUgFb0ipCb2npJCsnYwj14h8NaZ4qA5N/N/1Ouaq48RHWi2kP9O7OyPiOeurnxF4a2jdKSI4osEpFFAH9zCN+t48RJVbJHERFqBxJMLjaTG88SQLQNlMq7iaBbMZ2EV+XFNVTVVtgLYT5NhLbRg1Q8f/AFn/AP/EAC0QAQABAgQEBQUBAQEBAAAAAAERACExQVFhIDBgcVCBkaGxEMHR8PHhQJCg/9oACAEBAAE/If8A0fxg+V6RyflX8ytHdygJQm3VXrTZU1fduFqWW1Jg+ShSSOfUywS4UiWj5cm175qGHs9SyM9jllu48FDJJ1HqTl3pZZceZPvHD26jm013mzXJs/QEQAxWt/7M6gk28c7cwmoeqCsGlNH2CwG8j71Bpd+/T6l9+J2Vre+V5VcVOSYaBvxqezWQDz7tEsSQ8xtjQzwkd4v0/YG/FCJ/3HyqwYF2Pjj7APY467NY1fgkolMkjAeS9QRfu4lQ5GBLY7VKceSUjjgHuASNYr+2q+H1pPcKDQuiZdQRaRHN0qmX6lwFFkNnU2aBBKx7k197b9P+dA5soy9h24EIAjiNYwMxvd+GHatNCyO+ffDp61t2Ozy00zFQAAsHFMC4vx22sEaZWPi74dqBAS4mfTgIiRpTE+HJipbN0ot6+vJVTi/svV29KUXzYjuam504kkNHkd1Lwy34QVAK7Vd+wzqx0csJNhkLJqOVfqjOBj3PSrzQBJHk2F2l1ZNO6aWyHahonyUT8dVimt8KAShNTo0BAE3r7zKcigzfSrFDuNHQfYc9xC4insNdy/ejLSTW7+fztxTig+asztHHPvIyaEjqENmHJq3k/P8AD9+9a65lhrOlSAIep9Fvo61JvsaclASEzrQ/79RYkQLBouv7rU3Fl+xejomlVgbkcsicCswj26h79tRUg/sC2FYPzbxaj/nNlTC47/Q5Dto1LIrICL0xNIEXZgdJcamZpm3TyIynmoVmmsaERFQXbV+m0p9XsI8rNpiLSTEcJ36ekm7ixpKCUS0MleIVh45rsqco+Nw1o+XynDWgzWR6f3U6GUCu3T3uXEsDaw2moTaypa5nbjCmJ3KFNTU36iMHfzr+8JaF1HKsBP8AzXfJ6UbdJRI9QetHNCQGLUU5I+gh3S3uGTue9T7/ANMsnTvtBlMTnn+NZDgeA9ebNN1wRVYxqOo5NaRcp65k7X2aw8gJ0IsEtZSe16cpfKjOr+9RQsgNv+LH3AMjSXCcvSrjF9vY3oAEyIPnvWKgczz1vFjmtE/sY41bTPAs/Jdsegs5GuVOSrwqyg7UKxJoowsjzmCSsrjuYfSWsV+kTn8NqHNY7EdqMwOwfT+9lYMPKjCtwa8hAoYhWhXNf8X371FJh2ydHR8eUCWwUjbfy5Mv7mqgz24IVLWMlhlfiwJakLsKL5rP5TRhB/umPm9KBCCwBhxDgShwoGTyHAYy5mgAgIOVOJpFtgoZBEGz2HXZ9+QCIAZtW+brgVoFTEj3a3GsDPnWHg+jVmPDiXDscssufBQiSYP0BpE3XhTG7yhjgURADFaUSd5h+7ymsUPSQ/d6qAABAYByY5JvnV4A6Nmvd4PqvDLYrAAKRkN9zTbpAJGsR/YyY9n1pJNhhCE0TJ4c4OWqp49gwOQgP7ooQCMjn4XqTgd6xZeZPvHB2+kuTIiE+VB0aWGNDHqrKy193GnJguqwFY1D+S4+S29SBmNMjtfnNYc5xidyg8K2H/CIwiYnZ1NmiKlt+jWz6tMTiRJbJ+hV5BrSVbvtyr2XeFptES83QqYfrdBspAVgO5nkZ/DegiCuBjtYfWXfwV2k7AJfUzrZjqKou1736SVaByzmXq/DwXeKs40w2pgpAwZlmnwcL0mT60HotLDIk+BybeOcOuoVgKLaLQPkYvYb0gHVx/ED538IUueISVEmNY39i3N0rmHtwDkp6QKBBg0Vob71k53hEH2az5uzTwMUtvxSSIJgs/vvQp+a8Dbx2ekucO1geF9jHOeVT6g81GEPahDNGv8Amh1N58ibQUMStvTwMQG/FkMT9qnlBA3IMeMR2fC5i0A45ULWDXjmjojwmD7zxRhHGYatXmo3dXPj2uA8L+D8cW4TM+1f5ohaOOmG0b2NBNDWxRbFFsUWxRbFFsUWxRbFFsUWxRbFFsUWxRbFFKC6GZM0KhUCyfljRvOGM93L/oi1xHN0KmXwyJOgeNjrKDhbjgnqPPzmnbpsn3qASWbPthR0gYAgP+jWnE71hzJ544O3hnybnakl3hVobvRy3jy4qAADA8MgPHE783R5d4WgESRpXH+HJhuDN0oN/fwt2eMCfipga1I/esbrLjGks3as99h5lhM128Mxq6EtWFIQ5wowFdquhDRjRgYDwyYBYL4/Y3aBedZk3TtR8WuzM5pyNmsm2TrypQdmDXw9AQklabo5Z0ZtOeO5oqCGx4YiQBdXKvWjju3Pu271GBb7ZXdq1igQMXj6rCkqbHYcaABVyKhTyPz6YtLJMiVQ6HQLz740aqQk4o46rl+DdoYQGfdmrvbbkXwN6mBT6VinkX+jNJ92nx8ihkCb59G4VYL+2FYmDsU4716/sVgvnM0LAG1qtMGpwtrFDYT6sqnU+DMROfdpUYSRjplwIzASq2Kw+6x3zd23eo8rfZK7vTcm/wDaY9lyFje9ihEkZPqDvFvSUYryW12CsElny+pUBzfLfY3aDxln3bq7+nTjjCK9ORpyrg3e3E5MCVWAr1Q3XdfLbvTcbEZK6rn07er8/MuZ+HgJgKcQ3dDdojCL/oVu+nT2q+BzdOc+30cmBKrAVhp5t13Xy+9PS8pdK6rn0/Z8znGtwgN3Q3aIoBn9Gt30OoZo6Dix1uJk38q/f+GseIBwCuYx6i+P8cSiJgMWgieYv0HVIGJ6jiuVLFSTS1FUKAKZVsvGE8UDdeolnyOd2UnqLVPLm6Y59upLUfn5lrPw9SMMJr0YOvKtRbhv1NB+1OT5LkLC2UACDDqi63dsKyYdmsS9H6OwDzrUrBG161U1Y/8A1n//2gAMAwEAAgADAAAAEPPPPPPPPPPPPPPPPPPPPPPPPPPPPPPPPPPPPPPPPPPPPPPPPPPPPPPPPPPPPPPPPPPPPPPPPPPPPPPPPPPPPPPPPPPPPPPPPPPPPPPPPPPPPPPPPPPPPPPPPPPPPPPPPPPPPPPPPPPPPPPPPPPPPPPPPPPPPPPPPPPPPPPPPPPPPPPPON9/PPPPPPPPPPPPPPPPPPPPPPPPPPPPPPPPPPPPPPPPPPPPPPPPPPOJ3/fL9/PPPPPPPPPPPPPPPPPPPPPPPPPPPPPPPPPPPPPPPPPPPPPPPPKfffffa/PPPPPPPPPPPPPPPPPPPPPPPPPPPPPPPPPPPPPPPPPPPPPPPPPfffffbNEPPPPPPPPPPPPPPPPPPPPPPPPPPPPPPPPPPPPPPPPPPPPPPPH/AH3/AN99q288888888888888888888888888888888888888888888888e99P99/fQ08888888888888888888888888888888888888888888888699999888o48888888888888888888888888888888888888888888888z899hf8888088887w1988888888888888888888888888888888888888+tt8888888408vAAACx8888888888888888888888888888888888888888888888884UBAAAARc888888888888888888888888888888888888888888888888vAAHIAAjY88888888888888888888888888888888888888888888888pAADgAAQ888888888888888888888888888888888888888888888888sAAAAAAXs48888888888888888888888888888888885McI98888884ofHAAAAGe88M0888888888888888888888888888888vf999vT/e88UM88vFBDJu8888s888+qjww3888888888888888888888l99999/8wEM888888/t88888888sQBAAAAB9880888888888888888880999999V88888888888888888888oAAAAAAQ8ko88888888888888888899Y999188888888888888888888vAADIAAS88888888888888888888Q9999978888888888888888888887AAABAAG8888888M888888888888t/999/v888888888888888888884sJAAAAAf888888888888888888888Nf+sc8888888888888888888884keIAACX8888888888888888888888888888888888888y/7w88Df8wU888sc9v88888888888888888888888888888888888889c99tr8vbg8888888888888888888888888888888888888888888888f9999910c8888888888888888888888888888888888888888888888q99+499tc88888888888888888888888888888888888888888888888r99+y99+888888888888888888888888888888888888888888888888s999999/888888888888888888888888888888888888888888888888869999+e8888888888888888888888888888888888888888888888888/wBMdaL/ADzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzz/xAArEQEAAQIFAQgCAwEAAAAAAAABEQAxECAhQVBRMEBhcYGhwdGRsXCA8PH/2gAIAQMBAT8Q/i8WVFeCq/h5VZ976pFKnAUZKcunXehBJySb038szD+LkvMDM5DakJNOjHIXOJadfjr8e+TedPinVHkPNjHenFiG9BBByLDpXzI7ItyV6UdvpSIw4t2D3oBBbskoM1fIPWgWh9adgxw9vzgLRHZspaCocwYgwSV/qI5JjO9EbRlQSGpbbt5U6MU6X4+c6DFQ1aElY6fo1oT1+OQnp+WIkittOK0PH45GMHfTPKTb54QuXS/ZXin4oe9RmvsRRkoJNy+VzUzvPByvU6dck6IaOduXoJpvpaPl+s8l9aGjW9qASM0oatG6MvhXtB7E0ErRCdT0q0hTpEorQl4fVRD8+7eFjNBO1nB1Z7EUtSm72Z7s9sqG3TDawdD0mnSfD/v67lI9TjrHlPp/tPPJAvU7rPlvrPZPLDoKd03+o7lI9DixFd/V/wB/OSBOh3U6/L5MVBNNKRu/vEz6juvhRzQRtd7tCHUjICzGMdT3WdqjTrkdgloNe7fuzkLlC97MUFhp3YaDQvkrwvel3xRmn17w8DqqlzJjajhTzfrhh5cUWxaN9VoK1xTbw9d6UhnXlxVqDWxFaJpcMMRdUwJcjiLp1wluXoIIOPJlYp33s2qrnIzp1xRQEtWsmMK6jkTPrxABQU5SEjYNsRJ9eRgDPfPLkm2R9souoXtckAhtSzP41eyh7DXRRUW/u+//xAArEQEAAQIEAwkBAQEBAAAAAAABEQAxECAhQVBRYUBxgZGhscHR8DDhcID/2gAIAQIBAT8Q/wCX6IJwlgRxUY9CgCDBB0aMsNIjDxKVLbNKQ8eJQ7MSLekRijUniFhiO/p88vn0yXo5/NGgnEJdj0MQRbUss8RiZb2zQEN78TFPUoRJMTY1tJUv8owA60rAr3H3FMayd59LREk8HuaMJf0/zuAfp1anfLiQWEqQdwef+8SsNBTa65UQlypQvZ7/ANrRqSUa24eond7fjEFYKgDDi/gX96Vqnw+a86PI4eWTq4uQ2qf2uI1fzU4irpfV953S30PD96cEWgUW/wAsBl1P6P4gEbUo3LP7llK7/p1q0mcDSNlAAIsp+g0sUG68vsfed0MlKT8zy+qVgQ0CsFOSkOb9XqKmu7u/xTvAUwjBzb/5Ssu+NJykpSFDr93qcmu5udm60c0wb1FDBH8UXFWRH83nfmef1lINqVYbcDVJ3YrVHW3t79iMBjGJ6x+9+7IYR2VD72PHTOmv3ns/GF0tczb7k7E5DjIhY97e2RynsqQOj2cbwjvo0QOhvzB8sWg7nsvWhmmHsysmyPx85FZX40xXkp7/AJ7K0bKAAS2p9Cx2ZnYSKvVmZpuYfg7MtKraa6lFsmn9XaJToNn4ajdDiE6FABdx9/XBmoE0i6FO1TVk0xHfr4bfvKkhjPBklKSk/daE6r5fVa/qc7vBnUtqNgaZATe5YRs2pVZeHogb0ADNrhZ4jMOmKBLV0TjNLk8RfpxRES0IOvXfFauIwKz2PHxIN6MwBEXEhRkobearAKRu1zU0nP8A7ff/xAAtEAEAAQMDAQYHAQEBAQAAAAABEQAhMUFRYXEgMGCBkaEQULHB0eHw8UCQoP/aAAgBAQABPxD/ANHlAVYDWnIa7fipa3KB92hMh5PzTgXdmPfFciKUnipBMCysPzUiI8geXZ5HYooCSDav5n4omd8B8TAyACVdKeMFlw/r3MkarebP4anyesOz4lV9wTV27sXJbf3KAEESRNfEY2m2C6rFIyKmVde8RzPrP0/HiONWx85x/c97K7CeU2+AYWlEAbrX+GS5GfEDraP0W+3eiiJZK47/AFFDj5RHrZIP1tRUXiKwSwQJurNAliCMwwv7+H+YG+/avoyBOyWYl03rJL+GTokPNPt25BxKbIzSArq1xl8mpndQkEwxvFXO9WuwNEMNml58Pjhme/acLCTquXsZTvFC3ELN3W+pj/Htix9CAreCwSYsLpNE5GDk/fNBIwFq0RRPl4gdQyvVf79q7+LrgJQdQmpDRLSWcJJwyduBfoYERCOayEObN9Qc/QYpzrIhmJd11648QQ+WkdT9fTvbdWeQL/GVhhQHoTaCed6NaQhGWC+XBZKcLFCIIyOE8PR6Weqf0UiKJCd4kXZX+Zfp2CwtAJE2SlUG3gQfylO0ypNcJ+ohsJWi+Hma4bT1PPuztxe2D80ZYCANDtDS2M9ODTcZHUamQytAhxe3Op00b4ROQOEdTw4GIUI6lITXbOvB/Pcl7JlwKFixdWVu9zI9CiyasLtvZus0XGoMI7pbmE3ht4cBAESEdaQWHla3k6V0Lwz037J1wwCVpAbi5fihRj0O7zosjsTuuRrGisSA4lvHO+TRYIh4OE7l5gHO9cUpg7dPoimpOOH3oGeo/BS5CcSv7ypIBnV6ig7XcSR8Gp2rIJKekS3g9sUltzkGtSOgKYH+G0iosrWxPfvUKTI8OOi3QtSBFG6R5cJ1iw1HaVbENVsFNqu5Xer257Q1d+oUAcGXTnk38Qo1IYcJhHImiXK/QQ0cA+mCBDCWRqMJrUJ1ykINlNkhOHeT4K5YsDK2KW9EYGx3KYFSBhGgmAG/Hc8RSYtn8SCWTu3ATFLTAT7TfRkfstG7KXbZ2TCaJSuXlDfd8+7m6GRrCMbm7U8QtXbIG8mskfVR6XeykmAg2/KxcskNHTvVdWfJZ9vp8Hm4UhHmIlrMLxXodF6YmCfQqPz967gGRtbzqIcN0RmEdSze30nw65rY25bvtHe5EB6qEAjI3GiPmkQDgXKEvr8Fk3SJVdhljrWhKNNwtqItxz4e6Kx0GPt2hUPLiNgnW1SmEgBYULOi47c5sqS8hFJWaFRt2NwmG2jQg0NQd4zeILUqhIt7lKS4GdPSOQ4joIMst5jh8PKXdfqdpKEsxOE3HGaIJcc2MTRsnNpi/bneEqTGpuRCSbZ8RKnp9V75ECJA5cHdXyVhJcWhwLnH1GagBqIDcTPiCaxYh9Ifce9cVKQFHiADyI+Ef6Fq+qGbidzCo0eTOMDI7dC5qHh1xFS6QXqlTdl/pFqSuKROiCYKzoGveqyT6oY9/p2LowJJBYBcNERKw+aY+Itb+4NBGCZAfAgIgBdVxTcLNj9lqCxy0D70rB6A/elYUuYrm+in/iNuOdtMxIXMFnZVuhrNh7i9svKaFnIrlOuqaMe7E7mj3ikiIuPY/t+0pJ3MZPypdIQN4tRt2Msi1sHUgamvgFQJWApdIOy8Om9cHCTY8uzzfJRSpwoudTWgK4yd8lrEIjZtGpA2qRPe98eHProqJFhCE6IiBGEtkqFTbVbq6rqvwgAI7cdvxSKAiWR07p7qMpr36HcarsQJ0TZNEuVA6+IXC4Nnzu1GG5U2IZBuGognz5GwCVdCmjRts9fHHczylN5t+1SFFkcjs9hxguyOKJSi8SaeXZURAAlXSo1SqmjkE6CtwpKUmmnxJ6lHZRvJgwDYDHamtxmD8HmrNREIe4YJmW3V2KBAAgAsHdKIRBMiYwx1hNEq+9BB7PhcTtC/cBhaVICnV28/0anC+0Me+aRlXlaCZCetRcEaTT0pgNTW9wt7VGzLgXpv8uVLivNXby7sWlbPBv1KBIISJqfCFNo7tkI21eCplwtuzDfYwfvsHAaUQBu0ye4eq9i+X1FKDs3lfc+t0FGWFAIA2O5h3RgWHRpN0B/wH2pe3bpHr8eNGMtLjulX9Ck5A+T0NO9j9qMDZGpEgsyA5lvPO2isiLY1Jd+BOzOiViyvxzVkkP8AAu89vFNkwCv1dyjrCkDInyuOE+tf0+VKoiqyrr3jIZv/AMbff4EgacKhLKkbaRWAERQLC9y2rCasZtdR5oJXw4G6uKXSHaZPmD0atlALFhNnUR4WjRKAAAAWA756WtxawR6CgBAAGh/wlRSIwzZbcQ5tDelEQSougrdtyNCghgBmLEnll0UN4q/LFtRaFTisBoNjukKY9lct+m/yuHW3nD+vr3t3rvJNviTXoeDdWxQqamn1Rj0wVGO5AlcVnDA+Sk9tS6bJTAzF5xTmcZfShCr6tz7nzqWdy+jSPK6Sur3iNOOy66fTHp2BQESLRhEcpTOsULWoq2SjixsSa0kCddMEIXiB71btEOFfFGX2d/kfCz9Fvt3uGaHigdmGq2q4tPrTa960VLv2GJeH1ody+UY+2GnUagHcSEpyELOnevNx6yz2AMFyZwON3TrBScMjMPq7vqzSFxOzzgc/QpD0s5mAq8WfR+R8xN9+1PWdJdREs2y0hH8zDQuMTCT2xoz5uARKMu6Au/yuQLkR1sfR76QWWCeSz9PjeR1YTdlSOkiWq4HBiQetStuRcc/sGrevYDaYAPL5HwMz37Sq2BMmr2v5UwUWUCH1Me/biHX8rT/Kifv297XcCySHXtzX+MH79/IITd/6lUIljzv9+1HWqNYFwyZjpSIFiWpdeq9qJYK/x3D5XO5v9DtcnGjjSN1cEOuuG+MzXGvWu4m/r2nxpthyTYZMxPfmzZs2bNmzZs2bNmzcHYhFrNhpvgZrliBPp9S1IwUIcoAfQ9f+iGS0zqfp9u9stZ5Av8s/rkI+3bYzuwQjBFp93t/1ksfbv5hjR5wQAkvuqzymFnpiHUaNT8CA4DH/AEQ4W+i/o86RSJCWR7xh8Wel+X6fLFkC109z79875AIct33flTveNhq38+7CxS5wbdWjaAIA0PlkasfJX/XnSQw57xmCV8k/Njz+VmWBCOpSsF9cvF457kOcHOWFG1AXVyt35Wt0QiFWCwXMFQC2wPqgSfamEVulgzuQiwukMW7lZ3Cec+veJZWQyND5YgEQRyNMptTI6bVMU7pZ6PZEvWAS0scpJXV0o+uMHywAHqRCy5DkgN5ig8WAs3lQJLhYLli5ViBSyEZeX2ANKn1HDuclKRVXjsO6ZzLj6t3j5e4EmRJGn5v7qPbFPfVEftWoHSClxVH9WiuA2QfLAPqpQAyroU6o7SEef+xCo44SzfHu8GDABalUKyKgkUHZg9D4zVTDqO5SAHaMdTTtplaAJWooil8h/NqiCDwvMnb9MSXQ/XWgO8ZRUDpgoW12zUFAG3Ek7QkLSAHvtuIBvNqK0h5VLi2EtwDQN6wQdrNMrW4B8sUyulCZe9vekIX/ABpQj9xUFcdyP3U4CGo/d/FStGquvPwaoFUAytMsx7vq/FLJ1IPq0xfecoNkl66/sY602G7rn4qB8gL++zJm6oJwrZCbzuUgJ+oAIsRFk2elNchWCwYeUdh1FhwBlVwVuY1CPN/JBUSUJVvj3foYAPDb6EYAytilwroJbz37YoiMJqU8lxGrq3oGZCRMPxOMyXMMtMTB+pkWgAEhRY8yxr05pc1ZXKACfT4xouCDxKbb6BzNqJcycraMoS3AGgb+HGvgPNdqcrbGgNu6YZyQLr/HaTXwcAyq4KnQrMWY9Z0uBVlxxVvzuvYwQW8Oq14R11P27xUmFZdf0/HYsWnMhyW5wOral8iJ9cws3LYg0GfD0cMR81sUssvePJ2Mclms0mvg4BlVwVrtbBesOKNgXpF99TfHddcYIPD9kOVZ7H377GcgkBi5a1kDq2or6p1Uws3PYg0yeIelp9X79ohMycim0BdGuZVEAmG/aszY4GARLrF/EUvK+h2jhCVGAN6ctIDWPKNmn+9uXBD6eIuvp9T7dosFH1Bgt2JoVdrcArhGz8kTp2pn1foeIpSGFb7n375WCG51b/fxFHheT1mKSGHPePFTKelnxIuQE201H37x0gFadP2z6eJBvkvTmlZyZ0R3SlEaQ6/x4mWyshMjuUkg6IY67dsFQBVwFPAcpreu1AgAEAaeJ0ERJHRpxnfb9H4qTfJB9Gko8g36UOwSdVY9dH3UyO1Ln4okZ+s/Ty/+s/8A/9k=\n", "text/plain": [ "" ] }, "execution_count": 8, "metadata": { "image/jpeg": { "height": 600, "width": 600 } }, "output_type": "execute_result" } ], "source": [ "Image('images/computation_graph_example.jpg', width=600, height=600)" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "Remember that the name of the variables (x, y and product) should be regarded as the output of the operations and not the opration themselves. Note that for some arithmetic and logical operation it is possible to use operator overloading instead of using `tf.`. For example the multiplication could be written `product = x * y`.\n", "\n", "It is possible to create additional graphs and control how operations and variables get associated with them. `tf.Graph()` creates a new graph. To check which graph is currently se as default we use:" ] }, { "cell_type": "code", "execution_count": 9, "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "\n" ] } ], "source": [ "g_default = tf.get_default_graph()\n", "print(g_default)" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "Now we create a new graph g." ] }, { "cell_type": "code", "execution_count": 10, "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "\n" ] } ], "source": [ "g = tf.Graph()\n", "print(g)" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "As you can see they are diffrent objects. Also, given a node, we can see to which graph is associated:" ] }, { "cell_type": "code", "execution_count": 11, "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "False\n", "True\n" ] } ], "source": [ "print(x.graph is g)\n", "print(x.graph is g_default)" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "Using `with` statement we can control which graph is set as default and add new nodes to it. For example we can do:" ] }, { "cell_type": "code", "execution_count": 12, "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "False\n", "True\n", "True\n" ] } ], "source": [ "print(g is tf.get_default_graph())\n", "\n", "with g.as_default():\n", " print(g is tf.get_default_graph())\n", " \n", "print(g_default is tf.get_default_graph())" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "TensorFlow uses graphs as an abstraction for various reasons. The main reason is that Neural Networks have a natural graph structure. Moreover graphs can split computation into small differentiable pieces, that allows TensorFlow to compute derivative of any node w.r.t. any other node in the graph. Neural Network learning algorithm is based on gradients. Finally having the computation separated makes it much easier to distribute work across multiple CPUs, GPUs, and other computational devices. (There is one more reason that we will see briefly: save computation, only runnig subgraphs that are connected to the value to fetch)." ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "### 2.2 Sessions" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "Once the computation graph is defined we can use it in a **Session** to actually obtain the result. Session objects are part of TensorFlow API that are responsible for the communication between data and objects and compuational resources. The computation is performed with the method `run()` of the session object. Without arguments `Session` uses the default graph. " ] }, { "cell_type": "code", "execution_count": 13, "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "[20.0]\n" ] } ], "source": [ "sess = tf.Session()\n", "result = sess.run([addition])\n", "print(result)\n", "sess.close()" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "It is necessary to close a session after last operation, to release computational resources; closing a session is done with the method `close()`. It is possible to use `with ... as ...` statement, that takes care of releasing resources when computation is finished.\n", "\n", "Tensorflow translates the graph into executable operations and it distributes the computation automatically on available resources. It uses the available **GPU** for as many operations as possible.\n", "\n", "It is possible to use a **specific device** for a session with `with tf.device(\"/gpu:1\"):` statement. For example previous command execute the graph on the second GPU of the machine. Try to change the string and execute the code on CPU (or another GPU of your machine)" ] }, { "cell_type": "code", "execution_count": 14, "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "[20.0]\n" ] } ], "source": [ "with tf.Session() as sess:\n", " with tf.device(\"/cpu:0\"): # optional\n", " result = sess.run([addition])\n", " print(result)" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "In the these example we requested a particular node of the graph, by passing it to the method `run()`. This is called **fetch** and the arguments of the function are the fetches. It is possible to fetch more than one variable by passing them simultaneously to the run() command (`session.run([var1, var2])`). The result is a list of values fetched. \n", "\n", "As we said earlier, TensorFlow computes only the necessary portion of the graph if we ask to compute only the product, only the output of node product is computed (involving only the nodes to which it depends)" ] }, { "cell_type": "code", "execution_count": 15, "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "[12.0, 20.0]\n" ] } ], "source": [ "with tf.Session() as sess:\n", " result = sess.run([product, addition])\n", " print(result)" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "In an interactive environment like IPython or Jupyter it could be useful to interleave graph construction and run operations, you can use `InteractiveSession`. The only difference with a regular `Session` is that an `InteractiveSession` installs itself as the default session on construction. The methods `Tensor.eval()` and `Operation.run()` will use that session to run ops." ] }, { "cell_type": "code", "execution_count": 16, "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "[0. 0.25 0.5 0.75 1. ]\n" ] } ], "source": [ "sess = tf.InteractiveSession()\n", "c = tf.linspace(0.0, 1.0, 5)\n", "print(c.eval())" ] }, { "cell_type": "code", "execution_count": 17, "metadata": { "collapsed": true }, "outputs": [], "source": [ "g = tf.get_default_graph()\n", "operations = [op for op in g.get_operations()]" ] }, { "cell_type": "code", "execution_count": 18, "metadata": {}, "outputs": [ { "data": { "text/plain": [ "['Const',\n", " 'Const_1',\n", " 'Mul',\n", " 'Const_2',\n", " 'Add',\n", " 'LinSpace/start',\n", " 'LinSpace/stop',\n", " 'LinSpace/num',\n", " 'LinSpace']" ] }, "execution_count": 18, "metadata": {}, "output_type": "execute_result" } ], "source": [ "[op.name for op in operations]" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "Running an operation makes tensorflow running all the operations required for the input of this operation, but doesn't return anything. It is also possible to get tensors and operations by name from a graph. \n", "\n", "Now we close interactive session." ] }, { "cell_type": "code", "execution_count": 19, "metadata": { "collapsed": true }, "outputs": [], "source": [ "sess.close()" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "**NOTE:** the session runs the default graph. Using multiple graphs is generally avoided, each graph will require a separate sessions and each session tend to use all available resources. Moreover passing data between them is a bottleneck because usually require using python/numpy and does not work in a distributed environment. It is better to have a disconnected subgraph within the default graph." ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "### 2.3 Tensors" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "TensorFlow uses **Tensors** to represents all data. Only tensors are passed between ops in the graph. Tensors are an n-dimensional arrays, and, in TensorFlow, are described with rank, shape and type. The rank its the number of dimensions (different from matrix rank), the shape is the number of elements for each dimension and the type is the data type assigned to the tensor." ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "#### 2.3.1 Tensors as results of Operations" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "When we create an operation for example with tf.add(), the operation is a *node* added to the graph but the handle we get is the tensor that results from the operation. This handle is the *edge* of the graph and it passes (**flow**) the yet-to-be-computed result of the operation to other nodes.\n", "\n", "Tensors objects have attributes and operations. For example the tensor `x` is a tensor with no shape (actually shape 1x1) that is a scalar, and it's dtype is a float32:" ] }, { "cell_type": "code", "execution_count": 20, "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "Tensor(\"Const:0\", shape=(), dtype=float32)\n" ] } ], "source": [ "print(x)" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "Tensors dtype can be specified at creation time or can be modified later using `tf.cast` operation. Tensor can be viewed as n-dimensional arrays. As we have already seen a $1 \\times 1$ tensor is a scalar, a $1 \\times n$ tensor is a vector, a $n \\times n$ tensor is a matrix, a $n \\times n \\times n$ tensor is a three dimensional array, and so on (it can be generalized to any dimension). As with dtypes, TensorFlow can infer automatically the dimension of a tensor object given the shape of the data. For example if we want to create a matrix with `tf.constant` and see what shape we get, we can use the following code:" ] }, { "cell_type": "code", "execution_count": 21, "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "(2, 3)\n" ] } ], "source": [ "w = tf.constant([[1,2,3], \n", " [4,5,6]])\n", "print(w.get_shape())" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "Other types of operations that generates number without user input are random number generation operators, linspace (as seen before), array of constant (one or zero values) and many others. They resemble the operations and API used by numpy. \n", "\n", "An important operation that we will use often is matrix multiplication. It is needed to perform operations like $Ax=b$, where $A$ is a matrix and $x$ is a vector." ] }, { "cell_type": "code", "execution_count": 22, "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "(2, 3)\n", "(3, 1)\n" ] } ], "source": [ "A = tf.constant([[1,2,3],\n", " [4,5,6]])\n", "print(A.get_shape())\n", "x = tf.constant([[1],[0],[1]])\n", "print(x.get_shape())" ] }, { "cell_type": "code", "execution_count": 23, "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "[[ 4]\n", " [10]]\n" ] } ], "source": [ "b = tf.matmul(A, x)\n", "with tf.Session() as sess:\n", " print(b.eval())" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "**NOTE:** if you experience warnings, it means TensorFlow wasn't compiled to use SSE4.1 instructions, but these are available on your machine and could speed up CPU computations. To suppress them use: \n", "\n", ">`import os\n", "os.environ['TF_CPP_MIN_LOG_LEVEL']='2'`" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "Tensors have a **name** as well, it can be assigned at creation time or TensorFlow assign it for you (as happened before). It can be displayed with method `name`. The name of the objec is simply the name of the operation followed by a colon and the index of that tensor in the output of the operation that produced it. If two or more operations exists, tensorflow add an underscore to its names followed by a progressive number to avoid tensors with the same name. In following code we will specify the name of the operation at construction time." ] }, { "cell_type": "code", "execution_count": 24, "metadata": {}, "outputs": [ { "data": { "text/plain": [ "'MatMul:0'" ] }, "execution_count": 24, "metadata": {}, "output_type": "execute_result" } ], "source": [ "b.name" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "Sometimes can be useful, especially in complicated graphs, to group together related tensors, that belong to a certain portion of the graph for example. **Name scopes** are used for this purpose. Here's an example." ] }, { "cell_type": "code", "execution_count": 25, "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "variable_a1:0\n", "group/variable_a1:0\n", "group/variable_a2:0\n" ] } ], "source": [ "with tf.Graph().as_default():\n", " a1 = tf.constant(4, dtype=tf.float64, name='variable_a1')\n", " with tf.name_scope(\"group\"):\n", " group_a1 = tf.constant(4,dtype=tf.int32, name='variable_a1')\n", " group_a2 = tf.constant(4,dtype=tf.int32, name='variable_a2')\n", "\n", "print(a1.name)\n", "print(group_a1.name)\n", "print(group_a2.name)" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "TensorFlow provides an interface similar to numpy, for example it implements broadcasting. Note that `a` is a list with 2 elements while `b` is a matrix 2 by 2. `a` is broadcasted (repeated) to match the shape of `b`." ] }, { "cell_type": "code", "execution_count": 26, "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "[[0 2]\n", " [4 6]]\n" ] } ], "source": [ "a = tf.constant([2, 2], name='a')\n", "b = tf.constant([[0, 1], [2, 3]], name='b')\n", "x = tf.multiply(a, b, name='mul')\n", "with tf.Session() as sess:\n", " print(sess.run(x))" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "Like numpy TensorFlow provides a way to create tensors filled with specific values. Next example create a tensor of shape (2, 3) filled with zeros." ] }, { "cell_type": "code", "execution_count": 27, "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "[[0 0 0]\n", " [0 0 0]]\n" ] } ], "source": [ "zeros = tf.zeros([2, 3], tf.int32, name='zeros')\n", "#zeros = tf.identity(zeros, name='add_zeros')\n", "with tf.Session() as sess:\n", " print(sess.run(zeros))" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "It is also possible to create a tensor with same shape and type of an existing tensor, for example we can create a tensor similar to the previous but filled with ones this time." ] }, { "cell_type": "code", "execution_count": 28, "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "[[1 1 1]\n", " [1 1 1]]\n" ] } ], "source": [ "ones_like = tf.ones_like(zeros, name='ones')\n", "with tf.Session() as sess:\n", " print(sess.run(ones_like))" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "It is also possible to use a specific value to fill the tensor with.\n", "\n", "Next we will create a constant tensor, but this time it will have a sequence of values. " ] }, { "cell_type": "code", "execution_count": 29, "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "lin_space: [10. 11. 12. 13.]\n", "range: [0 1 2 3 4]\n" ] } ], "source": [ "lin_space = tf.lin_space(10.0, 13.0, 4, name='linspace_example')\n", "ran_ge = tf.range(5, name='range_example') \n", "with tf.Session() as sess:\n", " print('lin_space: ', sess.run(lin_space))\n", " print('range: ', sess.run(ran_ge))" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "What if we want to generate random number? (We will see it is specially usefull for initialize Neural Networks weights). For example we can sample number from a normal distribution or a truncated normal, that doesn't return any values more than two standard deviations away from its mean." ] }, { "cell_type": "code", "execution_count": 30, "metadata": { "collapsed": true }, "outputs": [], "source": [ "truncated_normal = tf.truncated_normal((10000,))\n", "with tf.Session() as sess:\n", " values = sess.run(truncated_normal)\n", "hist, edges = np.histogram(values, density=True, bins=50)" ] }, { "cell_type": "code", "execution_count": 31, "metadata": {}, "outputs": [ { "data": { "text/html": [ "\n", "
\n", "
\n", "
" ] }, "metadata": {}, "output_type": "display_data" }, { "data": { "application/javascript": [ "(function(root) {\n", " function embed_document(root) {\n", " \n", " var docs_json = {\"74b8b447-e2a6-4b27-9505-ee870fd69d64\":{\"roots\":{\"references\":[{\"attributes\":{\"callback\":null},\"id\":\"68b1a027-87d4-4a50-b59f-3c2e7621089b\",\"type\":\"DataRange1d\"},{\"attributes\":{},\"id\":\"4354700d-4b90-43cd-b53e-706ac7d627d1\",\"type\":\"BasicTicker\"},{\"attributes\":{\"source\":{\"id\":\"42997811-208e-4b96-ad37-7065c79f60cf\",\"type\":\"ColumnDataSource\"}},\"id\":\"2f0a422b-9f90-4af7-9c57-37252edcc7ac\",\"type\":\"CDSView\"},{\"attributes\":{\"callback\":null,\"column_names\":[\"left\",\"right\",\"top\"],\"data\":{\"left\":{\"__ndarray__\":\"AAAA4IX8/79xPQoX8LT+v+F6FE5abf2/UrgehcQl/L/C9Si8Lt76vzMzM/OYlvm/pHA9KgNP+L8UrkdhbQf3v4XrUZjXv/W/9ihcz0F49L9mZmYGrDDzv9ejcD0W6fG/SOF6dICh8L9wPQpX1bPuv1K4HsWpJOy/MjMzM36V6b8UrkehUgbnv/YoXA8nd+S/1qNwffvn4b9wPQrXn7HevzAzM7NIk9m/9Chcj/F01L9wPQrXNK3Ov/AoXI+GcMS/8Chcj7BntL8AAAAAAKwxPwApXI8Ii7Q/AClcjzKCxD9wPQrX4L7OP/goXI/HfdQ/ODMzsx6c2T9wPQrXdbreP9ijcH1m7OE/+ChcD5J75D8UrkehvQrnPzQzMzPpmek/VLgexRQp7D9wPQpXQLjuP0jhevS1o/A/2KNwvUvr8T9oZmaG4TLzP/YoXE93evQ/hutRGA3C9T8Wrkfhogn3P6RwPao4Ufg/NDMzc86Y+T/E9Sg8ZOD6P1K4HgX6J/w/4noUzo9v/T9yPQqXJbf+Pw==\",\"dtype\":\"float64\",\"shape\":[50]},\"right\":{\"__ndarray__\":\"cT0KF/C0/r/hehROWm39v1K4HoXEJfy/wvUovC7e+r8zMzPzmJb5v6RwPSoDT/i/FK5HYW0H97+F61GY17/1v/YoXM9BePS/ZmZmBqww87/Xo3A9Funxv0jhenSAofC/cD0KV9Wz7r9SuB7FqSTsvzIzMzN+lem/FK5HoVIG57/2KFwPJ3fkv9ajcH375+G/cD0K15+x3r8wMzOzSJPZv/QoXI/xdNS/cD0K1zStzr/wKFyPhnDEv/AoXI+wZ7S/AAAAAACsMT8AKVyPCIu0PwApXI8ygsQ/cD0K1+C+zj/4KFyPx33UPzgzM7MenNk/cD0K13W63j/Yo3B9ZuzhP/goXA+Se+Q/FK5Hob0K5z80MzMz6ZnpP1S4HsUUKew/cD0KV0C47j9I4Xr0taPwP9ijcL1L6/E/aGZmhuEy8z/2KFxPd3r0P4brURgNwvU/Fq5H4aIJ9z+kcD2qOFH4PzQzM3POmPk/xPUoPGTg+j9SuB4F+if8P+J6FM6Pb/0/cj0KlyW3/j8AAABgu/7/Pw==\",\"dtype\":\"float64\",\"shape\":[50]},\"top\":{\"__ndarray__\":\"rRBTKPZIsT9TA63xyj6yPzPOFBceFrY/omJVEdavtz85IBcA/ny8P7iL1gVG47o/ub/fjwT3wD9XQCe4ewHEP0G6G0Xdhsc/iLPIqccByD96ajf92UnNP/kd+V4hU9A/Y2o3/dlJzT+b18lXR1PSP3qeQIeYXdM/EqmEGLCv1T8LM4GBUPfUP85Ka0mTU9Y/dBg1FPri1j/vmHw8ce3ZP3+3uZ/Jzto/vwHanKWb2z/8hINqMF7bP2Hfq0NL7dc/ZggtOLsg2z8w45w5TbraP6OL1gVG49o/IA+A09Cl2j/vmHw8ce3ZP0mSKaFbaNo/vp/P14Zy2T+or9ezxTTVP44AS0y3htU/RRuXuQ2b0z96nkCHmF3TP9nkb45yXdE/aI2pWmuG0T9jajf92UnNPyVJmPRtsM8/sHfdMwVUzD/w/eimo87IP27buk1JIMU/uuhghHQqxD8VCgCN4MPBP4sFy5KnaL4/JCAXAP58vD9dfaF+LMS1P7m/348E97A/UwOt8co+sj8FE3HJ0nKtPw==\",\"dtype\":\"float64\",\"shape\":[50]}}},\"id\":\"42997811-208e-4b96-ad37-7065c79f60cf\",\"type\":\"ColumnDataSource\"},{\"attributes\":{\"active_drag\":\"auto\",\"active_inspect\":\"auto\",\"active_scroll\":\"auto\",\"active_tap\":\"auto\",\"tools\":[{\"id\":\"3e6866eb-8d0c-4cab-9bcd-29b7b5cc8742\",\"type\":\"PanTool\"},{\"id\":\"0e5d33c4-7d8e-4d7d-b2c0-d92cb20eeb8c\",\"type\":\"WheelZoomTool\"},{\"id\":\"fca792ca-d58a-438f-8461-e76738505285\",\"type\":\"BoxZoomTool\"},{\"id\":\"b8b4c59d-39a3-4fc7-b3d2-ce12f97f2dc0\",\"type\":\"SaveTool\"},{\"id\":\"0af35c2e-2bc9-43e0-8109-e5d860e9edb0\",\"type\":\"ResetTool\"},{\"id\":\"c21cd0b1-0e53-4f5c-b4f9-53660cda79f8\",\"type\":\"HelpTool\"}]},\"id\":\"db1dd093-9e37-433a-9f1b-7a9b7c47cce6\",\"type\":\"Toolbar\"},{\"attributes\":{},\"id\":\"381e7a20-ecfb-4a17-b55f-e9b58783d240\",\"type\":\"LinearScale\"},{\"attributes\":{\"dimension\":1,\"plot\":{\"id\":\"5cbe84c2-e474-477a-89af-6aa4f7dab369\",\"subtype\":\"Figure\",\"type\":\"Plot\"},\"ticker\":{\"id\":\"c542924e-ffa9-43fa-9fea-52c46703bc95\",\"type\":\"BasicTicker\"}},\"id\":\"5c0fa66f-333a-40cc-a722-b9925f3191ca\",\"type\":\"Grid\"},{\"attributes\":{},\"id\":\"0e5d33c4-7d8e-4d7d-b2c0-d92cb20eeb8c\",\"type\":\"WheelZoomTool\"},{\"attributes\":{\"plot\":{\"id\":\"5cbe84c2-e474-477a-89af-6aa4f7dab369\",\"subtype\":\"Figure\",\"type\":\"Plot\"},\"ticker\":{\"id\":\"4354700d-4b90-43cd-b53e-706ac7d627d1\",\"type\":\"BasicTicker\"}},\"id\":\"aed9f51b-478e-4519-946b-28063d1c5ee4\",\"type\":\"Grid\"},{\"attributes\":{\"overlay\":{\"id\":\"c6f4288c-cb51-4b87-b2b2-a77ad4cfeddd\",\"type\":\"BoxAnnotation\"}},\"id\":\"fca792ca-d58a-438f-8461-e76738505285\",\"type\":\"BoxZoomTool\"},{\"attributes\":{\"data_source\":{\"id\":\"42997811-208e-4b96-ad37-7065c79f60cf\",\"type\":\"ColumnDataSource\"},\"glyph\":{\"id\":\"55925f2e-797b-46d7-828a-ba3b578c8033\",\"type\":\"Quad\"},\"hover_glyph\":null,\"muted_glyph\":null,\"nonselection_glyph\":{\"id\":\"de6c2666-ba2e-4ced-8531-4e52f813dca2\",\"type\":\"Quad\"},\"selection_glyph\":null,\"view\":{\"id\":\"2f0a422b-9f90-4af7-9c57-37252edcc7ac\",\"type\":\"CDSView\"}},\"id\":\"627e66b6-5b85-4b87-a400-4dd3459ef880\",\"type\":\"GlyphRenderer\"},{\"attributes\":{},\"id\":\"89ee60cc-8680-45db-8e47-0c3c31b1199a\",\"type\":\"BasicTickFormatter\"},{\"attributes\":{\"bottom\":{\"value\":0},\"fill_alpha\":{\"value\":0.1},\"fill_color\":{\"value\":\"#1f77b4\"},\"left\":{\"field\":\"left\"},\"line_alpha\":{\"value\":0.1},\"line_color\":{\"value\":\"#1f77b4\"},\"right\":{\"field\":\"right\"},\"top\":{\"field\":\"top\"}},\"id\":\"de6c2666-ba2e-4ced-8531-4e52f813dca2\",\"type\":\"Quad\"},{\"attributes\":{\"below\":[{\"id\":\"9f5eedf7-5137-4174-b24e-2301824a01bf\",\"type\":\"LinearAxis\"}],\"left\":[{\"id\":\"d4e25d71-75e8-4f64-a5b0-1924f29d8f4f\",\"type\":\"LinearAxis\"}],\"plot_height\":350,\"renderers\":[{\"id\":\"9f5eedf7-5137-4174-b24e-2301824a01bf\",\"type\":\"LinearAxis\"},{\"id\":\"aed9f51b-478e-4519-946b-28063d1c5ee4\",\"type\":\"Grid\"},{\"id\":\"d4e25d71-75e8-4f64-a5b0-1924f29d8f4f\",\"type\":\"LinearAxis\"},{\"id\":\"5c0fa66f-333a-40cc-a722-b9925f3191ca\",\"type\":\"Grid\"},{\"id\":\"c6f4288c-cb51-4b87-b2b2-a77ad4cfeddd\",\"type\":\"BoxAnnotation\"},{\"id\":\"627e66b6-5b85-4b87-a400-4dd3459ef880\",\"type\":\"GlyphRenderer\"}],\"title\":null,\"toolbar\":{\"id\":\"db1dd093-9e37-433a-9f1b-7a9b7c47cce6\",\"type\":\"Toolbar\"},\"x_range\":{\"id\":\"c668e897-eb5c-47c0-a8b0-e22ac5feed58\",\"type\":\"DataRange1d\"},\"x_scale\":{\"id\":\"381e7a20-ecfb-4a17-b55f-e9b58783d240\",\"type\":\"LinearScale\"},\"y_range\":{\"id\":\"68b1a027-87d4-4a50-b59f-3c2e7621089b\",\"type\":\"DataRange1d\"},\"y_scale\":{\"id\":\"7f6cd576-ac9c-4452-88b4-bbaed652af8c\",\"type\":\"LinearScale\"}},\"id\":\"5cbe84c2-e474-477a-89af-6aa4f7dab369\",\"subtype\":\"Figure\",\"type\":\"Plot\"},{\"attributes\":{},\"id\":\"7f6cd576-ac9c-4452-88b4-bbaed652af8c\",\"type\":\"LinearScale\"},{\"attributes\":{\"bottom_units\":\"screen\",\"fill_alpha\":{\"value\":0.5},\"fill_color\":{\"value\":\"lightgrey\"},\"left_units\":\"screen\",\"level\":\"overlay\",\"line_alpha\":{\"value\":1.0},\"line_color\":{\"value\":\"black\"},\"line_dash\":[4,4],\"line_width\":{\"value\":2},\"plot\":null,\"render_mode\":\"css\",\"right_units\":\"screen\",\"top_units\":\"screen\"},\"id\":\"c6f4288c-cb51-4b87-b2b2-a77ad4cfeddd\",\"type\":\"BoxAnnotation\"},{\"attributes\":{},\"id\":\"0af35c2e-2bc9-43e0-8109-e5d860e9edb0\",\"type\":\"ResetTool\"},{\"attributes\":{\"formatter\":{\"id\":\"89ee60cc-8680-45db-8e47-0c3c31b1199a\",\"type\":\"BasicTickFormatter\"},\"plot\":{\"id\":\"5cbe84c2-e474-477a-89af-6aa4f7dab369\",\"subtype\":\"Figure\",\"type\":\"Plot\"},\"ticker\":{\"id\":\"4354700d-4b90-43cd-b53e-706ac7d627d1\",\"type\":\"BasicTicker\"}},\"id\":\"9f5eedf7-5137-4174-b24e-2301824a01bf\",\"type\":\"LinearAxis\"},{\"attributes\":{},\"id\":\"c21cd0b1-0e53-4f5c-b4f9-53660cda79f8\",\"type\":\"HelpTool\"},{\"attributes\":{\"formatter\":{\"id\":\"85a42ee0-1672-412a-a2e2-a47fcce7b755\",\"type\":\"BasicTickFormatter\"},\"plot\":{\"id\":\"5cbe84c2-e474-477a-89af-6aa4f7dab369\",\"subtype\":\"Figure\",\"type\":\"Plot\"},\"ticker\":{\"id\":\"c542924e-ffa9-43fa-9fea-52c46703bc95\",\"type\":\"BasicTicker\"}},\"id\":\"d4e25d71-75e8-4f64-a5b0-1924f29d8f4f\",\"type\":\"LinearAxis\"},{\"attributes\":{\"bottom\":{\"value\":0},\"fill_color\":{\"value\":\"#1f77b4\"},\"left\":{\"field\":\"left\"},\"line_color\":{\"value\":\"#1f77b4\"},\"right\":{\"field\":\"right\"},\"top\":{\"field\":\"top\"}},\"id\":\"55925f2e-797b-46d7-828a-ba3b578c8033\",\"type\":\"Quad\"},{\"attributes\":{},\"id\":\"c542924e-ffa9-43fa-9fea-52c46703bc95\",\"type\":\"BasicTicker\"},{\"attributes\":{},\"id\":\"3e6866eb-8d0c-4cab-9bcd-29b7b5cc8742\",\"type\":\"PanTool\"},{\"attributes\":{},\"id\":\"85a42ee0-1672-412a-a2e2-a47fcce7b755\",\"type\":\"BasicTickFormatter\"},{\"attributes\":{\"callback\":null},\"id\":\"c668e897-eb5c-47c0-a8b0-e22ac5feed58\",\"type\":\"DataRange1d\"},{\"attributes\":{},\"id\":\"b8b4c59d-39a3-4fc7-b3d2-ce12f97f2dc0\",\"type\":\"SaveTool\"}],\"root_ids\":[\"5cbe84c2-e474-477a-89af-6aa4f7dab369\"]},\"title\":\"Bokeh Application\",\"version\":\"0.12.13\"}};\n", " var render_items = [{\"docid\":\"74b8b447-e2a6-4b27-9505-ee870fd69d64\",\"elementid\":\"d6fb485a-a1e9-4b8c-867c-62f83082294b\",\"modelid\":\"5cbe84c2-e474-477a-89af-6aa4f7dab369\"}];\n", " root.Bokeh.embed.embed_items_notebook(docs_json, render_items);\n", "\n", " }\n", " if (root.Bokeh !== undefined) {\n", " embed_document(root);\n", " } else {\n", " var attempts = 0;\n", " var timer = setInterval(function(root) {\n", " if (root.Bokeh !== undefined) {\n", " embed_document(root);\n", " clearInterval(timer);\n", " }\n", " attempts++;\n", " if (attempts > 100) {\n", " console.log(\"Bokeh: ERROR: Unable to run BokehJS code because BokehJS library is missing\")\n", " clearInterval(timer);\n", " }\n", " }, 10, root)\n", " }\n", "})(window);" ], "application/vnd.bokehjs_exec.v0+json": "" }, "metadata": { "application/vnd.bokehjs_exec.v0+json": { "id": "5cbe84c2-e474-477a-89af-6aa4f7dab369" } }, "output_type": "display_data" } ], "source": [ "fig = bk.figure(plot_width=600, plot_height=350, title=None)\n", "fig.quad(top=hist, bottom=0, left=edges[:-1], right=edges[1:])\n", "bk.show(fig)" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "TensorFlow offers a series of **operations** similar to numpy, for example: `tf.abs, tf.negative, tf.sqrt, tf.exp` and so on. It also has operations for arrays, `tf.concat, tf.split, tf.shape` and for control flow, checkpointing and Neural Networks building blocks. We will see them in following notebooks and we will describe them when needed (most of them are self-explanatory)." ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "TensorFlow supports native python data types, if you pass a list or numpy array to a constant function like one of the above, the resulting tensor will have corresponding data type. Single values will be converted to 0 dimensional tensors (scalars), lists will be converted to one dimensional tensors (vectors), nested lists will be converted to two dimensional tensors (matrices) and so forth." ] }, { "cell_type": "code", "execution_count": 32, "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "42\n", "[42 42]\n", "[[42]\n", " [42]]\n" ] } ], "source": [ "tensor_d0 = tf.constant(42, name='scalar')\n", "tensor_d1 = tf.constant([42, 42], name='vector')\n", "tensor_d2 = tf.constant([[42], [42]], name='matrix')\n", "with tf.Session() as sess:\n", " print(tensor_d0.eval())\n", " print(tensor_d1.eval())\n", " print(tensor_d2.eval())" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "But tensorflow has its own data types, for example `tf.int8` or `tf.float16`. TensorFlow integrates seamlessly with NumPy types, and as we have seen it is possible to pass numpy types to TensorFlow." ] }, { "cell_type": "code", "execution_count": 33, "metadata": {}, "outputs": [ { "data": { "text/plain": [ "True" ] }, "execution_count": 33, "metadata": {}, "output_type": "execute_result" } ], "source": [ "tf.int32 == np.int32" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "As we have seen using python types to specify TensorFlow type is easy, and it is useful for quick prototyping. However TensorFlow types are more specific, for examples pyhton has one integer type only whereas TensorFlow has more int-8 to int-64. For bigger projects it is preferable to use native TensorFlow types.\n", "\n", "Aside from the fact that costants are, well, constant, there is one more caveats to their use: constants are stored in the **graph definition**. The graph definition is stored in a protobuf, protocol buffers, Google's language-neutral, platform-neutral, extensible mechanism for serializing structured data." ] }, { "cell_type": "code", "execution_count": 34, "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "node {\n", " name: \"const_value\"\n", " op: \"Const\"\n", " attr {\n", " key: \"dtype\"\n", " value {\n", " type: DT_FLOAT\n", " }\n", " }\n", " attr {\n", " key: \"value\"\n", " value {\n", " tensor {\n", " dtype: DT_FLOAT\n", " tensor_shape {\n", " dim {\n", " size: 2\n", " }\n", " }\n", " tensor_content: \"\\000\\000(B\\000\\000(B\"\n", " }\n", " }\n", " }\n", "}\n", "versions {\n", " producer: 24\n", "}\n", "\n" ] } ], "source": [ "with tf.Graph().as_default():\n", " const_value = tf.constant([42.0, 42.0], name=\"const_value\")\n", " with tf.Session() as sess:\n", " print(sess.graph.as_graph_def())" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "This makes loading graphs expensive when constants are big, and could reach the limit of 2GB for the size of the graph. It is better to use constants only for primitive types. \n", "\n", "For these reasons we introduce variables next." ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "#### 2.3.2 Variables and Placeholders" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "**Variables** are used to maintain state accross executions of the graph. In this example `state` is initialized to zero and updated each time `update` is run. When using variables, they *must be initialized* after launching the graph, that is after creating a session. In order to initialize the variables it is necessary to add an init operation that must be run *before* all other operations." ] }, { "cell_type": "code", "execution_count": 35, "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "state: 0\n", "state: 1\n", "state: 2\n", "state: 3\n", "state: 4\n", "state: 5\n" ] } ], "source": [ "state = tf.Variable(0, name='counter')\n", "\n", "one = tf.constant(1, name='one')\n", "new_value = tf.add(state, one, name='new_value')\n", "update = tf.assign(state, new_value, name='update')\n", "\n", "init_op = tf.global_variables_initializer()\n", "\n", "with tf.Session() as sess:\n", " sess.run(init_op)\n", " print('state: ' + str(sess.run(state)))\n", " for _ in range(5):\n", " sess.run(update)\n", " print('state: ' + str(sess.run(state)))" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "Here `assign` is part of the computational graph as `add` and other operators. Variables are typically used to represent parameters of a model, for example in neural network they are used to store the weights matrix, that it is updated at every execution of the graph. Parameters are updated by the operations in the graph as the result of the optimization process. \n", "\n", "A Variable exists outside the context of a `session.run`. Variables can be initialized to random values, using a distribution (in the previous example it was initialized to zero). For example we can use a random normal distribution with given mean and standard deviation, as well as specify resulting shape." ] }, { "cell_type": "code", "execution_count": 36, "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "tensor object: \n", "tensor values after init: [[-0.19331591 -2.5231225 -1.8801746 0.44765785 0.21164836]]\n" ] } ], "source": [ "init_val = tf.random_normal((1, 5), 0, 1, name='init_val')\n", "var = tf.Variable(init_val, name='var')\n", "print('tensor object: {}'.format(var))\n", "\n", "init = tf.global_variables_initializer()\n", "with tf.Session() as sess:\n", " sess.run(init)\n", " initialized_values = sess.run(var)\n", " \n", "print('tensor values after init: {}'.format(initialized_values))" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "Note that if we run the code several times the output changes, but also the object changes (look at the name of the object!). A new variable is created every time `tf.Variable` method is called. \n", "\n", "A better approach would be to reuse the variable already created. There are two main approach: pass the variable oject around, or encapsulate variable object inside a **variable scope**. As stated in the documentation of TensorFlow, passing variables around is more explicit, but it is sometimes convenient to write TensorFlow functions that implicitly use variables in their implementation. In general using `tf.Variable` directly is discouraged. The preferred method is to use `tf.get_variable`, that allows also to reuse a variable multiple times." ] }, { "cell_type": "code", "execution_count": 37, "metadata": { "collapsed": true }, "outputs": [], "source": [ "def var_reuse():\n", " with tf.variable_scope(\"reuse_example\", reuse=tf.AUTO_REUSE):\n", " new_var = tf.get_variable(\"new_var\", initializer=init_val)\n", " return new_var" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "Previous function defines the scope in which variable is declared and it allows you to create a hierachy of definitions (suitable for example for Neural Networks). The parameter `reuse=tf.AUTO_REUSE` tells TensorFlow to create a variable the first time it is called or to reuse an existing one ater." ] }, { "cell_type": "code", "execution_count": 38, "metadata": { "collapsed": true }, "outputs": [], "source": [ "new_var1 = var_reuse() # Creates new_var.\n", "new_var2 = var_reuse() # Gets the same, existing new_var.\n", "assert new_var1 == new_var2" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "Both `variable_scope` and `name_scope` have the same effect on all operations as well as variables created using tf.Variable (the scope will be added as a prefix to the operation or variable name). However, name scope is **ignored** by tf.get_variable.\n", "\n", "Each *session* maintains its **own copy** of the variables." ] }, { "cell_type": "code", "execution_count": 39, "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "tensor object: \n", "tensor values after init: [[ 0.03998312 -1.1565742 0.15244694 0.41520226 0.38697594]]\n" ] } ], "source": [ "new_var = var_reuse()\n", "print('tensor object: {}'.format(new_var))\n", "with tf.Session() as sess:\n", " sess.run(tf.variables_initializer([new_var], name='new_var_init'))\n", " initialized_values = sess.run(new_var)\n", " \n", "print('tensor values after init: {}'.format(initialized_values))" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "So far we have covered how to store values in constant and use variable to update their values (these operations are called *source* operations). Remember that a the worklfow of a TensorFlow program has two phases: assemble the graph and use a session to execute operations in the graph. We need a method to assemble the graph without knowing the values for some tensors needed for computation (simply the inputs of our model).\n", "\n", "TensorFlow provides a method to pass values to the variables with a **feed** mechanism. A feed replace the value of a tensor with the value that you provide. Tensorflow provides a special structure called **placeholder** for feeding input values. Placeholders have an optional shape parameter, that can be set to `None` meaning that the shape can be of any size (we will see that it is common for mini-batches later on). To feed the values we construct a dictionary whose keys correspond to placeholder variables and values can be list or numpy arrays to feed to each variable." ] }, { "cell_type": "code", "execution_count": 40, "metadata": { "collapsed": true }, "outputs": [], "source": [ "input1 = tf.placeholder(tf.float32, name='input1')\n", "input2 = tf.placeholder(tf.float32, name='input2')\n", "output = input1 * input2" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "One caveat of using operator overloading, is that you cannot assign a name to the operation, because a tf.Operation (or tf.Tensor) is immutable once it has been created. " ] }, { "cell_type": "code", "execution_count": 41, "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "mul_1:0\n" ] } ], "source": [ "print(output.name)" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "The typical way to rename an op is therefore to use tf.identity(), which has almost no runtime cost:" ] }, { "cell_type": "code", "execution_count": 42, "metadata": { "collapsed": true }, "outputs": [], "source": [ "output = tf.identity(output, name='placeholder_multiplication')" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "**NOTE:** the recommended way to structure a name scope is to assign the name of the scope itself to the output from the scope (if there is a single output op):" ] }, { "cell_type": "code", "execution_count": 43, "metadata": { "collapsed": true }, "outputs": [], "source": [ "with tf.name_scope('name_scope_multiplication') as scope:\n", " input1 = tf.placeholder(tf.float32, name='input1')\n", " input2 = tf.placeholder(tf.float32, name='input2')\n", " output = tf.multiply(input1, input2, name=scope)" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "A variable declaered as placeholder expects a feed and generate an error if it is not supplied" ] }, { "cell_type": "code", "execution_count": 44, "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "[array([14.], dtype=float32)]\n" ] } ], "source": [ "with tf.Session() as sess:\n", " print(sess.run([output], feed_dict={input1:[7.], input2:[2.]}))" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "You can feed_dict any feedable tensor. Placeholder is just a way to indicate that something must be fed. A convenient function called `tf.Graph.is_feedable(tensor)` returns true if tensor is feedable. " ] }, { "cell_type": "code", "execution_count": 45, "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "[12.0]\n" ] } ], "source": [ "init_placeholder = tf.global_variables_initializer()\n", "with tf.Session() as sess:\n", " sess.run(init_placeholder)\n", " print(sess.run([addition], feed_dict={product: 4}))" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "The practice of feeding values for some operations is extremely helpful for testing. Feed in dummy values to test parts of a large graph avoids computing those portions of the graph. \n", "\n", "Remember, never declare an operation at execution time! It will add a new operation to the graph each time. We will illustrate this concept as well as the concept introduced so far with the aid of a tool called TensorBoard." ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "#### 2.3.3 TensorBoard" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ ">The computations you'll use TensorFlow for - like training a massive deep neural network - can be complex and confusing. To make it easier to understand, debug, and optimize TensorFlow programs, we've included a suite of visualization tools called TensorBoard. You can use TensorBoard to visualize your TensorFlow graph, plot quantitative metrics about the execution of your graph, and show additional data like images that pass through it. \n", "\n", "TensorBoard is a useful tool for visualizing and debugging training of a Nerual Network. It is used to *babysitting* the learning process, display useful information like the graph of the model and to manage experiments. \n", "\n", "We need to tell TensorFlow to save the graph of the model in order to visualize it. " ] }, { "cell_type": "code", "execution_count": 46, "metadata": { "collapsed": true }, "outputs": [], "source": [ "writer = tf.summary.FileWriter('temp/graphs', tf.get_default_graph())\n", "writer.close()" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "Then in the current directory (remember to activate your environment) type:\n", "\n", "`tensorboard --logdir=temp/graphs`\n", "\n", "A common source of error is to add operations at run time, to the graph. While it is perfectly normal in any computer language, it is a problem in TensorFlow. Let's look at it with TensorBoard" ] }, { "cell_type": "code", "execution_count": 47, "metadata": { "collapsed": true }, "outputs": [], "source": [ "x = tf.Variable(10, name='x')\n", "y = tf.Variable(20, name='y')\n", "\n", "with tf.Session() as sess:\n", " sess.run(tf.global_variables_initializer())\n", " for _ in range(10):\n", " sess.run(tf.add(x, y)) \n", " writer = tf.summary.FileWriter('temp/graphs', sess.graph)\n", "writer.close()" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "Sometimes this is called *lazy loading*. As can be seen we need to defer the creation and initialization of an object until it is needed." ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "## 3 Building Models" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "Now tensorflow is especially useful to train Machine Learning models. We will briefly sketch the workflow of building simple models. We will switch to more complex models (Neural Networks) in following notebooks." ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "### 3.1 Linear Regression example" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "The most simple model to build is based on a target variable $y$ that we are trying to predict given input data $x$. We have a set of $x$ and $y$ values that we use to train the model. Suppose that $y$ is a vector with continuous values, we have a regression model that can be described with a simple equation: $f(x) = w^Tx + b$ and $y = f(x) + \\epsilon$, where \n", "$f(x)$ is a linear combination of our training data $x$, with a set of weights $w$ and an intercept $b$. The target $y$ is equal to $f(x)$ with added gaussian noise. \n", "\n", "We implement this simple regression with tensorflow. First we need to create some input data, then we will create the computational graph, we'll train the model and look at the results.\n", "\n", "Now we define a set of points to used for training and as target." ] }, { "cell_type": "code", "execution_count": 48, "metadata": { "collapsed": true }, "outputs": [], "source": [ "data = np.random.randn(2000, 3)\n", "w_real = [0.3, 0.5, 0.1]\n", "b_real = -0.2\n", "\n", "noise = np.random.randn(2000) * 0.1\n", "y_real = np.matmul(w_real, data.T) + b_real\n", "target = y_real + noise" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "**Phase 1:** assemble the graph.\n", "\n", "*Step 1:* create placeholders for input and target" ] }, { "cell_type": "code", "execution_count": 49, "metadata": { "collapsed": true }, "outputs": [], "source": [ "X = tf.placeholder(tf.float32, shape=[None, 3])\n", "y = tf.placeholder(tf.float32, shape=None)" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "*Step 2:* create variables for weights and bias" ] }, { "cell_type": "code", "execution_count": 50, "metadata": { "collapsed": true }, "outputs": [], "source": [ "init_val = tf.random_normal((1, 3), 0, 1)\n", "W = tf.get_variable(name='weights', dtype=tf.float32, initializer=init_val)\n", "b = tf.get_variable(name='bias', dtype=tf.float32, initializer=0.0)" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "*Step 3:* define the model" ] }, { "cell_type": "code", "execution_count": 51, "metadata": { "collapsed": true }, "outputs": [], "source": [ "linear_model = tf.matmul(W, X, transpose_b=True) + b" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "**NOTE:** How to train such a linear model? There is however a simple closed formula to solve a linear regression model, but we will define a loss function and an optimization operation, that will be inserted into the graph (it will be helpful later on when we will see Neural Networks). The loss function defines a method to evaluate model performance. The loss function defines a distance between the predicted values and the target values, and it is used as input to the optimization procedure that finds a set of parameters that minimize the loss function. The most common loss function is the Mean Squared Error (MSE), that finds the squared distance between a set of points and then averages the results over all points. The optimization operation is gradient descent that iteratively updates the weights in a way that decreases loss over time. The update rule is based on the gradient of the loss function. If the loss is a multivariate function of the weights $F(\\hat w)$, then in the neighborhood of some point $\\hat w_0$, the *steepest* direction of decrease of $F(\\hat w)$ is obtained by moving from $\\hat w_0$ in the direction of the negative gradient of $F$ at $\\hat w_0$. We will give more hints on the gradient descent algorithm and its variants when we will use it with Neural Networks.\n", "\n", "*Step 4:* define the loss function" ] }, { "cell_type": "code", "execution_count": 52, "metadata": { "collapsed": true }, "outputs": [], "source": [ "loss = tf.reduce_mean(tf.square(linear_model - y)) " ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "*Step 5:* define the optimizer and the training operation" ] }, { "cell_type": "code", "execution_count": 53, "metadata": { "collapsed": true }, "outputs": [], "source": [ "optimizer = tf.train.GradientDescentOptimizer(0.5)\n", "train = optimizer.minimize(loss)" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "**Phase 2:** train the model\n", "\n", "*Step 1: define an init operation" ] }, { "cell_type": "code", "execution_count": 54, "metadata": { "collapsed": true }, "outputs": [], "source": [ "init = tf.global_variables_initializer()" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "*Step 2*: run the optimizer with training data" ] }, { "cell_type": "code", "execution_count": 55, "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "0 [[-0.24243562 -0.5938537 1.0206949 ]] 0.0\n", "5 [[0.29791233 0.5040582 0.09849343]] -0.1979366\n", "10 [[0.2979123 0.5040581 0.09849343]] -0.19793658\n" ] } ], "source": [ "NUM_STEPS = 10\n", "with tf.Session() as sess:\n", " sess.run(init)\n", " tmp_ = sess.run([W, b])\n", " print(0, tmp_[0], tmp_[1])\n", " for step in range(NUM_STEPS):\n", " sess.run(train, {X: data, y: target})\n", " if ((step+1) % 5 == 0):\n", " tmp_ = sess.run([W, b])\n", " print(step+1, tmp_[0], tmp_[1])" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "### 3.2 Classify Digits with Logistic Regression" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "#### 3.2.1 The Dataset" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "We will learn to classify MNIST handwritten digit images into their correct label (0-9). MNIST is a standard dataset hosted on [Yann LeCun's website](http://yann.lecun.com/exdb/mnist/). The digits have been size-normalized and centered in a fixed-size image.\n", "\n", "The importance of classical datasets is twofold. First they are good for people who want to try machine learning techniques while spending minimal efforts on preprocessing and formatting data. Second they are useful for comparing machine learning algorithms, since we know well how they work on these datasets.\n", "\n", "Each image is 28 pixels by 28 pixels, representing an handwritten number between 0 to 9." ] }, { "cell_type": "code", "execution_count": 56, "metadata": {}, "outputs": [ { "data": { "image/png": "iVBORw0KGgoAAAANSUhEUgAAAdkAAACGCAIAAAAeimfDAAAAA3NCSVQICAjb4U/gAAAAGXRFWHRTb2Z0d2FyZQBnbm9tZS1zY3JlZW5zaG907wO/PgAAIABJREFUeJztnVdz2+r19cHewd4pkZRoueSUmWRyk3z/mUwyiTP/42NbFklR7BVgBcEGvBfr5R7GOjlxkQQS2r+LzIlNyQAILDzPLmtbdF0XGIZhGEOxGn0ADMMwDGsxwzDMEcBazDAMYzysxQzDMMbDWswwDGM8rMUMwzDGw1rMMAxjPKzFDMMwxsNazDAMYzysxQzDMMbDWswwDGM8rMUMwzDGw1rMMAxjPKzFDMMwxsNazDAMYzysxQzDMMbDWswwDGM8rMUMwzDGw1rMMAxjPKzFDMMwxsNazDAMYzysxQzDMMbDWswwDGM89sf+BywWy2P/EyeErutf9Xm+egRfuu+Br94387WX7pvhdTHDMIzxPPq6GDzZu+Vo+Z6FxjO/enzpvge+et/ME28OeF3MMAxjPKzFDMMwxsNazDAMYzysxQzDMMbDWswwDGM8rMUMwzDGw1rMMAxjPKzFDMMwxsNazDAMYzysxQzDMMbzRD3QzKmz2+222+1ms1mtVoqiKIqy3KNpmsfjcbvdHo/H6/X6fD632+1wOOx2u93ONxjDfBH8qDBfhKZpqqouFovJZNLv9/v9/mAwGI1Go9Fos9lEo9FoNBqLxRKJRCKRiEajXq/X6/WyFjPMF8KPCvNF7HY7VVVns9lgMLi9va1Wq7VardFoNBoNVVXPz8/Pzs7Oz88vLi40TXM4HBaLxel0Gn3UDHMymE2LD52lNE3Tdf03vaawuVYUhT6z2+2wDXe73YFAIBAIaJo2n89ns5miKKqqqqq63W4tFovFYrHb7b7/xOPxPOFZPh24JtvtdjweQ3mbzWaz2Wy1Wp1Op9/vS5K0Xq9dLpeu6+v1WlXVyWQyHA6LxWKxWPT7/Uafwemx3W7X6/V6vVYUZTKZTCYTQRDC4XA4HPb7/Xa73Waz2Ww2ow/TeLbb7Xw+XywWi8UC/6GqKp53h8OBKxYMBh0Oh9PpdDgcRh/v/8BsWkzouq5pGhT2/t+Ox+PhcNjv9zebjaZpmqZBR1RVDYfDWOVtNhuIDhRHluXlcmm1Wq1Wq9frTSaTyWQylUql0+l0Om1WLUZoYrlc9nq9T58+vXv3rlarSZI0Ho8nk8lsNluv19vtdjqdapqmKMp4PG61Wq1WS9O0SCSSTqeNPoPTY7vdLhaL2WzW6/Vub29rtZogCFdXVy9evMhkMm632+VysRYLgrDZbCRJ6na7nU6n0+m0221JkqDFXq8XVwwLgkAgwFpsDFjqapqGNd39D0wmk1ardXt7u1wuodeKouDtmslkdF0XRXG1WtVqtffv31erVejLdDq12WxWqzUUCpVKpVKp9OLFC6vVKopiNBp9+tN8Ana73Wq1ms1m/X7/5ubmH//4R7lcXq1Wq9WKXmO6rk+n08ViMRwOW62W2+3OZDLRaPTly5dGH/5JguWeJEl3d3dv3759+/atruuqqvr9/lAohG2Z0cd4FECLa7Xap0+frq+vP3782Gw2ocXBYPAvf/mLxWIRRVEQBJfL5fV6jT7e/8EpfanaAbjimqZt/pPtdosPkLwqinL/V/V6vXa73W63V6vVbrfDuhgSo2laIBBwuVyqqpbL5Wq1end3NxgMJElSVRX7nd1uZ7VaHQ6Hy+Wy2+1Wq9lKA6lqAi+tdrt9c3NTrVZ7vd54PMbf6rqOiI3VarVYLIjzqKq6Xq89Hs94PB6Px9PpFAUVNpsNztzHNrwHp7ler+m+stlsLpfL6XQaJXnb7Xa5XOICKoqy2WysVutuzzP3dxcEYbvdYgs7HA5rtVq5XL65uanX6/1+X5ZlUob5fI64IsnFkXNKWqzrOgkuVr6bzeYwWjSfzxVFoRDnaDSSZVmSpPu/ajabTafT6XSKX6VpGv4c4uJ0OjebzXK5rFart7e3g8FAURRd151OJ8oDIpEIygai0ajf7z/+7c/XstvtlsvlYrHodDpYcVQqlbu7u8lkQjc3VBgbBVw3YR8aQt3bZDKRZRlXzO12C8cnxIIgbDab+Xw+n88RbEHCIBQKBYNBA7UYV28+nwuC4PP5HA4HF6UQiImNRqNms1mpVK6vr8vlsiRJy+WSbkLLHqMP9is4pW9X13W8EmkVs16vEcmVJAn/MR6Poder1aqz5/6vwlr4cJWBRa7b7YaUT6fT5XKJePFsNsP3ip1OMBiMRCKRSCQWi0UiEb/fb76CAWjxdDrtdrufPn365z//WalUkMk8jPlAiyHHVqsVb0d8L6h+k2VZ0zS73e50OrF7OLbHA1osSRJWoOv12u/322w2n89n1CHRxV8sFoIgIDPs8Xg4Rgy22+1kMul0OrVarVKpYMeG7w4fOFThY7vffocT0GKqcEDUst/vj8djKOl6vcbyFunm6XQKscDGczAYDIfD31wX38fpdAYCgVgsFgqFRFFEmGK73TocjtVqhY22x+MJBAJ+vz+RSBSLRURFTbAupmoT2lLIstxut1utVqVSub297XQ6siyv1+vNZkNvL4rSeDweFJNYrdbRaITKim63++HDB4vFgiRnPB7HZ44tbIetlSRJ0+l0tVoheev3+38zzfCo0IXFjY0Nn6qqm83G6XSeykb7CcDGaz6f45HHbpiuD8JlwkmpMDgBLT5M5X/8+PH6+rrdbm+3291ut9ls1D3L5VJVVQR8kbXDrfwl/4TFYvH5fOl0+uLiIhwOe71ej8ez2+1SqdRisdA0zel0Op1Ot9uNHXcwGMxkMqlUKhqNosfssS/Co0KpTtRRLZfLdruNfMjt7W29XpdlebVaIZ5DPwUtxtVIpVLJZNJut1er1dVqJctyq9XSdX04HObz+UKhkM/n8Zkj1OL5fI5wFs59s9nEYrGn12ICXwRiREh42Gw27AWFZz8PVNjHKvHII7L0O9WrJ8QJaDF1GWCp9be//e3m5gYrX4r2Hub08JUglUSB4N8B70+fz5dKpa6urqLRqMPhQLcCfoPNZnO73aglQlYHAhQMBrEYNEHujvYZiC1Ai9++fVur1bDzWK1Wn93uVqvV6XR6PJ5wOHx2dnZ5eelyuVarVa/X6/V6zWaz3+9Xq9XXr1/P5/PNZmOxWAKBgIHn+JtQjGIwGCDroOs6KhqNOiTSYmWP3W5H8OfU5eZBwEoLWowlwpc85sfPCWixIAhIEwmCQGKxWq3wSvySH7fvoSwTpBzPG/7E6/XG4/F8Ph+PxykfRT/udDpdLhc0Gv8Xa2dzRIpVVUWcR5bl4XA4HA7v7u7K5XKr1RoOh7jU9293hO/RlSAIgsvl8vl8gUBAFEW/349varvdDofDwWAQjUZTqZSBAvffWK1W4/G40+kMh0PcErivnl71aFWhKIosy91udzAYrFYrq9Xq2YNNmAne/d8A9sGbzUaW5X6/j4Li8Xi8Wq0sFgvyFog0iqKYTCYvLy+TyaTf73e73SeR9jyBQ8ReGL4zHo8Hmogv5gt/A9U/UM8Sii7m8znVA6A6IpfLJZPJw8IAYZ+hws9StsrpdJoml6IoSrPZvLu7Qzddr9frdru9Xm80Gt0PTRDYr+i67nK5FovFarXyer0ulysYDEajUST60Iy3WCyQCzVw4//fUFVVkqRGozGZTPCuNepIqEBzOp0OBoN6vd7pdCDBwWAQOQwkJ0xz430Vu90Oj22n02k0GujCHw6HqqpaLBaHw+F2u4PB4MXFBXo+ERkLhUI+n4+1+GGAFuu6jtIoRAlQdPmFv8HpdKJO3rEHdYiqqqJSGFocjUaz2Ww6nf4s6k+6/Nl/mGZ5oihKu93+9ddfb25u0OKMcpT1eo1Sk9/UYqRQkFlSFAVBDNJiXF6smqHFKPZ8+rP7fVarlSRJqJZBKZtRR4KtN3XWNBqNwWCQTqdDoRAOTBRFn89nphvvq0DHgCRJiIDd3t7e3d0hS4Q6VDzCL1++/POf//zmzRu8vSDErMUPAzUa+Xy+WCyWy+UohaqqKlasgiAgy7FcLukH8Vd2uz2ZTOZyubOzs0MtHo1Gw+EQ4f/VahWJRERRxPLZsFN9WlA1sdvtxuNxr9er1+v1er3b7fb7fVRTCYJgs9nooqGpX9d1XGral1itVrvd7na7/X5/NBqFKGuahsUydT9+Yfj+iUGUHJHHwyqRpwfNC+hf6PV6k8lkvV4jQOH3+z0eD+JjRh2eURw6ojSbzUajUS6X6/X6cDhE0dRut3M4HKIoxuPx8/NzGBjkcjncrgjpnMTb6wS+WgSDhIP0ms1mkyRpNBopioJNnCAI6KM71GJkljweT6FQePPmzR/+8Ac0U9nt9ul0KssyFBnE4/FAIPCs7nXUsS6XS2Suer3ecDicz+eHq1ebzeb1ev1+v9/vF0VRFMXdbtdutzudzna7hVJDLFB2res6VFtRlMFgMJlMTiLddAxZ+MVi0Ww2y+Xyhw8fOp2OqqrIjmJ9YKaY2FdBbUfdbrdSqXz8+LFcLiOmRPlMu90eCoXOz89LpVImkwmHwyjHpobPk6hvOwHpgRZbrVa/359OpzebjcfjQbJ+sVhAIARBsFqt2NzRT1HBQz6f//HHH//617/ihrbb7ei7m0wmiJPWarVEIoEif0PP9UlBr+10OpUkaTgcdrtdRN/ua3E4HCZvYjwA4/F4Pp8jjYn6YlEUI5EIwkEul6vf79/e3lJNyzGI3ZEDLX737t3Hjx+73a6qqm63+1CLT2Jx9+AgNIH8aqVS+b//+79yuTybzbAoxk1lt9upmIe0+ORa705Di4W9tobD4d1u53Q6RVEMh8OKoiBtiuzzZDJBXhWpcJRGpFKpbDabzWZTqRTyHpB1/AaUqXk8HjigP4d1MTkboAq43W5fX18jRqwoCiqEsCJzOByhUOjs7CybzSYSCUQt4Yo5m83Q+SKKYjqdLhaLiUQCz0AgEIB9Eow0qSx/NpuhLJ8WLMdwEVCTflimatQhfVbpvNvtbDYbEnehUOjZ9kBTkxc2vp1OZzAYUPUOohDBYDCRSJydnaEO6lSSdZ9xSkdst9v9fr/FYvF4PKFQKJlMInfv8XjwhcFrGM3Qs9lMFMVMJlMqlbLZLOwFqFINTXQ2m03XdbfbHY1GA4FAIpEwR43a74NNn6qq7Xb7w4cP79+/r1Qq9Xp9NptRSNfhcMBpMJvNvnr16uXLl/F4HD8+m81yuZyu68vlEivldDqdz+ez2Ww4HEbRtyAI0GK8IyVJcjqdsB4dj8fH0NFL/jJoNUaa0ZBSNgIV8RRbx9YbIfhYLGaC9s5vAwZV7Xa70WggRoyUO1YMiI8lEolMJoMwsSiKLpfL6KP+Fk5MiwOBgNfrDYVC5KmG6KSqqnDUnU6nVqtVVdX5fA4tvrq6ymQypMX0q6xWK1bE0WgUoc+TMJz+fiBD0+m00+mgd6bZbKLCj8TI4XAEAgEkQ16/fv2nP/0pGo2i0dzhcGia5na7LRYLUiXJZBI5a7fbTeGIYDCIdTGWeIIgYCYTIsjI9Rl7EZbLJQrvFosF6u2MdUFDwpO8BgVBwMgCzK965lrc6XRQ7T6bzajtCPuGUCiUSCRSqVQul8vlcqdSNXGfUzpo5EMhvm63G8KB3a7T6QzuGY1GuGtxZ1Nh1mHkiFKrSD0ZdkpPCDrr4CWI/gsM6Wi324PBAF0G5PKD5Vg6nUZ4J51ORyIRVBMighmJRGw2WzabzWQyCBMfZvmR9Me2A18Tdt/dbrfVam23W9TkG3g1DgMCi8Xi0DPziY+EmkVRz4OYCUWK0EGDhoVnFS+mvnwUgLdarWazCd9aTdNQ2OdyucLhcCaTOT8/T6VSKCU2+sC/nVPSYgLfhN1u13Wdwg7owoAiwB9kPp8jg4Q+nM/k+LkBV3g427VarUajcXd3h6o+smOmCxiJRJLJJCLF4XAYZd2iKGLXDMlAUDgQCCAj+t+uLbbe6/ValuVGo4FEK/7XQFar1WQyQT/LbDZDi7YhJXd4XVFTL3li4N5GCwM6x56bFmOnoijKaDTCMAeq88PDjoRQsVgslUpHaHXytZykFguCcNijTJ0X2J5gcUdabLfb0+k0fAaMPWZjgRbTnV2pVNC2BI8rqDBavbHcQFF2NpuNRCJod8SWGYtrCitDwX8nYY1VDFpXm80mxglmMpmnPfvPgcNfv98fjUYwL8ar/emPBPXXGPEFLYY3G9bFdrsd38hza33GK3yz2SDfgHUx4uloOLDb7V6vN5FIFAqFUqmUSCROfYN7klp8f0IENc5lMhnMshsOh5qmTadTQRAwlWM2m0FQnlXcjarKsNeD10StVoMT5mQy2e12aJbDIhe50GQyWSgUstksZY2+Z94lnqvlcinL8mAwwJS8Bz3LrwYP+Xg8plwQOY3gJnmyLRQMMSaTSbfbpRcDgm84GMSFfmfnYSYQsTksi6pWqxhkh/4jtDuj5j2VSiFlhzq2E03ZESepxfex2WyRSCSfz9tsNnSRUpfzarWC63Gv14NtzVM+accAQm+z2QxzEG5vb9E/OhwOcX+LopjP54vFIhpeAoEA+eWjmupBah5g8I9Gx98cCPuUICaAGjtoH9z40DX7lMM95/N5q9Wq1+vX19edTgdls0iBIEwMIT7cCJoYxM03m81oNMLc1evr61qthhEnADOeU6lUsVg8Pz9Pp9OxWMzr9Z76Gss8WowXYyAQgLVKt9sld/nBYAA5xnLjpAP8XwvlQPDMf/jwgUapYnCUsNfin3766ezsDGPMPR4Poj1ut/tB6s8Q/jseLUZlMbQYwVmMuUOHocvlerKAwGKxQHHhzc0NtBgV9OiggRfg4cBAc4N8+3K5HI1GlUrl7du3nz596nQ61JQv7LU4l8tdXFycnZ3BRvx79m1Hgkm02Gq1YgXhcDiy2WyxWJxMJvV6fbVaYbRHq9X69OkTpFlRFKw1ECTFNtCswThy/4LpRK1Wq9frqC3DBOtgMHh5eXl5eVkqlc7OztDHiMI1avp4kLucklTHMECTwpGbzYaKeT0eD3ZOD67FVOq33YMKn81mc3t7W61W0dcrSRJMMjFiBrUBeC8+k3UxzZfCTHGMMkAdG6Ln6OxIJpPYydElMvrAHwCTaLEgCHh4nE5nPB4vlUoI8C+Xy263O51Oa7WaIAgY9pNIJBAVRXFiOByORCJm1WIMskQCBKYTCL3tdjtRFAuFwsXFRalUevHiRS6Xi8VitBCmAaOw1Tf6PB4daDHs0B68FYVmHdCc3MkejJS/u7vr9/sIUKDtqFAowIEXXWTPZF2MWsPhcIgSl8OAPoYQwpQGi+JCoRCLxU49TEyYR4tRSuF0OhOJhM1mE0URfiJ2ux13vCRJ8XgcWoxHThTFbDZrtVqDweCpB5v+G2hqmEwm0GIkiLA49fv9hULhj3/844sXLzKZTCaTQfsyPfb6fp6uWV9UhyAvjxvD7XY/rBYjTIS5X/Ci6na7GIyLCoFWqzWdTtH753a74/E4NitwQ0eG4zlo8Xa7hRajxEWWZbQgIcOMwvZUKnV2dgaHYp/PZ2zT0ANiEi0+rKyA3Zrb7a7X641GI5fLIQKFDqvJZDIajUiL4esGF4VDhz3T3Poon8Ag0cFggGlJiI2iTr5UKhWLRYSJH/a2xq6cZJ0qOh7wn3hAYEF1WKV3/zP6fnAXVSJTGSyN+zo8zc2ew6Gu0OL+AcPhcDKZLJdL/Dgal1KpFDbgT5lINApcHxQ+drtdPLmYegWncpS3o87y8vISzR2RSOR0u+zuY5LTIBCacLvdmqbBSwEzifv9frfbRVU/fG1QQg9t2m63WCxDj/AFm0OL5/N5s9n88OFDuVzGGCEsAGEGjY0Cxq0+0gN/Ki026Es8jCD/5mfQfE81eRg2AUPnwxmMgAIRUHAEQxVFQacfDQnDzgNF8biBH2+FfpzgqZzNZvV6vVwu//rrr5VKpdfrwaAc4cRkMlkqlV69elUqlfL5PFwNzBS6MZsWC3uvCavVmslk1uu10+l8//69qqrNZnO9Xs9mM0EQbHswk00QhOVymU6nqbvJNA/AfD5vt9swACIt9vv94XA4Ho/HYrFYLBYKhR7DOI3WxUe7Fj7kfjbv/mdoGjy2U7qubzYbSZLgeYQCStikYI3c6XQQi6AGa/pfzBRH0sm6B1eMsoiPEbk+TmiyVKPRqFQqHz58uLu7g2cT+uuCwWA6nS6VSj/99FOpVIJloMlCN2bTYppCaLVaI5EIyoMwan46nSJtgu8Y2yKU06MqebVaCYKwXq9hlYDiihP9smnjPB6PselDDnO73cLailQYMyMe/ADuaxlVrRxJOy+FwvH9rtdrNFygt3Cz2dyvfUQ5CqzdhP/UYlmWD5fMeANRFIJWvmhupKFfoVDI7Xaj3QMiLggCVTqjosPEWqzvx63O5/Ner1etVj99+nR7e9tsNgeDAUxN0X8fCATQl49eUHpsjT6Dh8RsWkxgtHM4HIbOOhyOSCSCbEm324UoK4qCZSONNVQUJZvNJpNJypmcaBXBer2G9Rry0ciBwOAK4chkMomeukcKtx06eVNaFbbRx2DFi2OjHgqLxaIoSqvVstlso9EI1mj3X1HQ68lkQqXZuHNQqQLtIO9AZINRcQVBcbvdMPrBwGyo7d3dnaZpaEbHwSCajziSiasthYMwMaqJ//3vf9/c3GD2INlJo5KHRg/jSpopNEGYWYtRmImkXDgcPj8///Tp083NjSAIcCaDa+J2u5UkaTqdwpYFq2M8BoIg4HE1+GS+HnjfIB89HA4xhgNNDXDiTyQS0OLHXlyQHMNyDI0kx1C1gooRcuxbLBZQgUajAaG8b2YNLT5cFyOyAQmmhTZkFJ0a0GKY7gcCARRQhsNh/C3+ajgc3tzc2Gy23W5HWuzz+RDEP8Xb7wvRNA1GHMPhsFqt/utf/6pWq/Riw+WlMfC4YmgjMGVtj5m1GJtBtFTBjRd3tqZpWJs4HA4MMJ7P51jmkFehw+HYbDahUIhmm54WND8JkRma04ymmFAolEqlHmqsFIUj8GjBkg2BeGSiUBkKvyGYDRlehwTTzlgshrcUOoB2ux3qGSaTCV7kn/0UpZhUVcV5EdhCYUXs34NscCgUQk8NxlBBi1GxY7FYyPsc3w5sKMikyYhr83Sggg1hNEw7a7Van30GNw8M9Sm1bsjRPjbmPKvPwHvVarXmcjlUE3e73V6v1+12qcxTVdXRaEStutPpFH0QKLcw+gy+GkTiaMmGnmMEaqFBCMU81LhVLGHQ5YhJxuhbxeQFn88XiUQKhcKrV69ev36dzWb9fv/3/6PfgyiK5+fnsMyHB8V4PP5scs/9t9Rut0NiTdM0RBhgXoH4g3cP/QlWvrS5xqXAghf3GEJkkiThTUAmmc/ElU1VVeTrarXaaDSiyeKHuFyuWCxWLBYvLy/j8fgpPoxfyHPRYpQoQohzuRyEuNPpXF9f73Y7zNyESxagIdOo8TT6DL6Fz+QYm1+s2mBol0gkAoHAQ/U3kxYjA3OoxRiIlc/nr66u3rx5AxX7/n/0ewgEAmdnZ1hqQSv7/T62ESho/c2eY13XPR4PFBlVKNFoFPJKq2BRFLH9orF+tILGf0DlsfrudDrQYrhII7eJpZ/54qH3gRbDqUqSpPV6ff+socWFQqFYLKIv1JBDfQLMr8VUWSHsJ+bF43FRFEOhUCQSgYsbghWbzQaFn2jEdLlcuVxuNpuhQ/q0IndkCUS1roIgQA6Qo4cH2/ckoyk0AWfk1Wo1Go0ajUa5XL65uen3+6vVimLTuT3ZbPYYbFxou4OYFXz+xuMx6hn+54+7XC7M+kPMHaDQKhgMfskBbLdbaDHeAfhHYV6KaVWGX6LHgzrC0WLXaDTgE4+gFj5D0a1AIIARM6lUCnFFYw/+8TC/Fh9CDb5+v1/XdZfL1Wq1aG2IOiQ0qloslsEecvU99cmk5LiPIVUwAPue9RdUfr1eD4dDWNRfX19fX19Xq1VZlhGXx9C8y8vLVCqFS31UNjcYAW61WuPxOLJGqDf/fex2O+p/UXaG5L7X6/3yOwSl7sPhUJZldNxRfQv2K2aNigoHL290IWIoLTxS6DOYfguT4mg0GgwGEQ46ofXQ12La7/s3wReJmCnWIHTfU0sCrY6He8LhsCAIJhBi0mLshV0u13dWT1NoAm6zlUrl48eP19fXjUYDq3KfzwctLpVKpMVHVbINLRZFES1zNPfz90GJHoWVcVW/Ks2LNCBKXBAsdrvdaH3GPWniBSANfpVlGaU+0OLDtyAqoGBWh1CSz+fDWsrAI39UzK/FtFun4ClZCtAW/vDzMJnE0EN0UqERy5BhaA8I5IPySFgUf8PvoYuJyUAYoIfQBIwfB4PBYrFA5BR+Y/D8huH3sT1LeCd9YWDhAYGZPTxSYGb/fNbFWBH3er1Go9HtdsmOAyV9NP0Wb/F8Po8LYvqqEtN+34dgogRq18ByuYTO/vLLL+12m6Z8C4JAE8Yw2yIWi0UikWNoT/hOcEbIOPl8vm/WRLylVqsVLMckSep0OuVyuVKpwJ8e82/y+Xw+nz87O0OYGI/Tqe8tHhXE1mHram4tns1m1Wr1119/RZcdpjuToz9C+alUqlQqvX79Gm6upz5X9Esw7fd9CIabLRYLxH8pYz6bzWq1WrvdxsIEH8bdgK6EcDiMRuHDgfMnymftdt+sxQj2IetSr9fr9ToG6N3d3cmyjIqCeDz+008//fzzz6VSCX+C/g4T77u/H/qC4vH44/VDHgPT6bRarf79739HXkGWZVVVsdlC+akoiqlU6vLy8ueff768vESti9FH/eiY7fumsgFsosF4T2ePLMtoZkVE+LCwEXt56lI9hgKsB4Ec7L5tkCWuJ83lxXK4UqlUKpV6vd7r9fr9/mazgb3s5eXlmzdvfvzxx6urq0c6HfPxqGb2R8Vyuez1ejc3N7e3twjTUwAQlU6xWCyTyZyfn19cXJyfn59ov9XXYsIzhNUWmoAxYZeycNhTj0ajxWIB98LZbIYsNn6WTGwxeOlIXGw3M7ROAAAKmElEQVS+jc/8gsn7JpVKfZaz/hLW6/V4PEayBePy4OzR7XbH47GmaTDevLq6urq6KpVKhUJBFMWHPyvm9EHjOIpHyfEZuFyuSCRyfn5+fn4OS5DTqiX9HsymxbDOWi6XGOnYbrebzSY6O3q9Hjqd4MxwOHYMWkx+MVRmcOol9yTHuCyTycRisQyHQ7hwfNWvohzd3d1duVzGpHQ0BGuahlFVmUzm9evXP//884sXL2C/8DinxZw2tMfCfvRQi51OZyQSOTs7QycOtPioCm8ej9PW4kOtQYEEjFCn06ksy7VaDfPnu3vIYZZ+A+kv/OPhIhSLxRKJBFqqTl2OCTSwoMlQluXpdIoKM7rXUdVHFSZUcII/lGW5Xq8jKHFzc1Mul/v9Pj6Dwox4PF4oFEqlEtbFRp/u6UF9klThY/QRPTCkv0ie0zjww+cLo0XT6XQmk3kmM02Ik9dikuDZbDafzyeTCZnGkpM3QsPI1X5WmoYQKlVNhMPhRCKRSqWSySRezic9YvawiwmLEVVVoarv3r2bTCZoWMAdb7VaMaleVdXFYoE1L0yFEKBAqB3Gb5vNBrYJeHgKe8zdpfp44E5GdYqiKIFA4NRrKO8zn8/xYL5//77T6aB4CSsAulHhHB8Oh0OhkAmKl76Kkz9VBJ6Wy+VwOMRsTQzLajabsixTVysqYe/Pa3A4HF6vNxAIIFeQz+czmQwaLiORiAkSuLS/o7ppaHEgEJjP5+l0Op1O+/1+FIqs12vsKkajUa/X6/V6MD6ez+fT6RQXU1EULN/I7zwej+fz+YuLi3w+/5u2v8yXgDsZvX8UNzMT8/m80WhcX19//Pix1+tBi4V9vys+g4lKMLczdwLzPqekxYcRCYqBYh0xmUzQ1Q7Pp7u7u3q9Dj9ijMMBEKZDrxYUrsFFjEbTo97eBCp8+H/1/dzM6XTabDZhT4OW32AwCJ9GDCqVJKnX6zWbTYxXgDovFguEdywWC9xwgsEgJual0+lisZjP57PZLPaVRp3ySUNmvhjAaJoYBfmizGazdrv98ePHSqUyGAxoZqCwNyeAPWYgEHgOphz3OTEtRkQCHV8IPFEsGGVV/X5/MBhgSgIFpAiyhcWLNxQKxfYkk0mEJiKRCMLERp3mo6Lr+nK5HI1GFotluVxiRLTf70dHL4b9AIjydDrF+2y73VJKE4a8sVgMrRywfIvH489wX/ng3G8EPXVgQgtXa2xe4UtH7vsY+wCL58MuO9Okar6QU3psqBRmtVqhd0OSJIzquL29RdH4eDxe7iHfXsLhcPh8PlEUM5lMLpc7OzuD/qZSKcz+8fl8NMTFqNN8PPCQI56Drrl2u41hw4j8UuYTIwHxCKHghIZNBIPBaDQaiUQymUypVCqVStlsllqrTdAUYwiHomMyOUaWAu4TcAKSZRkWHDRvELnfTCZTKBQw4uD7jatOjuN9bCgiAUlFEpa6b2ez2XQ67Xa719fX7969u7m5QVhTURT6DRSRoB0QwhHRaBTW1NARaDHtrM3x9ZMNEEFjhhE6n0wm1O0NIbbb7Yj5LBYLFBuRnRCmgYiiSKOj4/F4Npu9urp6+fJlJpMx+nTNADUoUfmK0Uf0MCAhjLw65BiJdKyLcXf5fL5EIlEsFovFIkZN0twTow//6ThqLT40OYWj1WIPsvyyLDcaDRS6Isp2+BvIzJCMZe9HJDB5zHyrYOwAwuEw1rAYiU0T4/EZPPnCPmuEeWvIGuEhwVoY1oWxWOzQqIXm8nKm7qGAWSvt7b62GedowXYW1WzY1FIWHXNMHA5HKBTK5XJwn0ilUpjC80zKiomj1mIsh6fTKfoL2u02TCRQvgbw32gkO9RiDLvEtIVsNgsX88QeFAD4/X6Itfkae6DFmqbRgDVVVQVBOEyYIPmJ5wTiS/Wt2E/Y7XaYXsIx69WrVy9fvkyn00j0wbWDtfih2G63iqKgU9RMWkz7WgixqqqUyMHWDQMYc7ncq1evLi8vMYPquS2KhePR4sM5FJADvEXX63Wv16tWq7/88kutVsMUXsxsRunr/V9FfrLIL8Xj8cvLS0QkIMTxeJxsZ836faNWz2q1RqPRRCKRTqe32y1VEJPmCoJw+Mxjw0hm8263OxKJ4E1WKpV++OGHH374gSMSjwS+GqwtDrcvpw7FXmhpDEs2Ye+RgjmwyWQS1ThGH69hHIsWw0ECww4guFjtKooCLb69ve31eijDOny1EtRBB3cVLIdBag8cqRE8NasKAzgcCYIQj8dLpZKu66lUCgUnkiShUuJwjQxcLhe6P9DTjBZEVPih5ppXwY+EyfJ1zDdwLFqMDdp8Pu/1erCeGQ6HWAJLkoR48Ww2Ix+J++bu2GWjiTmdTmez2YuLi2KxeHFxQY5rcPwx94oY4FJgdJCu66FQKJPJoAumXq9brVZYmH/2U06nE2NJs9lsJpNBsRqaoMg/zJDTeSawHD9njkuLZVlut9s3NzcYJIwyNbJ+RziYahIPB65YLBZUAni93mQyWSgULi8v4Rn24sUL2x5z6+8hOF+HwwEj/HQ6HY/Hg8EgMtSIwuNqHHagwt/n4uLi4uIC8zgSiQQMiNkG/mGhG9j0y4LDahw6ZQQujD604+JYtBhdBt1ut7Wn3++jZAIqrGkaZV0x59Hj8aAQDVEn9G7APxek0+lQKPQcbvffARdNEIRQKJTNZlFVks1mX79+fVj/BxC2Q1AiHo9Dx51Op/lym4ZDE7JlWZYkyXyVPARV46DHNRqN0vQyow/tuDgiLYa7LmmxJEkI9iMigfkrmNjm9/tRHgBXRpRMoFIik8kg4olKCTJae85ajPh4KBSCIWE2m0UJyn3bTJjoew8g2yBDDt7E0OjbwWBAYwBNeZdiUM52u0UeIhKJzOdzi8VyOMCBEY5Hi6mgB/WVaP3CX2G7LQgC8q2BQCAcDmPlGw6HqYfyxYsXpVIpn88jWMHjfAA2hsI+L2f04TD/H5pu1+/3w+EwYkfUemOmeBrWxYIghEKhaDSaTCaXyyUKKHVd93g82OA+ty67+xyLFvt8vnQ6vdvtRFE8Ozt78+bN/R00ShE9Ho/P50NEwufz4ftzOp0ok6BiNSNOgmG+FI/HE41GBUHQdd3n8+VyOZvNhlJuVP6YxmIJOzNd12Ox2MuXL+12e6lUQumerut4/WCcks/nM/pgjcTy2KlbShD9/scURUEfBzVx3N9BU1Ovw+FAjwYllGw2GyISVCV+bHL8hdfhQX7KZJjy0qmqCh9BmJFi5AriQjAqCwaDD+IEbfjVo9EEsKSAGQVaB4T9rhfhmmMzR3ziW+hYtNj0GP5InC586b4HvnrfzBNfhONaPDIMwzxPWIsZhmGMh7WYYRjGeFiLGYZhjIe1mGEYxnhYixmGYYyHtZhhGMZ4nqjv7pl3N34nfPW+Gb503wNfvaeE18UMwzDG8+h9dwzDMMz/hNfFDMMwxsNazDAMYzysxQzDMMbDWswwDGM8rMUMwzDGw1rMMAxjPKzFDMMwxsNazDAMYzysxQzDMMbDWswwDGM8rMUMwzDGw1rMMAxjPKzFDMMwxsNazDAMYzysxQzDMMbDWswwDGM8rMUMwzDGw1rMMAxjPKzFDMMwxsNazDAMYzz/D9uiXMSYT378AAAAAElFTkSuQmCC\n", "text/plain": [ "" ] }, "execution_count": 56, "metadata": {}, "output_type": "execute_result" } ], "source": [ "Image('images/mnist1.png')" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "We can interpret this as a big array of numbers:" ] }, { "cell_type": "code", "execution_count": 57, "metadata": {}, "outputs": [ { "data": { "image/png": "iVBORw0KGgoAAAANSUhEUgAAA4cAAAGOCAIAAACrH+EgAAAAA3NCSVQICAjb4U/gAAAAGXRFWHRTb2Z0d2FyZQBnbm9tZS1zY3JlZW5zaG907wO/PgAAIABJREFUeJzt3Xt0nPV97/vf88x9pJFk2cKYOPhSuxizsSxMnGSXi3FCNknLPT2rTbNOEroPXWGFJuEkzYJ410cN8cnqDqc7pIuek0VpcnpK9unem4J7gZBAIKdtCnF8ix0cHLAFji35MrqNNTPPPJfzxw/9/PiZkTwezcxzmffrD62RPPP9PtKMxIffbTTHcQQAAADgK93vCwAAAABIpQAAAAgAUikAAAD8RyoFAACA/0ilAAAA8B+pFAAAAP4jlQIAAMB/pFIAAAD4j1QKAAAA/5FKAQAA4D9SKQAAAPxHKgUAAID/SKUAAADwH6kUAAAA/iOVAgAAwH+kUgAAAPiPVAoAAAD/kUoBAADgP1IpAAAA/EcqBQAAgP9IpQAAAPAfqRQAAAD+I5UCAADAf6RSAAAA+I9UCgAAAP+RSgEAAOA/UikAAAD8RyoFAACA/+J+XwAAnHPil4cmKk7117VE37o1y9p/PQDQgfz6U6w5To2uAOCLbVuufa5Q6x+6b9n10sPtvhoA6Eh+/SlmBh9AGGSSfl8BAHSKZMafvszgAwia/rue2PnQhrTflwEAHeqPn931x+d/pbR/x233PJVvcd9GUqmmaU2/DgDtwaIdAEAwMYMPAAAA/zU+g8+ICxAuzHIAAIKMsVIAAAD4j1QKAAAA/7EHH0AY2flXHn/k6bFF/fZU99b7771+IEZ96ge/VxD6+tg6Sk8r9VvDuXgNPxCAj0Lxm/vlGzdt2nTzV/cV57+bdeZ7n7vt8y/kLceeeOmLdz7w/JnmXgb1o13fr15B6Otj6yg9rR1Yv7jvqzdv2rTpxi8390o8mMEHED6FfTv3xK9c2a0LLbtiXXz3zr3Up37wewWhr4+to/S0Ur9FSKUAwid/ZLSc6k3rQohYpi9VPjFCfeoHv1cQ+vrYOkpPK/VbhFQKIHzMckUdTafpml2apj71g98rCH19bB2lp5X6LUIqBRA+md6M7ljytm1aejpHfeoHv1cQ+vrYOkpPK/VbhFQKIHxyy5ely1NFSwhhFSfKyaWXU5/6we8VhL4+to7S00r9FiGVAgif3OCdm+2Dh6dtYU+/cdDYeMcQ9akf/F5B6Otj6yg9rdRvEc25+DcOle9b2MADAfgoFL+527Zc+1yh/64ndj60IT3vHe3JXX/16PcLvanJU4kbPvPpLUubfPgy9aNd369eQejrY+soPa0dV7+0f8dt9zyV775l10sPN/dS3EilQKcIxW9u3akUANA+7UmlzOADAADAf6RSAAAA+I9UCgAAAP+RSgEAAOA/UikAAAD8RyoFAACA/0ilAAAA8B+pFAAAAP4jlQIAAMB/pFIAAAD4j1QKAAAA/5FKAQAA4D9SKQAAAPwX9/sCAKABdv6Vxx95emxRvz3VvfX+e68fiFGf+sHvFYS+PraO0tNK/dZwLl7DDwTgo1D85n75xk2bNt381X3F+e9mnfne5277/At5y7EnXvrinQ88f6a5l0H9aNf3q1cQ+vrYOkpPawfWL+776s2bNm268cvNvRIPZvABhE9h38498StXdutCy65YF9+9cy/1qR/8XkHo62PrKD2t1G8RUimA8MkfGS2netO6ECKW6UuVT4xQn/rB7xWEvj62jtLTSv0WIZUCCB+zXHFmb2u6ZpemqU/94PcKQl8fW0fpaaV+i5BKAYRPpjejO5a8bZuWns5Rn/rB7xWEvj62jtLTSv0WIZUCCJ/c8mXp8lTREkJYxYlycunl1Kd+8HsFoa+PraP0tFK/RUilAMInN3jnZvvg4Wlb2NNvHDQ23jFEfeoHv1cQ+vrYOkpPK/VbRHMc58L38jxG04QQDTwQgI9C8Zu7bcu1zxX673pi50Mb0vPe0Z7c9VePfr/Qm5o8lbjhM5/esrTJhy9TP9r1/eoVhL4+to7S09px9Uv7d9x2z1P57lt2vfRwcy/FjVQKdIpQ/ObWnUoBAO3TnlTKDD4AAAD8RyoFAACA/0ilAAAA8B+pFECATJhCCGO8YPt9IQCAc+zCuCGEMCda2oVUCiBApiwhRGXyrOX3hQAAzrHOTlaEENZUS7u07yAKNJfaSS1vuD/WeWMucqe2+/b8H+ssBdTj8pT4eaVr5dKE3xcCADgnsXRll9hdTrX2vH1SaYg5jmPbtm3b6oa67fkoeW7LIvMkTk3TdJdYLKafz5NNq28AAADUiVQaYrZtW1XcX5Q5Vd3wfOo4jnY+MRtGVSSNx+PxeDwWi3k+yhvuYKppmiwor41gCgAALgqpNMTkkKdpmpZlVSoVs0p1ZvXkVxkrVQb1RNJYLJaoRSZadRnVuVZeG8EUAADUj1QaYnLg0zTNyizDMNw35kqrkgyOMoyqGXn3p/F4PJVKJZPJZDKZmiXjrBBC13VRNbaqro1ICgAALgqpNKzk8lCVSsvlsmEYno/uwFpNjpXOI5FIpF3k4KuKpLFYTN5QkVQWlJdHKgUQQN/5zncaeNQnPvGJhbf+xje+cbEP+exnP7vwvkCIkEpDzD1WahhGqVQqlUrlclndMOYg729ZltrAVPNGMpnMZrPZbNYwDBVJ5QirnNyXa0nV6lIVTD1rTAEAAC6IVBpicl2pnKaXg6OlUqk4SwZTNW7qJr8iU6mMoZ4b8nYqlVJLAlQelTP7yWTSsiw1MirzqLwhB3GJpAAA4KKQSkOseqy0WCzOzMzMzMycPXu2WCyqcVP3GKq6rVLpXNLptEylchGqmJ24TyQS8osyg8qLUXlU8vUHg05g5195/JGnxxb121PdW++/9/qBGPWpvzDW1C//7V/fKKbTurZk6LqrFrWhpRDCKZ88dODtomMUzsaWX7Npda4dbf348ba7b9hfomGv3yBSaYh51pWqVFooFM6ePXv27FnP0Kn7RqlUMk2z+sgn9TEWi2Uymeq1pHKgNJ1Oy4387uuRE/fq2nz4iaBj2PkffGXHoVu//fWtfdMvf+lTX3vxqkdu7qc+9RvnlEde2rlvyW/d8X5t/3/9m71HBq9a1Nfaju+0PbH7teTgDVdmNXv6tRd+MnLp1tXZVjf14cfb9r5hf4mGvX7DeMfRsHJP36sZfJlKz549Oz09PTU1NTk5OTExMT4+Pj4+ns/nz5w5c/r06ZMnT548eXJ0dHR0dHRsbGxsbOzkyZPy48mTJ0+dOnXq1KnTp0+fPn36zJkz4+Pjk5OTk5OT09PTcvxVDsHKDf7Vh6EyVor2KOzbuSd+5cpuXWjZFeviu3fupT71F8Keeu1ff5m+6td7Y3rP1Xd8/M4Nva3u+A7r7MTpkbfHTUdosXgs1pa1T+3/8ba/b9hfomGv3zDGSgNqnmCn3lxUHZjvXlfqnseXPJ+qr5imGT+fOiFfchwnOUtN/cu5e/kxHo8L1+FQRFK0Tf7IaDl1Q1oXQsQyfanyiRHqU38BnPLYG3nLOv7Tf82L4njpkvfecHXLRyyFEEJomXddsWjXv/zjM7+64t0J5/KN12ba0LTtP14f+ob9JRr2+g1jrDTo3O8R6h6blHlUnf2k9te7s6l7G757E707OLqLeypXH8hf/annfaQYLsUCTZhCCGO8YM9/N7NcUa8wTdfs0nRzL4P60a5fxSkXSrZldF35vn//G9etL73y7K7TLe44S8u8++r1S/v00V/84u18oWS14y9n23+8PvQN+0s0gPXtwrghhDAnmnslHqTSQPNEUndqVBnRfYS+e0uT+3wodZa+CqZztah+CyhPNq3+1D2VTyTFAo2UhRAzR0eN+e+W6c3ozjvLmm3T0tO55l4G9aNdv5oe10VyyWU9MaHFu3rjk0febnVHyZ4+9KO98fd8+I67b9l02cyBl38y1oam7f/xtr9v2F+iAaxvjB6dEUKUWzuqSioNLhXvqrOpJ5W6B0olz8lQMraqQU3PniRP6pXvYjpXEp3rTU1ZXYqmGOwSQvQNrUnPf7fc8mXp8lTREkJYxYlycunlzb0M6ke7fhU93d+fErN/smxHaO35j6NTPPG2tWJlX1xLDay//oMbspNn2tC17T9eH/qG/SUawPrpNUN9QoiuweZeiQepNOiqx0qrh0vd2VQFU/dB+u6BUs8kvvs4J89Yac3p++rBVMIo2i83eOdm++DhaVvY028cNDbeMUR96jfgrd2vvpY3HSESl6xflZo8ddYWjjEx4QysW9mijtLxg/vfmDQdIRLdXeXR0yVHCOFYJSNzySUt7Su17cfrY9+wv0TDXr9h7HYKNOd8MvnJj/NEUnX2k/v9nGrO4HuKe/rOv7RUkufqy1P0CaZoG633uge3HX70W998PTV5avUDD97U5DNNqB/t+srbh352Krnuiv4ePbn8+v/w6y//6J+ejdtW/40fGWztsVCjb76eT6xe3dudXLb5N6Z/8uq/Hk8I04ovHbpmSUv7Sm378frYN+wv0bDXb5jWQICQb9tD8mgpzxBp9W3TNMdnqeOfJiYm1O3p6WnTNNX0vQqv6rZlWTJQqmTpvq1pWjabXbRoUV9fX19f36JZvb296nYmk0nMS735E4IgFL+527Zc+1yh/64ndj604QKT+EADvvOd7zTwqE984hMLb/2Nb3zjYh/y2c9+duF9gaYo7d9x2z1P5btv2fXSw63rwlhpcHkGMqsn8dVAqXu3k3vDk7qDGtqsHs50Ly11HEe+RZNt27qu11xX6vlURthYLMZYKQAAWAhSaQjMtdvpgsFU7ZH3bEtyqt6BybZtdeaovOHe8+RZbOpZVxqLxeRHgikAAGgYqTTQPHlUbUiqHit173Nyp1IVYe3z34fJ00XdkJO8MpiadXBHUvIoAABoGKk0uKq3Os0zVurZ7SQ3PM2zMlUF0OpxUzG7ALF6M75nBl/elsHUs8GfbAoggJqyQrQxLBIFLohUGiZOrdPv68x/nrvJyfq5/lV+Wr2Y1ZM73bfVlZBHAQBAA0ilwSWn0dUNuTterfuUe4wUzzvayy3w7rFVuVRUzE7Tkx0BAECgkEqDzhNMVTZVm99lJJUfZRhNJpPyo1pOqmmaZb3z3mJMrwMAgAAilQZazUgqhFCpVNd1FUndo6TJZFKmUnnQvSqohlp9+5YAAABqIZUGnVaLe6C0OpK6D7F3Z1CZZd2HQPn7rQHVJkwhhDFesC94TwBA29iFcUMIYU60tAupNDTUQKmkgql7Ht8zXFqpVOSd3fuTZDD16ZsALmCkLISYOTpqCJH1+1oAAO8wRo/OCCHKIy3twhtCBlf1+GjN4dK5ZvDVR/l1mV9lEb+/M2BOg11CiL6hNbzdKAAESHrNUJ8QomuwpV0YKw06FUPF+cOl7oHS6hl8ua5UTtOrk0rlG9+7CwJAR3nssccaeNR9993XlO7Dw8N13nP79u1N6QiEC6k0BNypVAghg2n10lI5LOpeVypm5+5lKpXH3TNcCgAAAohUGmjq7ZdUHpVLQtUG/Nj53ME0mUwKIeQQqfxXNYNPKgUAAEFDKg0Bd4iUh5XK3fSeGXxPJE0kEnKUVL4dqAym7pWpPn5HABA4cmZp9nASXY/xZxJoM1Jp0M0VH+Wop3vQtHozvgqj8l/Vlqk2fwtAC9j5Vx5/5OmxRf32VPfW+++9fiBGfeovQOXYC999fsRUny9+72/felWu1RuCHXPq7QOHyqs3re1p9Td4nrb/eH3oG/aXaNjrN4hUGlxq+r76i2J255Mnm6p4qhKqe9aegVJEhp3/wVd2HLr121/f2jf98pc+9bUXr3rk5n7qU79h1sxkfP0NH1qWjcV0K//agfLVV3S3/IyaI/v3nCmcHi0uX9XeP8nt//G2v2/YX6Jhr98wToYKB89O/JonRnkWm+q1EEkRDYV9O/fEr1zZrQstu2JdfPfOvdSn/oLoi9auW7X8XZddujh+5syS91y9JN76v5SrNlyzcW1/+weHfPjxtr1v2F+iYa/fMFJpCLjHR6uD6TwDpe5s6sm1fn4/wILlj4yWU71pXQgRy/SlyieafLAz9aNdv1osd9ll3TEhrPyBveUr1i2O9Dxi+3+87e8b9pdo2Os3jFQadO4YKuYYKK1eWqoiafWKUiIpIsAsV9TSFk3X7NI09am/cE7p2M9+1fdrixPtaecXv3687ewb9pdo2Os3jFQaAjXzaM1gWh1Pq8dKyaaIgExvRncseds2LT2doz71F8yZeetnxxMD3e3d9tH+v8U+/Xjb2jfsL9Gw128YqTTQPNnRE0+rV5RWT+Uzg49Iyi1fli5PFS0hhFWcKCeXXk596i+UY5x6M6/nMm3fi3z+ntbW8+fH296+YX+Jhr1+w0ilQVdzBl/dqA6jHux2QrhMmEIIY7xgz3+33OCdm+2Dh6dtYU+/cdDYeMdQcy+D+tGur/zqZ3sOT5jvhEK7OF6w9ESs3X8iHSHaG0vb9uP1sW/YX6IBrG8Xxg0hhDnR3CvxiPSK7qjQNE2eD+UOlHNteKqex9c5GQrhMVIWQswcHTWEyM5zN633uge3HX70W998PTV5avUDD97U5DNNqB/t+srxXx46k1izpi8nhBBaLJXN9PSm2jZac+zg7uNnThlFfd8u49LV668YSLWnb9t+vD72DftLNID1jdGjM0KIcmv3RWnOxU8daLXO0URzzfPjlf9k2/bU1NT09LT8KG9MTU0VCgV1o1gszsyqvl2pVOa/hkwms2jRokWLFvX29i6a1dfXp25nMplUKpVKpZLJZDKZVLfVR11nMD5AQvGbu23Ltc8V+u96YudDG9J+Xwsi6LHHHmvgUffdd19Tug8PD9d5z+3btzelI9Aspf07brvnqXz3Lbteerh1XRgrDZ/qzU9zbXiqua6UsVIAABBADGWFknM+27blR8WyLPt8nof4/R0AAACch7HScHDnSHlb5VEVQyWzivy6O5v6930AQOdiXh6YH6k06FSIVGFUflqdRz2RtFKpuCMpwRQAmrVCFEArkEoDzZ1EPR+rp+xrxlP1Fc8kvr/fFwAAgAepNBzcYVTecK8fnWu4VH7qXmPqruDztwQAAOBCKg06d450j3RWb2xyJ9GaY6XM4AMAgMAilQaXZ4dTTXMtLZWLSufa7UQqBQAAQcPJUIE2Vxidf11pzQ347mDq97cFAADgRSoNh3kiaT178Kt3O5FNAQBAoDCDH3TV60prnpk//3ml1cfp+/1tAYAPeMdRIMgYKw00p+qw0osaMXV/kUiKUJgwhRDGeMH2+0IAAOfYhXFDCGFOtLQLY6Uh4FSdVFo9aCpvuFeRerbeN3ZeaXUC9gTf2CzrfPKespGmaaqg+3b1p8BIWQgxc3TUECLr97UAAN5hjB6dEUKUR1rahbHScHCqziuda9uTO5uqTU7uFNtA65qRtDKresHAPCsHGK/F/Aa7hBB9Q2vSfl8IOo/jOK7/peevFOCWXjPUJ4ToGmxpF8ZKg849iV9zuLR6mWl1NvWMlYr6TtGff5S0UqkkEolYLFapVHRd13U95iLvLGZHQ7VZsjJDpFgwO//K4488Pbao357q3nr/vdcPxKhP/QWoHHvhu8+PmOrzxe/97VuvyrV64MYxp94+cKi8etPanlZ/g+dp+4/Xh75hf4mGvX6DSKUh4EmT1eba+bSQSKpaVxd3D5R6wmg8Hnd3F6486gmmgmyKBbDzP/jKjkO3fvvrW/umX/7Sp7724lWP3NxPfeo3zJqZjK+/4UPLsrGYbuVfO1C++oruls8lHtm/50zh9Ghx+ar2/i1s/4+3/X3D/hINe/2GMYMfXNXZca5IWnOs1D7/jFK3i70Mx7Vo1TNc6j6x35p9g9MLxmJggQr7du6JX7myWxdadsW6+O6de6lP/QXRF61dt2r5uy67dHH8zJkl77l6Sbz1SXHVhms2ru1v/+CQDz/etvcN+0s07PUbRioNtHnSpCeSOrX243tCqnP+4tT6L8BdWaXPudaV1tz+77hWlJJNsXD5I6PlVG9aF0LEMn2p8okmL8CnfrTrV4vlLrusOyaElT+wt3zFusWRnkds/4+3/X3D/hINe/2GkUrD5ILT92p8tHrAsoE46OniGSt151H3cGn1ZvzqEVNxMasIgGpmuaJeQJqu2aVp6lN/4ZzSsZ/9qu/XFifa084vfv1429k37C/RsNdvGKk0NGpGUk82tVxvLupOhI5rX1QD2dQzVuqeu59rG76KpE1ZRQB4ZHozumPJ27Zp6ekc9am/YM7MWz87nhjobu+2j/avr/fpx9vWvmF/iYa9fsNIpSHgiXHuJFpzuLRmJG0sEVanXuv8k6Hmmbiff6wUWIjc8mXp8lTREkJYxYlycunl1Kf+QjnGqTfzei7T9r3I7f6b6M+Pt719w/4SDXv9hpFKg845/2Qo5/yd+J6hUHd2nGu/kafsBbt7IqlVdV5pzaWl1cOl1eO7LfqJoRPkBu/cbB88PG0Le/qNg8bGO4aoT/0G/Opnew5PmO/8MbKL4wVLT8TaPXjpCNHeP4dt+/H62DfsL9Gw129YpFd0h191iJxr+t4ziV9zhHIhw6WeyOteURqPx+caLtX1d/63Rx0L5bCoFM2g9V734LbDj37rm6+nJk+tfuDBm5p8pgn1o11fOf7LQ2cSa9b05YQQQoulspme3lTbRmuOHdx9/Mwpo6jv22Vcunr9FQOp9vRt24/Xx75hf4mGvX7DtAbygTxmkmDRUtXDn56gWalU8vn8+Ph4Pp+fmJjI5/PyU3V7enraHR+rb9v2Bd5qPJlMdnV1dXV1ZbNZz0cpcyHymP14PD7XR44sbadQ/OZu23Ltc4X+u57Y+dAG3t4JzffYY4818Kj77ruvKd2Hh4frvOf27dub0hFoltL+Hbfd81S++5ZdLz3cui6MlQadHGJUg46KOr4+Ho/H4/FEIpGclUql0um0nFiX77qkxilV2K2ntTO79V7N1xuGIZvKfFO9TlSNicq+juPE43H3feQ/BTwYAQCA9iOVBpd7ytv93kgyodaMpDKPptPpTCYjo2R1KrUsy/MeS/OQ95elDMNQ4ViV8qxYrU6l6g5iNpLKmX2CKWqaMIUQxnihrv9xAgC0h10YN4QQ5kRLu5BKg04lSJVHZZ7TdV1FUkmmUhVMTdOMxWIykgrXkgD1lQvyjJWqIVIhhFpg6gmX7lQai8XUmtfqtEoqRU0jZSHEzNFRQ4is39cCNBnz8ggvY/TojBCi3Nrz9kmlgabyqBBC13WZKWWec7/1vIqkarhUjpW6I6mKmGr0tJ4LUKta5bCrOD+qelKpZ0A3FovJ+7u/aFmWTKsS60rhMdglThT6htawqBQt0awVokCnSa8Z6hNH8l2DLe1CKg0BFd3UKKOcjnfP4EupWXJdqbyb2izlXmZaT181Ta8Crnunv9ov5RkHVTuZEomEDLL6rFgspnZrte7HBQAAwohUGlw1s6NaHuoeKHWPlabT6XK5bBiGaZqeHU7uOf36x0rVOlT3KGkikZAtRK0hUvlRbbFXgVV+sXqPFAAAAKk06FR8VBFz/mBaqVTkQKllWWJ2Cl4eCCVHMWVqrKe1iqHyUxVJY7GY3IyvAqs7ksr6aqzU/fV4PO7eINWinxgAAAgjUmmgyQDqCaZidsNTzT34MpWqRZ+ed2O62LFSGUzF7GZ8/Xwqlbrn6GX2lUHZ83V5VYyVAgCAaqTSoPPER811ELpKpdXBVB5W6s6j7oHSi4qk8qNWi8qpatZehtF4PC4vRm3JSiQSKpKSSgEAQDVSaUDVDKOer6jA557B97zzpzpqVIZFGR8vdsPTXP8qN9R7xkfdqVR93f1OpOx2AgAA1UilIeYOpslkUs7de0Yi5aJSOXoq82L9kRQAAKBtSKUhJhd0yhn8ZDIpj2pyD23KSGrMcg+XEkwBdKAnn3yygUd97GMfa0r34eHhOu/ZxPP2n3nmmQYedfvttzfrAoD6kUpDzL2RSA6XelKpZVmGYZRnXewefAAAgLYhlYaYewZfrtcUrsNBNU0zTVPm0VKplEwm3TP4jJUCwDlOaeyXbxVzlyzJGvmx8di71r4ry/+9A+1GKg0xNYNv23YymZRfVAOomqZVKhUZSeUB+8zgI0Ls/CuPP/L02KJ+e6p76/33Xj8Qoz71F8Ca/uWP/uGVSSFEbGDwI7evbk8kdcyptw8cKq/etLan1d/g3OzxXd99qvDBT2xZ2vJE0M6nNewv0bDXbxCpNKzcb5iUSCTkxL179FTX9UqlUiqV0ul0dSr1+/KB2iZMIYQxXrjAKQ12/gdf2XHo1m9/fWvf9Mtf+tTXXrzqkZv7m3gZ1I92/Vq0xKXvve3D6y5d0t+dbNP/tB/Zv+dM4fRocfkqH0cJ7Mn9z73w5szmdrRq49Ma9pdoAOvbhXFDCGFONPEyqpFOQkxlUHUSUyqVymQymUwmm81ms9lMJpNOp9PptIykiUTiYk+GAtpspCyEmDk6asx/t8K+nXviV67s1oWWXbEuvnvn3uZeBvWjXb8GTdMSuYFLF7ctkgohVm24ZuPafl8Hh5ypQ6+MXvKuTFuyQDuf1rC/RANY3xg9OiOEKI8090o8SKUh5t6Dn0gkUqmUzKDZbLarq6urq0tm0+qxUtaVIrAGu4QQfUNr0vPfLX9ktJzqTetCiFimL1U+0eQ/lNSPdv2anKnX//mFF3/4/b/f+aM3LjRYHxHO2cOvHl+5eVW2Pf89aOfTGvaXaADrp9cM9QkhugabeyUepNIQU28xr47QT6fTcqC0q6uru7u7q6tLDpcyg4+IMcsVddiEpmt2aZr61F8QPX3J6qvef9PWmz5ww/KR7z37WqHlHX3nlI6+evSyzWu72zVG0c6nNewv0bDXbxjrSkNMTsTruu7MErPvxuQ4TiwWm5mZmSuVMlaKUMv0ZnTHkrdt09LTOepTf0G0rpXXXCmb9+cq/9+hMXFVd8ubys7taePllI/95PUlm2/u0cwzbWrZzqc17C/RsNdvGKk0rGSsnCdcapqm8qiMpA283WjTLhdoqtzyZenyVNGe5l/rAAAgAElEQVQSImYVJ8rJpZdTn/oLYRx7/v/+p8pN//NHfi3ttP0dkX35U1s59dpb+amTf/e6sKaOzIynv/8PyY/f8RstbdnOpzXsL9Gw128YM7mda/73uAeCLDd452b74OFpW9jTbxw0Nt4xRH3qN2Bk148PnjEdIfR4qmvZyoGk5hhn3sqn12y8rEUda3CEaONf4jd//KN9pyqOSC7/0O/97kfvvvvuu29730C896qbf6u1kVS08WltQy/qtwhjpR3Hk0TVkVI+XQ7QCK33uge3HX70W998PTV5avUDD97U5PNlqB/t+srIwf0nk+uvXNwbX/re6y99+eV/PCoqpcQ1t31wdaZFHd2OHdx9/Mwpo6jv22Vcunr9FQOpNjR9c99PR1Mbrh7o04QQwszvef75V46bE/ln/z79yTvf39LWbXta29CL+i2iNTBaJhMMw2wBVyqVTp48OTrr1KlTnttTU1Pu+zcQTNPpdE9PTy6X6+np6e7urr4tt1up0wDUkVUK+67aKRS/udu2XPtcof+uJ3Y+tOEC2/CBBjz55JMNPOpjH/tYU7oPDw/Xec/t27c3paMQ4plnnmngUbfffnuzLgDRUNq/47Z7nsp337LrpYdb14VM0Ilq5pKAhxUAABBtpNJOUXPifv77NFAWAACgMaTSztXEEVN1KNVCrwkAAHQqdjt1kOrU6DhO9XLSml8EgAho1grRxjRxtWj9WCGKEGGstNMxwAkAAIKAVNpZ6py1X0hUJeZiISZMIYQx3iFvQw4AIWEXxg0hhDnR0i6k0o5T8/D8xoIpARRNN1IWQswcHTX8vhAAwDnG6NEZIUR5pKVdSKUAAmSwSwjRN7SGw0oBIEDSa4b6hBBdgy3tQirtRPVsZmrWfQAAAOpBKu04NaNkE/MlURUAADSAVNpZ6oykJEsAANBmnFfaQepMn0RSAGiF4eHhOu/ZxJNNn3nmmQYexSmn8AVjpZ2r6bP2xFkAANAwUmmn8ETGuRLkxSZLkigAAGgKZvDxDvIlQsXOv/L4I0+PLeq3p7q33n/v9QMx6lM/+L08HHPq7QOHyqs3re1pX1Mve3zXd58qfPATW5a2PBFE6WmlfkswVtoRLjhQSiRFuNj5H3xlx6EP/NGXv/CFz33gyJ997cU89akf/F4eR/bv2Xvg0LHpiuPjH2B7cv9zL7w50463RInS00r9FiGVdhwiKSKgsG/nnviVK7t1oWVXrIvv3rmX+tQPfi+PVRuu2bi239cpS2fq0Cujl7wr05YsEKWnlfotQirtaJqmEUkRRvkjo+VUb1oXQsQyfanyiSa/CR71o13fr15B45w9/OrxlZtXZdvzn4EoPa3UbxFSaWdxZ9AL5lHtQlp8scCczHJFzThqumaXpqlP/eD3ChandPTVo5dtXtvdrr/lUXpaqd8ipFIAATJhCiGM8YI9/90yvRndseRt27T0dK65l0H9aNf3q1dNPv3/vVM+9pPXl2xe19O+/lF6Wjuwvl0YN4QQ5kRzr8SDPfgAAmSkLISYOTpqCJGd52655cvS5amiJUTMKk6Uk0svb+5lUD/a9f3qNQenHVuNvCqnXnsrP3Xy714X1tSRmfH09/8h+fE7fqOlLaP0tHZgfWP06IwQotzauX7GSgEEyGCXEKJvaE16/rvlBu/cbB88PG0Le/qNg8bGO4aaexnUj3Z9v3rV5gjRxlj65o9/tO9UxRHJ5R/6vd/96N133333be8biPdedfNvtTaSimg9rR1YP71mqE8I0TXY3CvxYKwUQP3M07ufefK/f++ffzHVtWzNez78sd/9D+sX1f4rYp9++Ykfr/rErZcnWnEdWu91D247/Oi3vvl6avLU6gcevKmf+tQPfi+PYwd3Hz9zyijq+3YZl65ef8VAqg1N39z309HUhqsH+jQhhDDze55//pXj5kT+2b9Pf/LO97e0dZSeVuq3iOZc/NSB3ObSwAPRTqVS6eTJk6Ojo6Ojo2NjY/L22NjY6OiovD09fYHVzRfcz5RKpXp6enK5XE9PT3d3d/Xtrq6uTCbT1dWVzWaz2Wwmk8meT9cZrW+fhf7mOsXX/+Z/vfe/vKqv3LhhVXfh6MH9R8YT63/3T3Z8ZuvyVNVrZebV4c/8+Lf/4rPrL+4/tNu2XPtcof+uJ3Y+tOECw6VA6AwPD9d5z+3btzer6TPPPNPAo26//fZmXQCiobR/x233PJXvvmXXSw+3rgtjpQDqYhz92z95YuLWr++8/8bLkpoQwp76+d89sv1Pv3Tnv3z4i3/yxY/+ux7X/2JUjv3Tt3+6+Hc+n/TtcgEAYcNIFYC6HP4fz+bu+/pnt8hIKoTQe9bfPfzk3//5vZf9259+8tb/+Mizv8gbthCOcXL3k9s+/X+M3nzP5jZu7wUAhB1jpQDqMvarJR/+5KXePxmJS973vzz6P97/t1/74z/7T7/33f8kkilhlIW45IN//H/+wZXMwQMA6kcqBVCXdf/Tx7XemrMres+/+50d//0j//HVf/63va8dKy1ad8Nv3jK0NMk4KXC+Jq4WrR8rRBEipFIAdbns/e+b75/1ntXv+8jq932kXZcDAIga1pUCAADAf6RSAAAA+I9UCgAAAP+xrhRAXfbv+M3ff2rsIo7gT67//H994vcu548MAKAu/AcDQF2u/qP/9uJnzPpTqRZLdXVd9F+YCVMIYYwX7It9IACgdezCuCGEMCda2oVUCqAuWjyb62l5l5GyEGLm6KghRLblzQAA9TFGj84IIcojLe1CKgUQIINd4kShb2gNB/AjgoaHh+u8ZxNPNn3mmWcaeBSnnMIjvWaoTxzJdw22tAu7nQAAAOA/UikAAAD8xww+gDCy8688/sjTY4v67anurfffe/1AjPrUD34vD8ecevvAofLqTWt72tfUyx7f9d2nCh/8xJalLU8EUXpaqd8SjJUCCB87/4Ov7Dj0gT/68he+8LkPHPmzr72Ypz71g9/L48j+PXsPHDo2XXG0tvWsYk/uf+6FN2cu4sy3xltF6GmlfouQSjuL4zg1b8915/mpIvJT27Zt27bOp74i/9XzQKAxhX0798SvXNmtCy27Yl1898691Kd+8Ht5rNpwzca1/b5OWTpTh14ZveRdmbZkgSg9rdRvEVJpR1MZcSEVPGHUNE3TNA3DMAyjUqlUKhXTNGsGU6Bh+SOj5VRvWhdCxDJ9qfKJJh9WQv1o1/erV9A4Zw+/enzl5lXZ9ozVRulppX6LkEo7TnUibDgjqkFTFUxlJJVh1BNMq7Ppgr8VdC6zXFEvIE3X7NI09akf/F7B4pSOvnr0ss1ru9u1fiBKTyv1W4RU2hE8EbBmMG0sJnrGSs1ZFZeakZQRUyxEpjejO5a8bZuWns5Rn/rB71WTT2tKnfKxn7y+ZPO6nvb1j9LTSv0WYQ8+znEcR9Mu+i9UdTCtnE+lUmbw0Sy55cvS5amiJUTMKk6Uk0svpz71g99rDr78Qayceu2t/NTJv3tdWFNHZsbT3/+H5Mfv+I2WtozS00r9FmGstFNccLh0/q/Pdee51pW6Z/DVQCnBFM2SG7xzs33w8LQt7Ok3Dhob7xiiPvWD36s2R4g2/jl888c/2neq4ojk8g/93u9+9O677777tvcNxHuvuvm3WhtJRbSeVuq3CGOlnUvmwgYGRz1F5llXOs9waZO+CXQorfe6B7cdfvRb33w9NXlq9QMP3tRPfeoHv5fHsYO7j585ZRT1fbuMS1evv2Ig1Yamb+776Whqw9UDfZoQQpj5Pc8//8pxcyL/7N+nP3nn+1vaOkpPK/VbRGsgH8gcQ7AIuFKpdPLkydFZ8vbY2Ji6PTU1Je9ZHUzrjKqJRCKbzWYymWw2m06n1e1MJiNv5HK5XC7XM0ve7u3tVbdjsWCc29sZQvGb+5nrrv23UvdNj/7Tf/73Wb+vBWiy4eHhOu+5ffv2ZjV95plnGnjU7bff3qwLQDTM/OsXP/KHPyyk37frn/+8dV2Ywe8sNRPJQnblV0/iV4+VerY6MYOPeYyUhRAzR0cNvy8EAHCOMXp0RghRbu0ZUqTSjlNnMK2/msqannWl8wRTIinmMtglhOgbWpP2+0IAAOek1wz1CSG6BlvahXWlnaieocp69uO7zyv1RNJYLBaLxWqeDMWxUAAAoBqptEMtcJOTUjOSapqm67qmaeVy2XCpDqk1r2eu2wAQak1cLVo/VogiREilHUfTtHq2N9UTB6vHSiuVisyj8uHlclkF0+oTTE3TlCOy8s7uj3VeAAAAiAxSaWepGfUa3oMvzj9C3zRNXdd1/dxi5fIsz3CpyqZiNiUrF3sBAAAgGkilnW7h55XKSKrrumma6uwh+fVkMplOp2uOmMqxUiGEGltV8/7yxsKvDQAAhAiptIPUOSZ6UQOl7rFS9UD1xeqBUvVuT5KmabZtyxFW+XCZRxt771MAABBepNLOtcBIKslg6o6kQgh1VpRnBt8zfS8fpeu6LKKm/tVXCKYAAHQOUmmn8CS8pkRSMTssqmma3FAvP43FYnKZaSqVmmsbvvyoaZr7lCjPpwAAoHOQSjtRsyKpmI2hYnbQVNd1tcw0FouVSqW5hkvVDH4sFnOnUjGbTRfw/QEAgPAhlXaECybOhiOpJ5WqHUvyoxwrnefUUrU7yr0Nn2P2AUTS8PBwnfds4smmjz32WAOPuu+++5p1AUD9SKUdp56p/PrJ7ChXiMqpfPf5o/PkUUltcpIV1AZ8IikAAJ2GVNq5mrKXqOagpqpcLpflJP5c55WqHU4q1BJJO9yEKYQwxgu23xcCdAA53zX7J1fXY2wxxVzswrghhDAnWtqFVBplMuTJt6SPx+OJRCKZTCaTyVQqlU6nDcMQrlhZfaNh6uHuA/blDifDMOScfqlUKhaLtm0nEgnLsmzbVmsAyKadbKQshJg5OmoIkZ33jnb+lccfeXpsUb891b31/nuvH4g190KoH+36fvXycMyptw8cKq/etLanfU1nVY698N3nR0z1+eL3/vatV+X0eR6xYFF6WjuuvjF6dEYIUR5p7nV4kEqjTKVSFUlTqZSMpJlMxjRNmQXVR8+nC78A9/uRulNpqVSSqVSeKpVIJFQUltcsr2HhF4DQGewSJwp9Q2vS89/Nzv/gKzsO3frtr2/tm375S5/62otXPXJzfxMvg/rRru9XL48j+/ecKZweLS5f5ccIpTUzGV9/w4eWZWMx3cq/dqB89RXdLY2kkXpaO7B+es1QnziS7xps4mVUa+1LEP6qOVaqUmk6nU6n06lUKplMJhKJeDwej8djsZj7jewXSEXS6uFSmUrd8/vyDvLO7HbC/Ar7du6JX7myWxdadsW6+O6de6lP/eD38li14ZqNa/t9GxzSF61dt2r5uy67dHH8zJkl77l6SbzF4ThKTyv1W4RUGllqKlxFUplKZRjNZDLZbFamUncwlRG2icfXy2A61wx+zWCqRm2bdQ2InvyR0XKqN60LIWKZvlT5RJMnlagf7fp+9QqUWO6yy7pjQlj5A3vLV6xb3Pp0HKWnlfotwgx+lLlTaTwer57BVyqVihofVceOLjwXut+PVKVSwzBKpZK8GDG7CNU9rKuWmS78J4CoMssV9frQdM0uTVOf+sHvFUBO6djPftV39YZEG3pF6WmlfouQSqNMRj1d191bnWQklalUboQ3DMMdSeVe+KZcgHo/Usuy5L57NX2fTCZnZmbc1ynTs5zEJ5VifpnejO5Y8rZtWno6R33qB79XTb7uendm3vrZ8cTV72/LXqsoPa3UbxFSaWR5ZvA9G55kKo3FYp5RUjmu2ax1peL8bfgymKoZ/EQioc9SywyIpKhHbvmydHmqaAkRs4oT5eTSy6lP/eD3moN/f+8c49Sbeb0n054DAKL0tFK/RVhXGmUq8MViMbmuVEVSaZ7dTk25ALUH372uVM7gF2epdaXqEFOCKS4oN3jnZvvg4Wlb2NNvHDQ23jFEfeoHv1dtjhBt/Gv3q5/tOTxhzp7eVxwvWHoi1p7x2ig9rdRvEcZKo6z6ZCj3Hnz5bvXyHHu1AFSebK8Ot18gzx58NYOvQrAnNCeTSXlnUinmp/Ve9+C2w49+65uvpyZPrX7gwZuafL4M9aNd369eHscO7j5+5pRR1PftMi5dvf6KgVQbmh7/5aEziTVr+nJCCKHFUtlMT2+qPQNUUXpaqd8iWgP/7VezvS24HjRNpVKZmpqanDU9Pa1uy68XCgU5VOneDu/+aJrmhdvMK51OZ7PZrlnVt3O5XPesXC6Xy+XUF+XtZuVjiJD85m7bcu1zhf67ntj50IYLHFkKhM7w8HCd99y+fXuzmj722GMNPOq+++5r1gUgGkr7d9x2z1P57lt2vfRw67owVhplc60rlW/spI4FVTPscqVpc2fwq9eVGoahRknVkVWpVEr+K+eVAgDQmRiIiji158m9xjR+PvlFdZ8mbnWq5swSs4N27o8AAKBjkUqjTHORkVQFU7eaebR1wRQAAKAaM/jR5x4uVUnUPUqqZu1VMCWSwi8TphDCGC/Yfl8I0HxNXC1aP1aIoinswrghhDAnWtqFsdIocw+UVs/jewZKiaQIgpGyEGLm6Kjh94UAAM4xRo/OCCHKrX1vUlJpxHkiqWesdK5FpQRT+GWwSwjRN7SGDfgAECDpNUN9QoiuwZZ2IZVGlmdRac2BUhVJ1SQ+kRQAAPiCVBp9da4rZbgUAAD4iFQacdXrSuvcgw8AANBOpNIoqzmJX72olD34AADAd6TSiKuOpJ6z9NnqBAAAgoDzSqPMM32vwmj1ulL3VH79wbTON2Ryv59T9RfdRNVbPc3fggANIESGh4frvGfTTzb1sTVQP1JplKlRUvV288lk0rIs27blO9RXKhXTNA3DMAyjXC4nk0l5N5lZ5fvRy1J1xkQPmTVt27YsyzTNSqViGIaMv/Ly3BdWLpcNw6hUKvKqTNO0LEveU307nhsAACAySKWRpQZKZcSUyU/mUfmvjuOoVCojqUyl7mHUxoYwFXcklY3UWKysHIvFZMdkMplKpcrlciqVktlUplJ1teo7cn+Dzf6ZAQAA35BKo0xN3MfjcZkOVciTuVDmv3K5XCqVZCh0B1PTNKtn2MVFDpfKvpZlVSoVtUJAft22bZlHE4lEKpVKp9NquNScFYvFxPnbtlRCbf7PC2Fi5195/JGnxxb121PdW++/9/qBGPWpH/xeHo459faBQ+XVm9b2tK+pj62j9LRSvyVIpVHm3nQvhy3l12VaFUKoVJpOp0ulknsGX46VyrFVRVUQ9WVT9SjTNFUeFa4x1Hg8nkqlEomEiqTuSXzLskTVSQLu+mTTjmXnf/CVHYdu/fbXt/ZNv/ylT33txaseubmf+tQPeC+PI/v3nCmcHi0uX9X2v2S+tI7S00r9FmEPfmSpGKf2Nsk5+nQ6nU6nM5lMJpORt1OpVCqVqp7BV3v23dvzL2qTvkqf7nWlcmi2WCzOzMwUi8VisVgqlUqlkjuVykiqyIFVWa164xQ6UGHfzj3xK1d260LLrlgX371zL/WpH/xeHqs2XLNxbb8vg0O+tI7S00r9FiGVRpk7lcpBUBlAM5lMNpvNZrMqm6pg6t7tNNc7P9V/ATJHyrFSlUplBpWpVAZTlUo9G55UJHUH05o7+hEZE6YQwhgv2PPfLX9ktJzqTetCiFimL1U+MdLcy6B+tOv71avDRelp7cD6dmHcEEKYE829Eg9m8KNMpVLHceQYp0yZcuRSCCGzoEylyVnuVOqesrdt2z1cWv8MvpyIF641pqZpyvrJZLJYLMqgrMZKDcNQ60plEc/Eva7rpNKoGikLIWaOjhpCZOe5m1muqFeApmt2abq5l0H9aNf3q1eHi9LT2oH1jdGjM0KIcmv/t41UGlkqhspP5Uin3GAk06GmaXKcUs3je9aVysOh3IOjMt1e1GWoXOvej6+WBySTSTlYK6f1q8dKPfFXbd4XF7npCmEx2CVOFPqG1qTnv1umN6M7s/+3Y1p6Otfcy6B+tOv71asmH1fHt7l1lJ7WDqyfXjPUJ47kuwabeyUepNIoU9uDNE2zbVuOfaoFmkKITCZTLBbT6XTNgVI1QqlSYGPrSsXseKccbXW/u2kqlSoWi5lMplQqebY6qVTq/nZkBUEk7Xi55cvS5amiJUTMKk6Uk0svpz71g99rDj7+OWtr6yg9rdRvEdaVRpl7D748r1QewCR3O7nXlbon8T3BVFbQqtRzAWoGX54MJXOnXEU6M8u94al6D757t5N7XWmrf3QIuNzgnZvtg4enbWFPv3HQ2HjHEPWpH/xetTlC+PUnrb2to/S0Ur9FGCuNrHrO9azegO+OpPH4uZeHXMp5sYnQc//qx3p237sPK5V51LP9330ZZNNOpvVe9+C2w49+65uvpyZPrX7gwZuafKYJ9aNd369eHscO7j5+5pRR1PftMi5dvf6KgVS0W0fpaaV+i5BKO51Ke3JMVB0jJYOpups6efRiJ/E9veaJkhw+iouh9177+//pWupTP1y9zrP8qmuW+9DWx9ZRelqp3xKk0o7mGYZUVDYVs3lU/ZNa2VlnfU8MrfkV8igAACCVdjpPJJV5VKVS97s6uY8sbW4wBQAAIJV2LhUuPaOk7mDqTqULnL73tCaYAgAAN1Jpp3PP4KtjRFUwVZvfPQOl9R+kL+bIoPKLnoDLVD6ACNu+fXsHtgbqRyrtaNWLStW7jMoZfJVKPdvhG2hUM5jWvA0AADoQqbTTeYKpZxu+SqXyfUobWFfqbsSsPQAAmAun6He06kjqGS6VN9wLTxd4MtQFv6jVccwqAACIHlJpp6s+E0qNlbqDqcym7kjacDAlcWIeE6YQwhgv2H5fCADgHLswbgghzImWdiGVdi6tSs1gOtdw6QJbV39KWoUQYqQshJg5Omr4fSEAgHOM0aMzQojySEu7kEo72lxhNOFSc7jU/fB5XLA1SRQeg11CiL6hNWm/LwQAcE56zVCfEKJrsKVd2O3U0bTZA6FkEk0mk6lUKpVKpdNpwzAsy1Kp0XEcy7Li8bhpmjLF+nvlABAuO3bsuNiHPPTQQ83qPjw8XOc9m3iG1MjIRY+rrVixolndEUak0s6lBkrdqTSZTKbT6XK5XKlULMsSQsgj9C3LMk3TNM3qEVMAAICFI5V2NDV9rybu5XBpOp02TdOyLHckrVQq7nl8v68dAABECqm0o6kVpWotaSqVMgwjnU5blmVZloykKpXKNaakUgAA0HSk0s7lnsGPxWJqBj+VSpmm6c6jMpIahkEqRWDY+Vcef+TpsUX99lT31vvvvX4gRn3qB79XNevssQP7f2VmkraZWbnhioFkm/64OubU2wcOlVdvWtvT1u9XsosjL/7l30zc+sWPrki0qkXIX6Jhr98g9qx0NPe60ng8rqbv0+l0JpPJZDLydiqVkpvx3amUYAof2fkffGXHoQ/80Ze/8IXPfeDIn33txTz1qR/8XtWs6V++smdq+eA1Gzdefcnp3XvH2nQm2pH9e/YeOHRsuuK0/w955e1n/6//8uePP/nyWzOtO5c47C/RsNdvGKm0o3l2O3m24UtyAFX+q/vsUr+vHR2tsG/nnviVK7t1oWVXrIvv3rmX+tQPfq8qTmns7Qktm41rQot3Zc2TR8fb03jVhms2ru33Z7Y08e4P/8ED9390XbaVgTjsL9Gw128Y2aJzuU8q9aRSOVCazWbdwdQzVur35aOj5Y+MllO9aV0IEcv0pconmnywM/WjXd+vXtU0Xdcc4QghhHBsuzJztp3dIyzsL9Gw128Y60o7WvUMvm3btm07jiOEsG27XC6Xy+VUKiWHS5nBR0CY5Yoze1vTNbs0TX3qB79XFS2zbM3SXxyZLjuLnYlfnak4WqWN3f3nXPguDQr7SzTs9RtGKu1onhl8dyTVdV2m0lKppGbwGStFQGR6M7pjydu2aenpHPWpH/xe1bTMuze/33ntF3sPZnu6epNaMdXW7u1sVkPrQmnoX6Jhr98wUmnncr/dqDyyVB4FJQkhkrPmf+tRmWKBdsotX5YuTxUtIWJWcaKcXHo59akf/F41OJVCJbd244q0Zhz755/3vGtJW7sLX/98O3bruof9JRr2+g1jXWlHU29GrxaYuif0LyjmotItk/tYiAlTCGGMFy6wNzc3eOdm++DhaVvY028cNDbeMdTcy6B+tOv71UsZe/21kWnLEcKZeeun//LTkbO2XfzVSGnZ+hVdbeh+jtPS8crz7Hvmb3/4dvm8bk4LQ3HYX6IBrG8Xxg0hhDnR3CvxYKy007mHS9XZpSqYznMjHo/L6X5JCOG+oWkaY6howEhZCDFzdNQQIjvP3bTe6x7cdvjRb33z9dTkqdUPPHhTf3Mvg/rRru9XL+XUW0cmEpdfnuvSMsuuWDN9+pf7dpv68vcNLUu16X/pjx3cffzMKaOo79tlXLp6/RUDLV85sP+l772Z2XLDuy+JmWMv//WT//LzfYXT8b/406n33PrJ3xnsbXq7sL9EA1jfGD06I4Qot3ZfVCPRgXnbaLAsS56NP9fHs2fPjo+PT0xMTExMyBvuT8fHxw3DcAdTyfOV+a9h0aJFS5YsWbx48ZIlSwYGBqpvq+OoVFb2fNqen1U0hOI3d9uWa58r9N/1xM6HNqT9vhagmXbs2HGxD3nooYea1X14eLjOe27fvr1ZTUdGLjrBrFixolnd0Vyl/Ttuu+epfPctu156uHVd+I96p1Mz+JqmxWIxy7JisZht2+5pevd8vScXyjFRmUTlatRQRB8AABA0pNKOprmoGXy1E/+C60rV1ihN02QklbdlZYIpAACoH6m0c6n46F5aKlOpnHy/YCq1LKvm3iaZUAEAAdHEefn6MR2Pi0Uq7Wju6Xt3MPWk0rl25VuWJYOpcE3cuz8FAEhNXCQKRBWptNN5pu/du5Q8m/E9i0rj8bhpmrKIew++CqYAAL0aa7kAAAwvSURBVAD1I5V2NDWuKVOp4zgqmwohqgdKq2+7A6jc8KQWqvr2XQEAgBAilXY6924nx3FisZj6+gXXlcqxUveZUPJ9SomkAADgYpFKO5pnD77nhNF4PJ5IJOSbjqZSqXQ6nclkDMMol8vyTFO5tNQ0TdM05Q33R9M059mGL/+pq6srm81mMpl0Op1KpdS7m3re19Q9/urepNX6nxAAAGgTUmmnc0dS+VHMRsZYLCZTqcqjpmnKHfrygfIr1ix1W92oTqWer/T29vb39/f19fX19eVyue7u7mw2KxOqew2r+92nyKMAAEQSqbRzqWznCabqi3KsNJVKVSqVSqUiU6a8pwyshmFY8xIXOk4/l8v19fX19PT09fX19vbmcrmurq5MJiPHTVUkVcHUM24KAAAig1Ta0Tzz4yqSSiqVyrFPeQqpzIjxeDyZTMqoKv9JfXR/esGD9Lu6unp6enK5XE9PT09PT3d3t0qliURCDpe6R0kZKwUAIKpIpZ2uZiqVn8ZisWQy6Z6113VdLTZNp9MqrUru2/LT6naenJrNZmUS7Z7lTqXu6Xt3JGW4NMImTCGEMV7gjRgQNS+//PLFPuTGG29sVvfh4eE679nE8/ZHRkYu9iEcvB9YdmHcEEKYEy3tQirFuRl8uYle0zS5j15utE8mk/OkUvt8cie++9P5W2cyGbnbKZvNenY+qVSqnX/OP3k02kbKQoiZo6OGEFm/rwUA8A5j9OiMEKJ80f+ncVFIpR3NvbRUCKG24ctgKsdKVR6Va0nVZvxyuaym6ef6eMFUKqtVUzP47vFR1pV2gsEucaLQN7Qm7feFAADOSa8Z6hNH8l2DLe1CKoUQrjzqeX8m9xCpWksqGYbhCaDug0vnj6Tqn5IucoeTui3HSt2Do8zgw8XOv/L4I0+PLeq3p7q33n/v9QMx6lM/+L2qOMbo7h/+25HJ4tTYeO59v33HtQOJ9vxlc8yptw8cKq/etLannd/vO+ziyIt/+TcTt37xoysSrWoR8pdo2Os3iFTa0TzBTtM0lSbd72ivtje5zyJVA6Xuh7iJC23AF7NvHyVXBaiP6oZMpcJ1OilhFJKd/8FXdhy69dtf39o3/fKXPvW1F6965OZ+6lM/4L1qdJ/e/9TTB1b9wR/eseSNJ4b/8v9d8euf2dzbhr5H9u85Uzg9Wly+qv1/UCtvP/vEfztw7MCrp7b8ZsuahP0lGvb6DdMvfBdEWvU2fEmu6XRP2cudSblcrre3t6+vb9GiRf39/YsXL168ePGSJUvkxyVLlgwMDAwMDFxyySXy4/zkAxctWtTb2yv34MulpfI4fTmD79mGz0AphBCFfTv3xK9c2a0LLbtiXXz3zr3Up37we1XT0pcNbV5/WZfu2EbF1uPJNg1Yrdpwzca1/f6MSyXe/eE/eOD+j67LtvKveNhfomGv3zDGSnHu3ZLUV9TEvXsQtOYNtwuOjM7V3TMU6v7UfYWAkj8yWk7dkNaFELFMX6p8oskL8Kkf7fp+9aqmJS97/61LS5Mnf/7DH01fdcfH13e3s3uEhf0lGvb6DSOVdq55oh4pEAFnlivq/4E0XbNL09SnfvB71WbPHH/9F28V0osXZWId9qe3kZGM+oT9JRr2+g1jBh9A+GR6M7rzzoG4tmnp6Rz1qR/8XrXpudXv2fLhu3/rXQf/n7945o12dvY7A7culIb+JRr2+g0jlQIIn9zyZenyVNESQljFiXJy6eXUp37we1Vxim88+/jj/3h4xhZ6dlHGmTzS5onUhtZdNa253bruYX+Jhr1+w0ilAMInN3jnZvvg4Wlb2NNvHDQ23jFEfeoHv5fyi5ee/8mY4QjHmhx588RZR2jCnDg+JRb9+to2dD/Hael45Xn2PfO3P3y7fF43p4WhOOwv0bDXb5jWwA4V90mWAMIiFL+527Zc+1yh/64ndj60Yf6D9O3JXX/16PcLvanJU4kbPvPpLUubvEie+tGu70Mv9zuO/sN//t+OXfeH976/XzdP73nuhcOlmFOaLOU23vLha5Ymz82rt+4dR48d3H38zNjYpN5zySWXrl5/xUBK/VOL3nH0rz//+29++H/f9qFLYubYy3/95L/8fNdP34yvHLrmPbd+8ncGzx2G1bx3HA37SzRw9Uv7d9x2z1P57lt2vfRwcy/FjVQKdIpQ/ObWnUqBkHGn0jq1LpXOo0WptE7NS6VosvakUmbwAQAA4D9SKYAAeasshDh7dKzi94UAAM6pjB09K4Qov9XSLqRSAAEybgohKuPTlt8XAgA4x5oerwghzPGWduEUfQABMtglThT6htawqBRR08RFog1o4mrR+rFINErSa4b6xJF812BLuzBWCgAAAP+RSgEAAOA/UikAAAD8RyoFAACA/0ilAAAA8B+pFAAAAP4jlQIAAMB/pFIAAAD4j1QKAAAA//HeTgDCyM6/8vgjT48t6renurfef+/1AzHqUz/4vYLQ18fWUXpaqd8azsVr+IEAfBSK39wv37hp06abv7qvOP/drDPf+9xtn38hbzn2xEtfvPOB58809zKoH+36fvUKQl8fW0fpae3A+sV9X71506ZNN365uVfiwQw+gPAp7Nu5J37lym5daNkV6+K7d+6lPvWD3ysIfX1sHaWnlfotQioFED75I6PlVG9aF0LEMn2p8okR6lM/+L2C0NfH1lF6WqnfIqRSAOFjlivO7G1N1+zSNPWpH/xeQejrY+soPa3UbxFSKYDwyfRmdMeSt23T0tM56lM/+L2C0NfH1lF6WqnfIqRSAOGTW74sXZ4qWkIIqzhRTi69nPrUD36vIPT1sXWUnlbqtwipFED45Abv3GwfPDxtC3v6jYPGxjuGqE/94PcKQl8fW0fpaaV+i2iO41z4Xp7HaJoQooEHAvBRKH5zt2259rlC/11P7HxoQ3reO9qTu/7q0e8XelOTpxI3fObTW5Y2+fBl6ke7vl+9gtDXx9ZRelo7rn5p/47b7nkq333Lrpcebu6luJFKgU4Rit/culMpAKB92pNKmcEHAACA/0ilAAAA8B+pFAAAAP4jlQIAAMB/pFIAAAD4j1QKAAAA/5FKAQAA4D9SKQAAAPxHKgUAAID/SKUAAADwH6kUAAAA/iOVAgAAwH+kUgAAAPgv7vcFAEAD7Pwrjz/y9Niifnuqe+v9914/EKM+9YPfKwh9fWwdpaeV+q3hXLyGHwjAR6H4zf3yjZs2bbr5q/uK89/NOvO9z932+RfylmNPvPTFOx94/kxzL4P60a7vV68g9PWxdZSe1g6sX9z31Zs3bdp045ebeyUezOADCJ/Cvp174leu7NaFll2xLr57517qUz/4vYLQ18fWUXpaqd8ipFIA4ZM/MlpO9aZ1IUQs05cqnxihPvWD3ysIfX1sHaWnlfot0vi6Uk3TmngdAFA/s1xxZm9rumaXpqlP/eD3CkJfH1tH6WmlfoswVgogfDK9Gd2x5G3btPR0jvrUD36vIPT1sXWUnlbqt0gjY6WO41z4TgDQMrnly9LlqaIlRMwqTpSTSy+nPvWD3ysIfX1sHaWnlfotwlgpgPDJDd652T54eNoW9vQbB42NdwxRn/rB7xWEvj62jtLTSv0W0Rj4BBAc27Zc+1yh/64ndj60IT3vHe3JXX/16PcLvanJU4kbPvPpLUubfPgy9aNd369eQejrY+soPa0dV7+0f8dt9zyV775l10sPN/dS3EilAAJk25ZrnyvU+oeB23Y9+8ftvhoA6Eh/8uFrd56q9Q8tTqXM4AMIg6Lh9xUAQKcwiv70ZawUQICc+OWhiUqNP0paom/dmmXtvx4A6EB+/SkmlQIAAMB/zOADAADAf6RSAAAA+I9UCgAAAP+RSgEAAOA/UikAAAD8RyoFAACA/0ilAAAA8B+pFAAAAP4jlQIAAMB/pFIAAAD4j1QKAAAA/5FKAQAA4D9SKQAAAPxHKgUAAID/SKUAAADwH6kUAAAA/iOVAgAAwH+kUgAAAPiPVAoAAAD/kUoBAADgP1IpAAAA/EcqBQAAgP9IpQAAAPAfqRQAAAD+I5UCAADAf6RSAAAA+I9UCgAAAP+RSgEAAOA/UikAAAD8RyoFAACA/0ilAAAA8B+pFAAAAP4jlQIAAMB/pFIAAAD4j1QKAAAA/5FKAQAA4D9SKQAAAPxHKgUAAID//n94KY+RXASdvwAAAABJRU5ErkJggg==\n", "text/plain": [ "" ] }, "execution_count": 57, "metadata": { "image/png": { "height": 500, "width": 500 } }, "output_type": "execute_result" } ], "source": [ "Image('images/mnist2.png', width=500, height=500)" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "Since each image has 28 by 28 pixels, we get a 28x28 array. We can flatten each array into a 28∗28=784 dimensional vector. Each component of the vector is a value between zero and one describing the intensity of the pixel. Thus, we generally think of MNIST as being a collection of 784-dimensional vectors. Flattening the image may throw away information about the structure and it surely does. There are methods that look directly at the 2D image but will be covered in later tutorials.\n", "\n", "Now, we load the data and we see how it is organized." ] }, { "cell_type": "code", "execution_count": 58, "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "Extracting example_data/MNIST_data/train-images-idx3-ubyte.gz\n", "Extracting example_data/MNIST_data/train-labels-idx1-ubyte.gz\n", "Extracting example_data/MNIST_data/t10k-images-idx3-ubyte.gz\n", "Extracting example_data/MNIST_data/t10k-labels-idx1-ubyte.gz\n" ] } ], "source": [ "from tensorflow.examples.tutorials.mnist import input_data\n", "\n", "mnist = input_data.read_data_sets('example_data/MNIST_data', one_hot=True)" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "The dataset is split into three parts, one for training, one for testingand one for validation. Each dataset is an n-dimensional array with shape [number of examples, 784]. Each example is an image with associated a corresponding label, a number between 0 - 9 that represents the digit depicted in the image." ] }, { "cell_type": "code", "execution_count": 59, "metadata": { "collapsed": true }, "outputs": [], "source": [ "# The MNIST dataset has 10 classes, representing the digits 0 through 9.\n", "NUM_CLASSES = 10\n", "\n", "# The MNIST images are always 28x28 pixels.\n", "IMAGE_SIZE = 28\n", "IMAGE_PIXELS = IMAGE_SIZE * IMAGE_SIZE" ] }, { "cell_type": "code", "execution_count": 60, "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "train examples: 55000\n", "test examples: 10000\n", "validation examples: 5000\n" ] } ], "source": [ "print('train examples: ', mnist.train.num_examples)\n", "print('test examples: ', mnist.test.num_examples)\n", "print('validation examples: ', mnist.validation.num_examples)" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "In order to classify the digits we will use an output layer with 10 units, one for each digits. For this reason our labels are encoded as \"one-hot vectors\". A one-hot vector is a vector which is 0 in most dimensions, and 1 in a single dimension. In this case the\n", "n-th digit will be represented as a vector which is 1 in the n-th dimension. For example, 3 would be [0, 0, 0, 1, 0, 0, 0, 0, 0, 0]. Consequently, mnist.train.labels is a [55000, 10] array of floats." ] }, { "cell_type": "code", "execution_count": 61, "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "[0. 0. 0. 0. 0. 0. 0. 1. 0. 0.]\n" ] } ], "source": [ "print(mnist.train.labels[0])" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "#### 3.2.2 The Model" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "We will use a simple linear model for classification, the logistic regression; logistic regression outputs a probability for each class. In particular we will use a softmax classification for the case of multiple classes." ] }, { "cell_type": "code", "execution_count": 62, "metadata": {}, "outputs": [ { "data": { "image/png": "iVBORw0KGgoAAAANSUhEUgAABLAAAAMgCAYAAAAz4JsCAAAABmJLR0QA/wD/AP+gvaeTAAAgAElEQVR4nOzdeZydZX03/s91ZiZhDaiExaXuK1ZA9Pe4kTkTqBYLJAFj26dP1bpgta1brYB1Sd2QWlv3X8X6aO1OlCRQpSpkZgIutVD91a3u1hWICiSBLDPnXL8/QvDMmGASMnPP8n6/XiPe3+s+93zGfzh+znXfJwEAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAGaF0nQAAIA7cf8kj+n5eXSSIyedc0qSa6Y5FwAA06i/6QAAAD0WJHltdpZVJye5R7NxAAAAAGCiI5PUffx5UiNJAQCYNq2mAwAAAADAnXELIQAwk21K8vkk1yW5NsmtSdY1mggAgGmnwAIAZpIdSd6WnxdWX8vO2wR3eWQToQAAAABgbz0ynoEFADDveAYWAAAAADOaAgsAAACAGU2BBQAAAMCMpsACAAAAYEZTYAEAAAAwoymwAAAAAJjRFFgAAAAAzGgKLAAAAABmNAUWAAAAADOaAgsAAACAGa2/6QAAAA24ZIqv//Qpvj4AwLyiwNoPF1100VS/6QUAduPb3/72Ee9973snzJ7xjGe87vjjj//JvlznvPPOW3lAg03ivQIAzBxjY63+TrevVVPKeGegP0l2jJWBWlulW0trfLy/P0nGuwP9tabU2mqNd1o7zxtvDZRd53VLX5J0On0LujWptdXX7ZS+lFbGO62BJOnWVn+3W1qpKePdXbO+vtS0DlqwY9MpJ/7XcJKcd955PuzaRwqs/VBrndI3vQDA7h1yyCG7mw3VWhtIs2feKwDATrUm453+dLqtdDqtdLp9Ge/0Z7zTSre2ktvXk2S800qtrdSajHV2PvGo0+1Pt5vUuvP1STLW6UtS0u220unePhvvS5J0u33p1pJaS8Y7fdP/B9+JmnKU9wj7T4G1H2qtmlIAaMD3vve9+yR5a+/sW9/61mvue9/7/vc+XmpKd0h5rwDAbLV9+6EHbx3vW7hjR//CbdsXHDo+PrBwrFsWjnf6Dx4bax2SMrBgvFMO6nTLId1O66Bu7VtYa1nY6ZRDu2ktrN2ysFvLwTWtQ2rNglrLwqb/ppli+46BG2utf9h0jtmqNB1gtnnzm99ck+T888/3vx2zxkUXXfTeWuu5tdbnX3DBBRc3nQf2xoUXXnhuKeW9pZSLzzvvvOc3nYcZ45FJvjhpdkqSa/bxOlP16efkYsz7BWYF7xWYjbxXmKi9cviwge0Di8YGxhbVWg4vnXpEkiNLq7Wo1hxeUxeVUhbVWhclObLULKoli0qyKMnh2fnPuzX6R8x1JT8cWdO+t15h/9iBBQDMR6ubDgAAk5187rUDd79+2+Lxvh2La8pxtZbFqTkqJccmObKkHpGURUkOryWLSs2iJEfU5IiMpTXWGk86ZeenJ2VnN7LrNvuy8+Dnn6wUn7Ikuen2f+6oya1JkpItpZaxpHaT3HL7+raSbE2SmrqppnSSjJdSN+98Sbmtdsv228+9uZTUmu6OWsrOa9ZsKbWOtbqtHdP0d81JCiwAAACYIk9Z+em7b+9uP7qOl8WtVllck+NSc1Rqd3FK69ikHp2axbXk6HLjlruPtZLUnc91Knf8xy4/Pyh1d9PZoSabSrK1JreWpJOUTbev3LRrvZV0anJrTXa0ku211NvSzXjS2lxKrUm5OUlqzS21Vbutbrm1prsjrWwrte4sm7rllv6B/m52lPFt3Z1l0+HdzbddccVTt+8+GTOZAgsAAAD2walnXXlMt691bEruXbtlcUk5upsc00oW15rFKTmuJouTHLV9bMeCpKSUn++GSnL7Dqnbj2fubqgt2Vk0bS7J5tRsTcmWpNyS1K01uS3JzSV1a2praym5qZu6tZXubbWWW2qr3Fpq3drf17+p1W1tubVTt37qsidtbvqPYnZSYAEAAECSM8+89pAtZdN90pdjay33qaUcV2q9V0q5V7rluJR675oc10kWpOaO/qlmZwFVkzuaqKYKqZp0y85b326uJZtLzaaabCo1m2rJpiQ3l1I21Vo3ldTNpbQ21W53Uyn1plYZ2NzpH9908JaDb7viisdtauhPgN1SYAEAADCntdvD/a3DusfUvnLvtH5eTiW5d6m5Z03uWZJ7bc6WI5LWzzdG7Wqk6h0H01VMbU+yMSk31NQbSurGlNbGnaXTz8unJDfXvnJLa7y7aby/f1NfX2fz6OqhLdMTEaaXAgsAAIBZbeXKLy+4YfwnD2zV7kNLykOS3K9bcq9d5VRNju2mb+eDpSaUU7f/9+mJubEmG0vKxqR7fUm5sZtsLLXcmNK9obTqxlYd2Lhg28D1dj/BL1JgAQAAMAvUsmTZhnv3lc5Dumk95HNf3XxOf6uTW2499Ld2jG18TivpS8qufuqOgmqqyqma7CjJDan5YcrOciol15e6c7dUut0bumnd0FrQ3bg4GzeuXv30zhRFgXlBgQUAzDS/keQFe1g7fDezi/Lzr8Ge7O+S/MuBCAXA9GgvHz6ylb6HdLJzN1VNfUhJHlIz+pCSHFLTSkly06Y7/pWwaApibExyfU2+X1J/nOQHJa0f1dL9UbfT+mFrYffHo6vbN0zcxwVMJQUWADDT3D87S6y99YQ7WfvsXcwCwBQ4/fTPLtp60NZHldp6RC3dB5RaHtBNji/JA5Ms7KZ7+86pescOqru6k+r2h5vfkJQfJfXHNflRqeXbJflxTfdHpVt/PNYd+NE1Hz1lTx+KAA1SYAEAADAlTj732oEjr7/t4Z2+elJqTkytD6slD9mabfdNLX01Nanljm/xu4vGavKdlPx3K/lWrfVnKa3vl27n+nTzg3HlFMxqCiwAAADuslPPuvKY8dL//5RST07KyTV5RLlxy307rfT9/MFUd62oqkk3yXdL8pW7L7p54J5H3fSU/v7OZdd++aHPUk7B3KbAAgBmmnfd/gPADPXkFZ86eqy7/aSavhNTuifVlJPGkweVpLWrorqLO6q2J/l6Ur5eUr9eS/la6Xb+e3x84Ou7iqoLL7zw3FLKU0op1yuvYO5TYAEAALBbp5/+sYXbFh70yNS+47upJ5eUk1PqI3fUsSNSWsntN//tZ1k1XpOvt5Iv11K/Xbqtr6R0vjw2NvBthRQwmQILAACAnHzutQOLNm56RK3lpCQn1lJO2lpzQpIjUuodD1XfD+Mp+VpqPl9qvtot+U5fq/W1bl/n66Orh7YcwD8BmMMUWAAAAPPQkjNH719atV1KltTkUblxyyNrWgt2rZf96KpqcltJ+WJqPl9KPt9t1c/v6Fvwpc+sfsLWA5kdmH8UWAAAAHNeLUPLhx9d0/ekkvrEbjJYUo/etboftwDWmny1lXy5pn6l1nLdgtp/3Scve9KPDmRqgF0UWAAAAHPQKWdvOK6/U4eS2q5ldKim9aCk3v7Uqn323SRfKCWfT6d8oVPy+Q3rBr9/gCMD7JECCwAAYA5Ycubo/fv68mtJPa0mT0q3e1zdx6aqJreVWj5fU69rpVyXbue6ow465hurVx+/Y2pSA+wdBRYAAMAsdOqKK+/RzcBgam3XkqWp9RF1HzdX1eTGVjKampFuX/6jLKpfHPlge9tUZQbYXwosAACAWaB99vC9S6e1tFvqE5Oc1ql5wB3fCrj3D1z/bk0+kZIrF3T6P+WZVcBsocACAACYgdpnDh9V+8tQak5Lclq6eUAtdV+fX/U/Nfm4wgqY7RRYAAAAM0T77OF7p5unpbaeVkt9fKlp7eMlfpqSDSUZ6XTL8IZ1S76UlL3fnwUwQymwAAAAGjR01vqH1lb57ZqyMt08IkmylzutavK9JP/WquVTff2ta678yCnfntKwAA1RYAEAAEyzJctG71NSn1ZKVtbkcUnKXt4aeFMtubrUOpxkZOjE9n+tWlW6U5kVYCZQYAEAAEyD08685lc6/eNPqzVPS+rjsnffGHhzkqtrzXBfX2t0yaNO+UJvYTWydsriAswoCiwAAIApckdp1c3K8TL+v7J3dwbeXEsuK6mrD9629ZNXXPHU7bsW1l86hWEBZjAFFgAAwAG0ZMXow1u1+5s1ZeV4xh+Rmr3Za/XjWrO6tOrq9gntT7stEGAiBRYAAMBd1FtalVofkZS92Wq129JqZM1UpwWYfRRYAAAA+0FpBTB9FFgAAAB7qX328L1rLb9dalam1sfuTWlVk5+1StZ1a1YfPbD4qtWrj9+RKK0A9oUCCwAA4E7VMrhs9IxS8qLazdKStH7pK5IfpebDpVVXD3mmFcBdpsACAADYjZUrv7xg49iNT09GX5zkMcmdP4u9Jj9LsraUunrL4sOvuu7ix4wldloBHAgKLAAAgB6nnnXlMd2+/udv3LHxBSnl2F9y+k+TurbUunrzMYvW7yqtADiwFFgAAABJlpw1elKr1X1xJ+W3UrNwT9utarKplFxSut0P11vKVSMjQ+PTmxRg/lFgAQAA81b7WcMHlZtbz+imPr+kPvrObxLMF0stf3Hwjlv/5Yornrp9ujICoMACAADmoaVnXHWv7kDrj3JzeVZNPeZOaqtOSS6tpb5jZM3QNdOXEIBeCiwAAGDeaJ89/LjSLS/rJMtKzYI7OfWmlHpxq3bft37tqd+atoAA7JYCCwAAmNPa7eH+ckTrf9eSF6VbT6650xsFv1pS3raj2/dPn7rsSZunLSQAd0qBBQAAzElPWfnpu28f2/6imvK8pN7zTk6tST6aWt/ePqm9ftWq0p2ujADsHQUWAAAwpwyduf749LXO275jx8qUctCedlvV5LbU/E1/q+/dV6055etJMrJuGoMCsNcUWAAAwBxQy+Cy0TNKynm11Ccm2eN9gjX5Xkn5y4GBHR+8cvWv3TKNIQHYTwosAABgVmuvGD4jdfT1SU7ceTfgHn0hqW8/ZPvWf7riiqdun6Z4ABwACiwAAGBWWrpidGm31jen5rF7PKlmWy35UF+r9d71ly75z2mMB8ABpMACAABmlaGz1j+022q9qVvriuz5CwVvTil/k3TfPbp26LvTGA+AKaDAAgAAZoVTzt5wXKvbXVWTZ5c9/X+Zmv+uJe9YeNv2D33iE0+5dZojAjBFFFgAAMCMdvrpn120dcHWV9Ru96UlOWR359Tk662UVw+vW7I6KXf6ICwAZh8FFgAAMCM98axrDh/oG/vTrXXbHyTlsN3dK1iTb6fk/KETBj+yalXpTntIAKaFAgsAAJhR2u3h/nJk69m1jL8mtdxrD6fdUFJes3jgqA+uXn38jtE10xoRgGmmwAIAAGaMoWWjT62lvqmmnpDd3QhYsy2t8q6F/QMXfnz1E3427QEBaIQCCwAAaNzQivUn1Np6W01t7+GU7Sn1benkL0bWtX8yndkAaJ4CCwAAaEx7+fCRpZTX1Jo/TDKwh9PWdkt55YY17a9OZzYAZg4FFgAAMO1Wrryk7ydjxzynpr6h1ize/VllNK3u+SOXDn12etMBMNMosAAAgGnVXj584sax1t8k9eQ9nPI/teRPRtcMrp7WYADMWAosAABgWpx++mcX3bZw21uSPC+pZTen3JKU124++tD3XHfxY8amOx8AM5cCCwAAmHJDy0afurVse3dJ7reb5VqTfyqtet7Ipe0fTHc2AGY+BRYAADBlTjl7w3GtbvdtNfXpu1uvyef6annp+nWDn57ubADMHgosAABgCtQyuHz090u3+6YkR+7mhJtS6ytG17Xfn5Q63ekAmF0UWAAAwAF16oqrHzKe0feVmiW7W6/JP/Z3x1921WWn3TDd2QCYnRRYAADAAdFuD/fniNYfd2rntSU5ePJ6Tb7dqt0Xjqxb+vEm8gEweymwAACAu6y9fPjEmvL+kvro3SyPp9S3Lho//HWXX/6Y26Y9HACzngILAADYb49f+emDF+4Ye01S/7gkA7s55QutVus56y9d8p/THg6AOaPVdAAAAGB2WnrW6JKFYzu+klLPzy+WV1tKyvPbJw6erLwC4K6yAwsAANgn7fZwf47M+d3U12R3u65q+VT6u88d/kj7v4fXTn8+AOYeO7AAAIC91j5r+EE5smxIyuuzm11XSf2j9klLlox8ZOi/m8gHwNxkBxYAALBXBleMPC81f5nksMlrNflEreW5G9a1vz9i1xUAB5gCCwAAuFOnnnXlMZ1W/wdSc/pulrcm9RWja9vvTkqd9nAAzAsKLAAAYI+Gzl7/G51u64NJjpq8Vks2lFqfObJ26LvTHgyAeUWBBQAA/IKVKy/p2zi2+NW1W16VpG/ScicpF25ZfOjrrrv4MWNN5ANgflFgAQAAE5xy9objNo53/yHJ0G6Wv1FLnjm6ZvAz050LgPnLtxACAAB3aC8bflpft/vV1F8sr2rNO3JkfdTomrbyCoBpZQcWAACQWpPPf+1hp9dSziqTPuiuyW2p+cPRde0PNJUPgPlNgQUAAPPcjrGBgS99+3658ea7LS+TF0u+3N9tPf2qdUu+0kQ2AEgUWAAAMK8tWTb6q9f8145ztu8Y+IW1WvOOQ7bf9oorrnjq9gaiAcAdFFgAADBPDS4bObOU+nfbdwwsmrS0JaX8/ujawX9oJBgATKLAAgCAeabdHu6vR5S3lpI/SjLhrsGafKX2leUbPjL4jYbiAcAvUGABAMA8ctrKTx4xPlb+viRnTF47eOG2r3S7i075+Oon/KyJbACwJ61ffgoAADAXnLri6oeM7xj4bHZTXt3/nj/OE3/18+9UXgEwEymwAABgHhhatv4pndr595Q8bNLS1ofe5/vrH/or/5O+vnQbCQcAv4QCCwAA5rihFSMvq6X10SRHTlr6ZrdbnviAe//om03kAoC9pcACAIA56swzrz1kaPnIJbXmrUn6JiyWrMlAPWnDZYOfbyYdAOw9BRYAAMxB7ZXDx25ubbmqJisnr9XkLxf337hyZPXQliayAcC+8i2EAAAwxyw5a/SkOlb/tZTcs3dek9tS8qzRNe3VTWUDgP2hwAIAgDlkaMX6wW6tl5bk7pOWflxa9eyRS4c+20gwALgL3EIIAABzRHvF6O/U2vr4bsqr/2iNdx6rvAJgtlJgAQDAHNBeMfqnqfXvkizsndfkH3NkXbL+X0/9YUPRAOAucwshAADMYo9f+emDF47v+FBqfVrvvCbdkvKy0bWDb28qGwAcKAosAACYpZ70G1ffrX9sxyVJTpu0NFZSXjCydvD9TeQCgANNgQUAALPQaWde8yvjrfGPJ3nYpKWftGpZtn7d4KebyAUAU0GBBQAAs0z7rOEHjbfGP5Hk/pOWvt/XbZ1x1WVL/quJXAAwVRRYAAAwiyw5a/SktOoVSY6ZtPTFgW7/r3/ysif9qIlcADCVfAshAADMEu0Vw7/eatWr84vl1ZUZqE9QXgEwVymwAABgFmgvG/3NWsu6JIdOXCkfOXj7bWeMrB7a0kgwAJgGCiwAAJjhBpeP/G5K/fuSLJiwUPPPiweO+t9XXPHU7Q1FA4Bp4RlYAAAwgw2tGHlRrXlbktI7r8mfj64bPD8ptaFoADBt7MACAIAZqr1i+Pd3V14l9c9G17bPU14BMF/YgQUAADNQe/noi1PrX2ViedUpNS8aXjf0nqZyAUATFFgAADDDtFcMvzm1njdpvL3WrBxZ1768kVAA0CAFFgAAzCCDy0cuSs0rJgxrtpWUc0bWDX6soVgA0CjPwAIAgBmhlsEVI28smVReJdtLLb85rLwCYB6zAwsAAGaA9vINr0nNKyeNt6bUs4cva/9bI6EAYIawAwsAABo2tHzkFUldNWl8a0n3zJE1Q8orAOY9O7AAAKBBg8tH/qgmF00ab6k1Tx1Zt/TqRkIBwAxjBxYAADSkvWz45SV5x6TxltLq/vrourbyCgBuZwcWAAA0oL1s9Pkp9c8nDGu2JXXF8KVLP9VQLACYkezAAgCAadZePvx/aqnvSVJ6xttLustH1g1d2VQuAJipFFgAADCNhs4aPSsp/7f0vBevSTepzx1et/TjTWYDgJnKLYQAADBNhlaMnl1rvSRJX8+4k5LfHlkztLqpXAAw09mBBQAA02Bwxcjja60fysTyKkl9yeiatvIKAO6EAgsAAKZYe8XwI1Pzr0kOnbBQ65+MrB16VzOpAGD2cAshAABMofY5ww9Lp6wvyd175zV50ei6oXc2lQsAZhM7sAAAYIosPfuq+6ZTPplkce+8JBeOrm0rrwBgLymwAABgCpx61pXHdLt9n0xy7wkLpb53eO3gnzaTCgBmJwUWAAAcYE8865rDO63+y5M8eOJK/Zf2Ce0XJqU2EgwAZikFFgAAHEAnn3vtQH9r/MNJHjthoWQ4R+ZZq1aVbjPJAGD2UmABAMABU8vhN25+X0mePHFerjt420HLRz44tK2ZXAAwuymwAADgAGkv3/C6pDxz0vhrnYFy+hVXPG5TI6EAYA5QYAEAwAHQXj764qS+atL4+2nV065evWRjI6EAYI5QYAEAwF00tGL07Jr6l5PGN6fUp45cOvSDRkIBwByiwAIAgLtg6Oz1T6y1/n2Z+N56e1JXjKwZ+lJjwQBgDlFgAQDAfho6a/1Da7e1LsnBu2Y16abUZ42sHRppLhkAzC0KLAAA2A+nrNywuNtqXZbkHpOWVo2sGfrnJjIBwFylwAIAgH10+ukfW9i3o64pyUN65zW5eHRt+/VN5QKAuUqBBQAA+6SWrQsP/YeU+sTeaUlWD504+IKmUgHAXKbAAgCAfTC0bPSCpJ4zafyFwzqHPWvVqtJtJBQAzHEKLAAA2Evt5cP/p5a8oXdWk+9loJ5++eWPua2pXAAw1/U3HQAAAGaDwRUjj08370tJ6Rnf0up0nzq8dun1jQUDgHnADiwAAPglli6/6oGl5rKUHNQzHk+pvzV8+dIvNxYMAOYJBRYAANyJ01Z+8ohO+i5LctTElfLykTVD/9ZIKACYZxRYAACwR7WM7xj465I8YtL8/SNrB9/eTCYAmH88AwsAAPagvWzDK1LyWxOGJcObFx/+goYiAcC8ZAcWAADsxtCy0afWUt/UO6vJ18d39J1z3cWPGWsqFwDMRwosAACYZMmy0V+tpf5Lmfh++eb+0nfmNR895abGggHAPOUWQgAA6NE+c/iolHpZksN6xp1a8vSr1pzy9aZyAcB8ZgcWAADcrt0e7k9f65+T3K93XksuGF3T/mQzqQAABRYAANyuHlHemtRTe2el5AOja9pvaSoTAOAWQgAASJIMrhh5Xql5Ue+sJp/LEfWFTWUCAHayAwsAgHlvaMX6E0rN2yeNN5bU3xz54NC2RkIBAHdQYAEAMK+1lw8fWWvrw0kO7hmPldJdObJ26LsNxQIAeiiwAACYt1auvKQvaX04yYN656WW5w6vWTraUCwAYBIFFgAA89bG8WPOn/zQ9iR/P7xu8EONBAIAdkuBBQDAvDS4bGRFan39xGm5LkfW5zWTCADYEwUWAADzztLlVz2wlPzfJKVn/NNWa/wcD20HgJlHgQUAwLxy5pnXHtIpfZcmOXLXrCbd0uo+c/2lp/5Pg9EAgD1QYAEAMK9sbm2+uNQ8qndWar1w+NKlH20qEwBw5xRYAADMG0MrRp+dUn6nd1ZLNuSWrGooEgCwFxRYAADMC6cu2/CIWus7Jo1/XPrrb46MDI03EgoA2CsKLAAA5rydz73q/kuSQ3vG47XmN0dWD13fVC4AYO8osAAAmPM29W15Z5JHTpzWPxld1766kUAAwD5RYAEAMKe1l48+pyTP7p2V5O9G1g69ralMAMC+UWABADBntZcPn5ha3zVp/NU6UF/YSCAAYL8osAAAmJOeeNY1hyflkpQc1DPemlKfPrJ6aEtjwQCAfdbfdAAAAJgKA2X84iQPnjCs5aUja9tfaiYRALC/7MACAGDOGVw28nsp+a1J478fWTf43kYCAQB3iQILAIA5Zck5ow8uJe+YNP5OUv+okUAAwF2mwAIAYM5YufLLC0qn/nOSw3rGW7u1LBtZO3RzU7kAgLtGgQUAwJzxk7GNq0ry6InTcsGGdYNfbCYRAHAgKLAAAJgTlq4YXdpNzpswrOWjI2uXTL6dEACYZRRYAADMeu2Vw8d2av2n0vP+tibfGx9v/W5SapPZAIC7ToEFAMAsV0vGWh8sydE9w05ft/zuNR895abGYgEAB4wCCwCAWa29fOTFSX1K76yWXLT+ssENTWUCAA4sBRYAALPWkrNGT0rKmyeNP1Nuqq9tJBAAMCUUWAAAzEpPPOuaw1utekmShT3jW7qd8jsjI0PjTeUCAA48BRYAALNSf2v8zUke1DurNS/dcPngdxqKBABMEQUWAACzztCK9ctK8sLeWS350Oi69geaygQATB0FFgAAs0p75fBhtbbeNmn83YH+sRc1EggAmHIKLAAAZpU6Vi5Kcr87jpNuq1ueeeXqX7uluVQAwFRSYAEAMGsMLVv/lJK8YNL4besvG9zQSCAAYFoosAAAmBWeeNY1h9fSem+S0jP+xqLOYa9uKhMAMD0UWAAAzAoDZezCJPfddVyTbquWZ11++WNuazAWADANFFgAAMx4QyvWD9ZSJtw6WFLes37d4KebygQATB8FFgAAM9rt3zr4gTLxveu3Fty27fzGQgEA00qBBQDAjFbGy+uS3L9nVGs3z/vEJ55ya1OZAIDppcACAGDGaq8YflK35sW9s5q8b/Sy9nBTmQCA6afAAgBgRnr8yk8fnFr+pvfWwZp875DtB/1Jk7kAgOmnwAIAYEZaOL79tUke2jOqpdbnXHHF4zY1lQkAaIYCCwCAGWfw7JHHppaXTxr/7ci6oSsbCQQANEqBBQDAjNJ+1vBBpZu/TdLXM/5B/8DYS5rKBAA0S4EFAMCMUm4ur0zy8InD+ojkJlwAACAASURBVIIrV//aLc0kAgCa1t90AAAA2GVo+fqTa3JB76wm/zi6Zuhfm8oEADTPDiwAAGaEk8+9dqCm9f5M/JD1hv4y/qKmMgEAM4MCCwCAGeHwGzefl+SECcNa//CqNaf9tJlEAMBMocACAKBxS5aN/mpNefWEYcmHR9YNfbihSADADKLAAgCgUSefe+1Aq9S/LcmCXbOa3Jjx+oImcwEAM4cCCwCARi26cctLk5zUO2vV8tKRy4d+0lAkAGCGUWABANCYU5dteERNXjdhWLJmeN3gPzYUCQCYgRRYAAA0YuXKS/o66b4/ycKe8U2d0vqDpjIBADOTAgsAgEZsHFv8Ryl53MRpfdnVly75cTOJAICZSoEFAMC0W3LO6INryhsnja8YWTv0wSbyAAAzmwILAIBpVkurk/eV5JCe4c2t8c7zGosEAMxoCiwAAKbV0LLRFyR1sHdWUs5b/6+n/rCpTADAzNbfdAAAAOaPpcuvemA3+fPeWU0+MbJ2yfuaygQAzHx2YAEAME1q6aTv4iSH9gw3l9TnJ6U2lQoAmPkUWAAATIv2spHnlGTpxGl95cjaoe82EggAmDUUWAAATLkly0bvU0t5a++slmxon9h+T1OZAIDZwzOwAACYcq1S35tkUc/o1jpenrVqVek2lQkAmD3swAIAYEoNLRt9RpLTe2el5DUbLh/8TkORAIBZRoEFAMCUWXrGVfeqpb5j0viawRMG39ZIIABgVnILIQAAU6bb3/fXSY64Y1CzrdTuc906CADsCzuwAACYEu0Vw7+V5IwJw1b9s+HLln6tmUQAwGylwAIA4IA79awrj0kt75o0/o/clL9oJBAAMKu5hRAAgANuvNX/jpLcY9dxTXbUWp6zYaQ93mQuAGB2sgMLAIADamj5+nNK8vTeWavkjRvWDX6xqUwAwOymwAIA4IBpLx8+spvWxG8dLPnyQdtuu6ihSADAHKDAAgDgwKl5Q0nu2TPptGrr2Vdc8dTtjWUCAGY9BRYAAAfE0Ir1J6SU3++dlZq/XL92yeeaygQAzA0KLAAA7rJVq2qr1tbfJOnbNavJt+vd6msajAUAzBEKLAAA7rLR/2/Ds5I8pnfWKt2XjXxwaFsziQCAuUSBBQDAXdJePnxkt9YLe2c1+djwmqXrmsoEAMwtCiwAAO6SWsufleTontH20q0vbiwQADDnKLAAANhvQyvWn1BK/qB3VpK/HLls6JtNZQIA5h4FFgAA+6mWWlvvSs+D25P8z2Gdw97QVCIAYG5SYAEAsF+Glm347SRP6p2VlFdcfvljbmsoEgAwRymwAADYZ6et/OQRNfWtE6fl48NrBy9pJhEAMJcpsAAA2GedHQN/mpJje0ZjfbW8rLFAAMCcpsACAGCfLFkx+vBa8pLeWU3eedW6JV9pKhMAMLcpsAAA2CetmncmGdh1XJMfjXf7VzUWCACY8xRYAADstaHl689J6qm9s1YtF3zqsidtbioTADD3KbAAANgrT37yxw+taf3VxGkZHV635O+aSQQAzBcKLAAA9srYIQvPS3KfntF4SvcPk1KbygQAzA8KLAAAfqlTV1z9kJq8YsKw1veOrBn6UkORAIB5RIEFAMAv1amdtyZZ2DPaOD7e/+qm8gAA84sCCwCAO9VeMXxGkjN6Z7XkT6/56Ck3NRQJAJhnFFgAAOxR+1nDB6WWt02cln8fOmHw/c0kAgDmIwUWAAB7dnNenuSBuw5r0i3p/MGqVaXbYCoAYJ5RYAEAsFvt5cP3S8ore2cl9QPDa5de11AkAGCeUmABALAHrb9IcnDP4Kd9pXNeU2kAgPlLgQUAwC9orxj+9aSeM3FaV1215rSfNpMIAJjPFFgAAExw+ukfW5ha3jFp/IXFAxv/30YCAQDzngILAIAJbjvokBcleXDPqJZW9w9Xr356p6lMAMD8psACAOAOS8+46l6l5tWTxv8wfOnSTzUSCAAgCiwAAHp0+1oXJTm8Z3RLX3f85U3lAQBIkv6mAwAAMDO0lw+3k/I7E6fl9VdddtoNjQQCALidHVgAAOTkc68dSCnvmjT+Ym7uvr2RQAAAPRRYAABk0cYtL0jN8b2zVikvGRkZGm8qEwDALgosAIB57pSzNxzXrXl976wkq9evGVzfVCYAgF4KLACAea7V7b6hJIt6RrfWVn1ZY4EAACZRYAEAzGPts4cfV5LfmzAs5cKRS4d+0FAkAIBfoMACAJinVq68pK92y7uTlJ7x1xb3H/WWpjIBAOyOAgsAYJ76ydgxzynJo3tnrZQXrV59/I6mMgEA7I4CCwBgHmqfOXxUN/XC3lktuWz92sFPNJUJAGBPFFgAAPNQ6SuvK8nd7xjUbBto9b20wUgAAHukwAIAmGcGzx55bDd5fu+sJn9+5UdO+XZTmQAA7owCCwBgXqmldPP2MvF94Hd2LFjw5sYiAQD8EgosAIB5pL1iwzOSPL53Vkp5+WdWP2FrQ5EAAH4pBRYAwDzRXj58ZGq9aNL4iuE1g5c2EggAYC8psAAA5olSymuSHNMz2p5ufVFTeQAA9pYCCwBgHmifM/ywbs0f9M5KyXtGLhv6ZlOZAAD2Vn/TAQAAmAad8paSLLjjuOb6g7YftKq5QAAAe88OLACAOW7pitGlSc7onZWU86644nGbGooEALBPFFgAAHPYqlW11an1LZPGnxlet+TvGgkEALAfFFgAAHPYyBc2/F5JHt0zqq20XpKU2lgoAIB9pMACAJijnvzkjx9aU1/XOyvJP69fu+RzTWUCANgfCiwAgDlqxyEHvbwk97xjULOtpr6ywUgAAPtFgQUAMAedcvaG45L68t5ZSd45snbouw1FAgDYbwosAIA5qL/T/bMkh/WMflJLfVNTeQAA7goFFgDAHLNk2eiv1pJnT5yWN4ysHbq5mUQAAHeNAgsAYI5plXpRkr6e0Tc2H33oe5rKAwBwVymwAADmkKXLR5+c5PTeWSnl/OsufsxYQ5EAAO4yBRYAwByxcuUlfZ1S3zJpfPXwmsFLGwkEAHCAKLAAAOaIG3cc/YxS86ieUW2l9fI9vgAAYJZQYAEAzAFPfvLHD03JGyYMa/5l/doln2soEgDAAaPAAgCYA7YfsvBlJbln7yilXtBYIACAA0iBBQAwy51y9objSvKKCcNS3jmydui7zSQCADiwFFgAALNcX+28NslhPaOfpnbf2FQeAIADTYEFADCLtVcMPzK1PHfitL5hZO3Qzc0kAgA48BRYAACzWK3loiR9PaNvbD768Hc3lQcAYCoosAAAZqnBFSO/VpKnThjW+srrLn7MWEORAACmhAILAGAWWrWqtkrNWyaNrxlZN/ThRgIBAEwhBRYAwCw08oWRZyQ5oWdUS+2+vKk8AABTSYEFADDLPPnJHz80pbyhd1aT1cPrlv57U5kAAKaSAgsAYJbZcciCl6bmXj2j7bVTzm8sEADAFFNgAQDMIu2Vw8cm5bzeWU3eveHywe80lQkAYKopsAAAZpMdrdcmOaxn8tPOWN8b9nQ6AMBcoMACAJglhs5cf3xKfV7vrJS86ZqPnnJTU5kAAKaDAgsAYJaofa03J+nrGX3zqP7F72oqDwDAdFFgAQDMAu1lw6clOaN3VkteuXr18TsaigQAMG0UWAAAM9yqVbWVUv5iwrCWT42uaa9uKBIAwLRSYAEAzHCjn9/wf5Kc0DOqJZ0/bioPAMB0U2ABAMxgZ5557SG1Vd/UOyvJh4fXLf33pjIBAEw3BRYAwAy2pbXlJam5V89oe6dTzmssEABAAxRYAAAzVHvl8LG15PzeWSl5z4bLB7/TVCYAgCYosAAAZqg6Vl6d5PCe0U/HdvS9vqk8AABNUWABAMxAQ2euP74kz584LRde89FTbmomEQBAcxRYAAAzUO1rXZikr2f0zcUDR72zqTwAAE1SYAEAzDDtZcOnJTlzwrCWV61effyOZhIBADRLgQUAMIOsWlVbKeUtE4a1fGpk3ZJLGooEANA4BRYAwAwy/IXR30lyYs+opq/78qTUpjIBADRNgQUAMEOceea1h5SSCydOy6Ujlw59tplEAAAzgwILAGCG2NS35cWpuVfPaHt/X+sVjQUCAJghFFgAADPAqWddeUxJLpg4rX995UdO+XYziQAAZo7+pgMAAJB0St+rkxy+67gmP+uM9f9Zg5EAAGYMO7AAABp26rINj0gpz58wLHnzNR895aaGIgEAzCgKLACAhnVKfVMm7oz/1tH9i9/eVB4AgJlGgQUA0KCh5etPTeqyCcNSX7V69fE7GooEADDjKLAAABpTS03fGydMkv9sn9C+pKlEAAAzkQILAKAhgytGn5bU/zVh2M3LV60q3YYiAQDMSAosAIAGnHzutQOlZsLuq9Ty0dHL2sMNRQIAmLEUWAAADTj8xs1/kOTBPaPxUjt/3FQeAICZTIEFADDNnvQbV98tKa/pnZWaDwxftvRrTWUCAJjJ+n/5KQDAHPfwJCcluVeSBUk2Jflmks8kubnBXHPWwEDnT2pyt57RreN9rdc2FggAYIZTYAHA/NSf5PlJXpyJt7H1Gk/ysSRvSPIf05TrQUm+cQCuc3qSfzsA1zngliwbvU9NfcnEafmrqy9d8uNmEgEAzHxuIQSA+ed+Sf49ybuy5/Iq2VlynZXks0neFO8bDohS6qokB/eMbshA96KG4gAAzAp2YAHA/HL/JFdn5+2Ce6uV5IIkxyZ59lSEmi/aK4YfmZpn9s5q8sbR1UNbmsoEADAbKLAAYP44KMna/GJ5NZrkL7LzNsGbkvxKkrOTvCzJMT3n/V6SLyb5qylP+nNjSdbvx+s2HuggB0QtFybp65l8Y8vRh/11U3EAAGYLBRYAzB8vSvKoSbO3JDkvSe2ZfTPJnyf5h+x8jtQje9bekOSfklw/dTEn2JTk16fpd02ppStGl3ZrPaN3VlJedd3FjxlrKhMAwGzhWRYAMD8ckZ1FVa+PJnlFJpZXvX6YZHmS23pmhyR51QFPN+fV0q31zZOGnxleu2R1I3EAAGYZBRYAzA/PS3L3nuPxJC/ci9d9K8nkB4yfm+QeByjXvDC4YvRpSR7bO2t1y/lJ2VN5CABADwUWAMwP50w6/miS7+3lay/OzsJrl4EkZx6IUPPBypVfXpCaCbuvavKx9ZcNbmgqEwDAbKPAAoC579gk/2vS7B/34fXX5xcfpL78LiWaRzaO/+R5JXlAz6hTazm/sUAAALOQAgsA5r6hJGXS7Op9vMbk85fu5ppMcvrpn12UWl/bOyslH9qwbvCLTWUCAJiNFFgAMPc9YtLx95L8eB+v8ZlJx4cnufd+J5onti3c9vIki3cd1+S2MtZ5dYORAABmpf6mAwAAU+7hk46/tR/X2N1rHp7k+/txrX1RkpyUZDDJQ5IcnWRHkp9l57ckfirJvyfZPsU59tmCg+6Rmrysd1ZKfef6fz31h01lAgCYrRRYADD3PXTS8d4+vL3XD5J0M3H39kOTfGJ/Q+2luyf5z19yztYkH0zy1uxfOTcl7vewZybJoT2jn05+mDsAAHvHLYQAMPfdfdLxj/bjGuNJNv6S6zbl4CQvSPLVJC9pOEuS5NDD75fj7nvGhFmteePI2qGbG4oEADCrKbAAYO47bNLxbft5ncmvm3zdpg0k+askf5uGHzB//0c8N6VMeJv1raMXLH53U3kAAGY7txACwNx36KTjbft5na2TjqeywNqc5Mokn0zyX0m+meSW29fukeRRSU5L8ntJ7jbptc/Izlse/3QK8+3REff41Rx13JMmDkt57erVx+9oIg8AwFygwAKAuW0gSd+k2f4WKZMflH7wfl7nztyS5FlJVmfPO8V+ePvPFUlem527rp476ZwLklya5LopyHgnSh74yBdOnl03smbJP05vDgCAucUthAAwt40l6UyaLdjPay2cdLy/O7nuzMbsvAVwb29z3JLkeUleN2lekrzpAObaK0cd98QsutvkL33svjwpdbqzAADMJQosAJj7bp10fNB+Xmfyjqst+3mdqfDaJBsmzX4tyTF7OL8e6J/S6q8POP75k3/JJ0bWDo3ctT8NAIBGH3A6G735zW/2CSoAs8qb3vSmbNq06Y7joaGhPOUpT9nn67zxjW/M5s2b7zg+7bTTctpppx2QjAfCt771rbzvfe+bMHv605+eRz/60b9w7vnnn3/Af/8973dWHnLiy+44rrWbJz3qSzn80P19Zj4AMJedf/75Opl9YAcWAMxxBx88ceNUb5m1t7rdbm69deJGrsnXbdr973//LFw48S7H66+/flp+d1//Ibnfw39vwuzGH1ylvAIAOEC0ffto1w4sTSmzyUUXXfTeWuu5tdbnX3DBBRc3nQf2xoUXXnhuKeW9pZSLzzvvvOf/8ldwJz6S5Oye4/VJTt3Ha9w3yXcnzX49ycf3P9aU+P+y8xsKd/nb7Hwo/GQHdEf1fR/6jNz/4c++47jb2ZHPXfW72XbbDd4vMCt4r8Bs5L0Cs5VeYf/4FkIAmPu+Oun4QftxjQfuxXVnglsmHS/aw3kH7A1je+XwsRkr30hy2K7ZD7+zJttuu+FA/QoAgHnPLYQAMPd9ZdLxryQ5bh+v8YRJx1uSfH+/E02de0w6/tmU/8YdeVV6yquxHZvyP1/7+yn/tQAA84kCCwDmvuH84i1zp+zjNSafv7trNu3Q/OJOsY1T+QuXnDP64JRybu/s+9/4p4yPbd7TSwAA2A8KLACY+36c5HOTZr/9/7N353F21fX9+F+fO5OwB1BA3KiirVXaWpe6AZmZgFhkS9BYa39tXarW1ra2Lqi1NtalarW1rf2qWPu12tZviZIFERHIzAREq1hpXVBbaV1qgaighCUzc8/n98ckcGdIIAmZObM8n4/HMJz3OefeFzzmcefmlc85dw/OPzrJimmzdfcq0cw4Lcl+02ZXzeQT9nXrm5Ms2bF9+23X57vXfmwmnxIAYFFSYAHA4jC9cDo9yYN389wXZep9M8eTXLAvQu1D+yX5k2mziSSXztQTrjh782Nr8sze2be+9qE03bGZekoAgEXLTdwBYHF4X5JXJzls+3Z/kvdkssi6Ow9Lcs602QeSfP8ezrt/kt+bNrswyeV3c84Dk3wve35pYn+SDyd5xLT5R3PXm7rvM01T356em8HX5KvXffuiR83U8wEALGZWYAHA4nBTkrdPm522fbarT+R7YJL1SQ7smd2W5I278Xz3y2Tx1fv1xHs453eSfCXJC3PXm7Hvys9mshRbPW0+nuQPd/Mx9tjgquFfTOpJU4Y1r661mamnBABY1KzAAoDF411JnpPkZ3pmr0zyC0nekeTzmSy6jkmyKsnLM1lE9Xp9JldJzZRHJjk3yf9J8ukkX0jypSTXJ/nx9mPuk+TnkpycZHAXj/OSJNfORMDBweH+1PKuaeNLRjcMzrXLKgEAFgwFFgAsHrclWZnJFUv375kPZtdFUK8PJXnnPk+1c/1JBrZ/7Ykmk6u9PrDPE21XDus8v6becbliTZpOaV45U88HAIBLCAFgsflmkhOS/NsenFMzeanh87Pn96eaTddmsvB6x0w9wSmnXHxQTV3TOyvJPw2vW7En/z8BANhDCiwAWHyuTfL4TN5z6pt3c1w3yceTPCmTq5q6M5zrXZn8xMPzknx7N8+5Lcklmfw0wEckuWJmok0aP2i/l2fq6rXb0qmvmcnnBADAJYQAsFhNJHn39q9HJXlskgckWZrk5iT/meTKJDfu5eNfnV3fHH5Xrkvy/u1fSXJEJkupn0hyZJKDMrkC7Efbc30jkyvJJvYy4x4ZXD18dB3PlEsFS827h88f+u5sPD8AwGKmwAIAvrr9a675/vavT7cdJEnKeHl9TQ7uGf2glvqW1gIBACwiLiEEALgHJ521+VF18vLGO5SSt4ysH7qprUwAAIuJAgsA4B50S/PmJH09o28e0X/ku9vKAwCw2CiwAADuxuCq4ROSrOydlVpev3btcWMtRQIAWHQUWAAAu1RLreWd04ZXDW9Y/pFW4gAALFIKLACAXRhYNfrMkjyhd1ZK84qk1LYyAQAsRgosAICdWL36K0tT89Ypw1ouHF63YrSlSAAAi5YCCwBgJ24Y3/LikhzbM5poOnlla4EAABYxBRYAwDQnr77k0JK8fsqw5IOb1w1c01IkAIBFTYEFADDN+MSSVyU5omd0y5Ju/x+3lQcAYLFTYAEA9Fh+1uiDS83vT52Wv7hk4wnfaycRAAAKLACAHn2d+oYkB9wxqLkuS5q3tZcIAAAFFgDAdsvPHH1MU/PrU4ad8qaRtUNbW4oEAECS/rYDAADMFZ1OfXOm/gXfNbmxeV9beQAAmGQFFgBAkoFVI09NcmrvrJTyupGRoYmWIgEAsJ0CCwBY9NasqZ1S82dThrV8enjdwPktRQIAoIcCCwBY9EauHnlOkkf3jGpJ9+Vt5QEAYCoFFgCwqA0+d3j/pLypd1aTdcMbVvxLW5kAAJhKgQUALG435qVJfqJnMt5f+l7TVhwAAO5KgQUALFonrbr0vinlD6cMaz33snUnfqOlSAAA7ER/2wEAANrSrX2vSXLYju2a/LhZ2veGFiMBALATVmABAIvSipWXPaym/E7vrKT+2eVrl29pKxMAADunwAIAFqVu+v6kJEt7Rt/ZtmS/d7YWCACAXXIJIQCw6AyeOfz4JL88ZVjrn3xm7VNuaycRAAB3xwosAGDx6XTekaT0TL585NIt/7etOAAA3D0FFgCwqAydvem0pA70zkqnefXatc/qtpUJAIC75xJCAGDRGBwc7q9N+bOp03LZ8PkrLmwnEQAAu8MKLABg8Tis8+tJHrljsyZN0ryixUQAAOwGBRYAsCiccsrFB9XUP+mddZJ/Hlk/dHVbmQAA2D0KLABgURg7cOnvl+QBPaNtNfW1rQUCAGC3KbAAgAXvpDMvvV9Szumd1eRvRtYP/XdLkQAA2AMKLABgweuWvj9KcvCO7Zr8sDve96YWIwEAsAcUWADAgrZ81egjU8qLpwxL3nrFhSfe2FIkAAD2kAILAFjQOjVvTNLfM/pWObT+dVt5AADYcwosAGDBGlw1fEJSnzF1Wl838sGh29tJBADA3lBgAQALVC2pnXdMnZUvjKwf/Md28gAAsLcUWADAgjS0avOqpD5x6rR5RVJqO4kAANhbCiwAYMF53IuuWtLU+qdThjWfHFk/NNJOIgAA7g0FFgCw4Cy7fusLS/JTPaNuk/Kq1gIBAHCvKLAAgAXl1FM/u6yWrOmdlZIPbd4w8KWWIgEAcC8psACABeW2/W773SRH3jGoub2U7hvaSwQAwL2lwAIAFowTz958/0y/VLBT3r3p/JO+1VIkAAD2AQUWALBg9DXdP01ySM/oxomxzlvaygMAwL7R33YAAIB9YeCskROT/FrvrNS87ooLT7yxpUgAAOwjVmABAPPemjW1k5J3JSk7ZrXk349YesP7WowFAMA+osACAOa94S+O/npJHts760v5/bVrn9VtKxMAAPuOAgsAmNdOXn3JoaXkT6cMS9ZtWjewqaVIAADsYwosAGBem5jof02S+90xqLm9v9P3ivYSAQCwrymwAIB566RVl/9UreX3e2el5C8u/diJ17aVCQCAfc+nEAIA81a3Nm8vydId2zX5XpbUt7SZCQCAfc8KLABgXho6a9PTknpW76yU8tqRtUNb28oEAMDMUGABAPPO41501ZKmdP586rT8y8i65R9qJxEAADNJgQUAzDuH3HDzi0vyqJ5RTad5WVJqa6EAAJgxCiwAYF4ZPGP4iKT8Se+sJP8wcv7QZ9vKBADAzFJgAQDzS1/nj5Mc3jPZWia6r2krDgAAM0+BBQDMG8vPGv3ZpL5kyrCUt276+En/01IkAABmgQILAJg3SqnvStLXM/qvHNq8s608AADMjv62AwAA7I7BlcMrk6yYMqz1VSMfHLq9nUQAAMwWK7AAgDlv8LnD+9eU6SutRkY2DH20lUAAAMwqBRYAMPf9KC8rybE9k4nSbV7aWh4AAGaVAgsAmNOeeuYVD0gtfzhlWMvfDl+w4istRQIAYJYpsACAOW28M/HmJAf3jH6w39Ilf7ir4wEAWHgUWADAnLVi5eYnJPn1qdP6povXPuWHrQQCAKAVCiwAYI6qpUnzriSlZ/jl3JR3t5UIAIB2KLAAgDlpcNXm5yR58pRhrb8/MjI00U4iAADaosACAOacU065+KCkvm3a+OMjG4YubSUQAACtUmABAHPO+IH7nZOaB/aMtnXSfVlrgQAAaJUCCwCYUwZXDj+k1rxy6rS+e9P6k77ZTiIAANqmwAIA5pSS8vaU7H/HoOa68WbJG1qMBABAyxRYAMCcMbhyeLAmq3tntZPXf3rjCTe3lQkAgPYpsACAOWH16vP6kvKuaeOrhh498IFWAgEAMGcosACAOWHL2P1+I8mje0a1U8vvrVlTmrYyAQAwNyiwAIDWDa4cPiylvnHqtJ63acPAle0kAgBgLlFgAQCtK6W8PsmRPaNb0skr2soDAMDcosACAFo1dOamR9Sal06d1neMnD/03XYSAQAw1yiwAIBWNZ3OnydZ0jP61rYl+72trTwAAMw9CiwAoDVDZ40+vSRP753V5DWfWfuU29rKBADA3KPAAgBaceqpn9ivlvqXvbNasnl0/eBH2soEAMDcpMACAFpx+/4H/naSh+/YrklTu+VlLUYCAGCOUmABALNucPXw0U3NH08bf3DzxoEvthIIAIA5TYEFAMy6Ol7eUJJlPaMf7VeWvKa1QAAAzGkKLABgVg2eOfz4JL/ROyvJWz617vgbWooEAMAcp8ACAGZRLSmdd5We9yA1+cYRS458V5upAACY2xRYAMCsGVq5eXVKPb531mnKK9euPW6srUwAAMx9CiwAYFacccZVB9bUd0wZ1nxyeOPAxpYiAQAwTyiwAIBZ8eO+rb+X5ME9o25f7ZzTH67l6AAAIABJREFUVh4AAOYPBRYAMOOGztz0iFLz+inDWt5/2cbl/95SJAAA5hEFFgAwo9asqZ3a6fv7lOzfM/7+xETnta2FAgBgXlFgAQAzauSLI89P6hN7ZzV5/RUXnnhjW5kAAJhfFFgAwIwZPHv4QbWUd06dlstG1w+8t51EAADMRwosAGDm1PJXJVl253Zub/rykqTUFlMBADDPKLAAgBkxcNbIqtSsmjLslDdt/tjAf7QUCQCAeaq/7QAAwMIzuHL4sCR/M218dW5s3tZGHgAA5jcrsACAfa6kvCXJ/XtG3U46Lx4ZGZpoKxMAAPOXAgsA2KcGVw2f0CQvnjot79m0fvnn2kkEAMB8p8ACAPaZwecO759a/rZMfY/xnfGm77WthQIAYN5TYAEA+85NnXOSPKJ3VGr5zU9vPOHmlhIBALAAKLAAgH3ipLM2P6qmTl1pVfLR4Q0Dn2gpEgAAC4QCCwC419asqZ1umg+UZGnP+Mb0199pLRQAAAuGAgsAuNdGvrj5hSl5Uu+spLx6ZO3QdW1lAgBg4VBgAQD3yorTL3tgLfXtvbOabBpev/z9bWUCAGBhUWABAPdK09//lyVZdseg5vb+0veSpNQWYwEAsIAosACAvTa4cnhlUp/ROyud/Oll6078RluZAABYeBRYAMBeGVw5fFhS/s+UYclXjug/8q0tRQIAYIFSYAEAe6fmTUnuf+dmmtI0L1i79rixFlMBALAAKbAAgD02dPam42spL+mddZL3DW9Y8S9tZQIAYOFSYAEAe2T16q8srbXzvjL1fcR399+2/6tbCwUAwIKmwAIA9siW8RtelZrjeme15ncvuuhJP24rEwAAC5sCCwDYbctXjT4yKa+bMixZN7phcF1LkQAAWAQUWADAbqqlpL43yX49w5u6pfPbbSUCAGBxUGABALtl8KyRF5Sa5b2zmrz28vOX/29bmQAAWBwUWADAPVpx+mUPTCnvmDa+YujnB97XSiAAABYVBRYAcI+6/X1/nuTQOwY1t5em+Y01a0rTXioAABYLBRYAcLeGVm06qyTPmjIs9W3DG1d8vaVIAAAsMgosAGCXjj/zikNq7fx176wmXz1yyVFvaSsTAACLjwILANil/jLxpiQP3rFdk6Z06gvWrj1urMVYAAAsMgosAGCnVpw1+pSUvLR3Vkp9/8j5Q59tKxMAAIuTAgsAuIvHveiqJU2p7y297xVK/ueA2w94VYuxAABYpBRYAMBdLLt+6yuT/GzvrNTm9y666Ek/bikSAACLmAILAJhi8BnDP11LXj9tvH54/YqPtRIIAIBFT4EFAPSoJd3Oe5Ps1zO8aUnT/9ttJQIAAAUWAHCHoVWbn5fUgSnDUl53ycYTvtdSJAAAUGABAJNOOvPS+zW1/tmUYS2fHnz08ve0FAkAAJIosACA7bql/10luc+O7ZqMlab74jVrStNmLgAAUGABABk4a+SMlDy7d1ZS3j58wYqvtJUJAAB2UGABwCI3uHr44JS8e9r4mgO23fKmVgIBAMA0CiwAWOzGO28syTE9k9ppym9edNHTt7WWCQAAeiiwAGARW7Fy8xOS+tKp0/qhTRsHNreTCAAA7kqBBQCL1JNXX3lAU5u/T9LfM97SV7ovbysTAADsTP89HwIALERLx8bempKfnjKs9bcuW3/yD1qKBAAAO2UFFgAsQkNnjT69lPzOlGHJ345sGPpoS5EAAGCXFFgAsMictOrS+9ZS/zZJ6Rl/c+kt217WViYAALg7CiwAWGS6Tf+7k9x/x3ZNmpT63E996mm3tBgLAAB2SYEFAIvI4Fmjv5SSZ/fOSspfjawbuqKtTAAAcE8UWACwSAyePfyglPreKcOSrxyw7ZZXtxQJAAB2iwILABaFWtKUv0tyWM9wvFM6v3bRRU/f1lYqAADYHQosAFgEBleO/HqSp/bOavIXm85f/q8tRQIAgN2mwAKABW75GaMPrSl/OW385QO33fr6VgIBAMAeUmABwAK2evV5fZ1OPlySZTtmNRlrmuLSQQAA5g0FFgAsYFvGjnxDSj2+d1ZSXrV548AX28oEAAB7qr/tAADAzBhcOTxYU15Teoe1XDiyYflftZUJAAD2hhVYALAAnXDa5Ycn5cNl6u/665d2+p+flNpaMAAA2AsKLABYgPqWdN+d5EG9s1KaF39q3fE3tBQJAAD2mgILABaYwVWjv1KS50wZlvq+4XUrNrQUCQAA7hX3wAKABWRw5fBDUuvfTBnWfO2Q7iF/0FIkAAC416zAAoAFo5aa8oEkh/YMJ9JXn3fBBY+/ta1UAABwbymwAGCBGDxr5OUlWdE7KyVvHDl/6LNtZQIAgH1BgQUAC8DQ2ZuOTyl/2juryaaBRw+8qa1MAACwryiwAGCeO3n1JYc2TedDmXpvy5v70n3RmjWlaSsXAADsKwosAJjXapkYX/IPJTm2d1rSPG/T+pO+2VYqAADYlxRYADCPDa7a/FtJTp8yLPV9w+tXfKydRAAAsO8psABgnlp+5uhjUus7p43/LYfmZa0EAgCAGaLAAoB56Pgzrzik06n/nGS/HbOa3NpXO88Z+eDQ7S1GAwCAfU6BBQDz0NK+ib9M8pNThiUvu2zD8q+2kwgAAGaOAgsA5pnBVcPPrjXPmzIsWTe6bvD9LUUCAIAZpcACgHnkpLM2Pyq1TCmqavLtibG+F7SVCQAAZpoCCwDmiZNXX3JotzQbkhy8Y1aTsdLUZ1xx4Yk3thgNAABmlAILAOaJibH+v0ny8N5ZqfUPRzYOXdVSJAAAmBX9bQcAAO7Z4MrR30vqr0wb/8PIhqF3tBIIAABmkRVYADDHDZ296fik/tm08TVZUl/SSiAAAJhlVmABwBw2eMbwEbUp/y/Jkp7xLSn1WSNrh7a2lQsAAGaTFVgAMEetXn1eX/o6/y/Jg3rnNXnhyLqhL7cUCwAAZp0CCwDmqBvGj3ptUk/qnZWa94+uH/xIW5kAAKANCiwAmIMGVo08tSSvnzb+8sHNwS9rJRAAALRIgQUAc8zyM0YfWmo+kqn3qry5NM0zL7jg8be2lQsAANqiwAKAOeSMM646sNNXz09y355xTS0vHN644utt5QIAgDYpsABgzqjl5r6t/5jk53unJXnTyIaBf24pFAAAtE6BBQBzxOBZm1+VZOW08ccHfn5gTQtxAABgzlBgAcAcMLBy5NRa6lumja8Zb/qfs2ZNaVoJBQAAc4QCCwBatmLlZQ9L8g+l5/dyTX6cvnr2pzeecHOL0QAAYE5QYAFAi84446oDu6Xv/JLcp2dcU/PckY8Nfa21YAAAMIcosACgNbVs7dv63lLzc9PmfzW6YXBdO5kAAGDuUWABQEsGVo6+uia/OnVaRnNTXtFOIgAAmJsUWADQgoGVI79ckjf3zmpy7X5Llpw9MjI00VYuAACYixRYADDLTjpz88+V5NwkpWe8tVOasy9e+5QftpULAADmqv62AwDAYjJ49vCDuk3zySQH94y7pTbPHF6/4t/aygUAAHOZFVgAMEvOOOOqA2tTNiS5f++8lLxqeMOKi1uKBQAAc54CCwBmRS1b+7Z+sCSPnTJNzh1eN/jnbaUCAID5QIEFALNgcNXm19Zk9bTxyNajDn5pK4EAAGAeUWABwAwbPGv4man1DdPG3+12Os/5wrmPH28lFAAAzCMKLACYQSvOGn1KSvlQkr4ds5rc2ul0zrr8/OX/22I0AACYNxRYADBDBs8cfnhT6oYkB+yY1aQptf76pvOX/2uL0QAAYF5RYAHADDjpzEvvl065JMkRvfNOyktGNgx9tKVYAAAwLymwAGAfO+WUiw/qdpZcmOQhU/fUdw6vHzi3jUwAADCfKbAAYB9avfq8vrED9v/npD6ud16TfxpZP/jKtnIBAMB8psACgH1oy8SRf5NST+ud1eTKclh9QVJqW7kAAGA+U2ABwD4yuGr4N1PLi6eN/7ssqc8Y+eDQ7a2EAgCABUCBBQD7wOCq4dNTy19PG9/UVzunjawduq6VUAAAsED0tx0AAOa7gVUjT641/1x6fq/WZCxNzr5s4/KvtpkNAAAWAiuwAOBeGHzG8E+XmgtKcmDvvJTyotGNg8Nt5QIAgIVEgQUAe2lw5fBDardcluS+U3bU8uqRdQN/304qAABYeBRYALAXTln16aOScklJHtA7LyV/MbJh4G1t5QIAgIVIgQUAe+iUUy4+aFsdvyDJw3vnJfnw8LqBl7cUCwAAFiw3cQeAPTA4ONw/dkDnn0vqE3rnNfnUzUcd/IKk1LayAQDAQmUFFgDspjVraqccXs5NqadN2/Xl7njfs79w7uPHWwkGAAALnAILAHZLLcNXj76n1jxvyjT5dmei+4tXXHjijW0lAwCAhc4lhABwD9asqZ2Rq0c/kOS503Zd35fuik0fP+l/WogFAACLhgILAO7ByNWb35m7llc/aGp56siGk77ZQiQAAFhUFFgAsEu1DK0c/cua+jvTdny/qWXF5g0DX2olFgAALDLugQUAuzC4cvOf12RKeVWTH5bSnKy8AgCA2aPAAoCdGFo18vqkvmza+JbUrBxet+LfWgkFAACLlEsIAWCagZUjb6s1r5o23tqp5WmbNgxc2UooAABYxKzAAoAegyuH15TcpbzallJXK68AAKAdVmABwHYDq0benJrXThtvK0151vDGwU+2EgoAAFBgAUBSy+Cqkfek5sXTdmztNOW0TRsHNrcSCwAASKLAAmCRW7Omdka/OPreWssLp+3aWmuevmnjwOWtBAMAAO6gwAJg0VqzpnZGrh45N6W8YNqum2vNaaMbBpVXAAAwB7iJOwCLVC0jXxz5q+Qu5dVtteQZyisAAJg7rMACYNFZvfq8vi3jo3+blOf2zmtyayfNmSPrVlzWUjQAAGAnFFgALCpPXn3lAVvGxtem1NN65zX5cV8tp27asOLKtrIBAAA75xJCABaNJ6++8oCl42MfnV5eJblpsrwaUF4BAMAcZAUWAIvCSasuvW93fOyiJL8wZUfJ/zQpTx1ZP3BNO8kAAIB7osACYME7cfXmI7vjzcVJHjNt1zdT68mb1w/+dwuxAACA3aTAAmBBO/mMK46ZmJj4RJLjeuc1ubZ2y1M3X6C8AgCAuc49sABYsIbO2HTcRN/Ep1OnlldJrlna9J+4+YKB/2olGAAAsEcUWAAsSCtWjp5S+zqfTfKgabs+k9SnXLLxhO+1kQsAANhzCiwAFpzBVcOnd1PXJTl4yo6S4aQ+fWT90E3tJAMAAPaGAguABWVw1fBvppb1JTmwd16Tv8uN9RTlFQAAzD9u4g7AgjG4avitqeWc6fOavH10/cCrk1LbyAUAANw7CiwA5r3BweH+HNb5i9T60rvuLWtG1w+8YfZTAQAA+4oCC4B57YTTLj88S7rnJfXkabu6KfWlI+sG39tKMAAAYJ9RYAEwb604e/Njm6a7IdM+abAmt5ZSf2lk3dDHW4oGAADsQ27iDsC8NLhq+BebptmUaeVVkptK6mnKKwAAWDgUWADMM7UMrhxek1o+keTQaTu/3N/X97iR9UMjLQQDAABmiEsIAZg3Vq/+ytIbxkffk5TnT99Xk0+V1F+69GMn3tRGNgAAYOYosACYFwZXDj9ky/iW80vymGm7alL/ZHT94BuSUlsJBwAAzCgFFgBz3sCqkSfXmo+W5AG985qMlZTfGlk/+IG2sgEAADNPgQXAnPaNbz34kaVmOMl+vfOa3JCaZ45sGLi8pWgAAMAsUWABMCeNj++39JpvHZP//cERJ951b7msdJtnj1ww9P3ZTwYAAMw2BRYAc87ys0YffOWXxn7/9vEld9lXat6//9gtv3PRRU/f1kI0AACgBQosAOaUwVXDz06t594+vuSQ3nlNbu3U8sLhDQP/1FY2AACgHQosAOaEwcHh/nJoeUuteUWS0ruvJt9LyTOH1w98pqV4AABAixRYALRuxcrLHtZNOS/JY6fvO+SgW//n1luXPvqydSf/oIVoAADAHNBpOwAAi9vQWZue1qTvM2Un5dWDjro+Tz7uK59UXgEAwOKmwAKgFWvW1M7gytE3NqXziSRHTtu99Zijr/+7nzn2v9LX13TbyAcAAMwdCiwAZt2KlZc9bPjq0c8k9XXlLr+Lyr90Ot2feeRPXPsv7aQDAADmGgUWALNq4KyRVU36/qUkT7jr3vr3h3QPWrHp/JO+NfvJAACAucpN3AGYFYMrhw8rKefWZPVOdv+olPL84XWD5896MAAAYM5TYAEw44ZWbXp0reUjNXnkTnZ/sSnlVzavG7hm1oMBAADzggILgBmzevV5fTdMHPUHteaNSfabtrsm9c+PXHLUa9euPW6sjXwAAMD8oMACYEYMnbHpuBvGO3+303td1VzXKeXXN60f/FQL0QAAgHlGgQXAPjU4ONyfwzvn1Fr/qNx11VVSy4VLO/3P/9S6429oIR4AADAPKbAA2GdOOnPzz010mv9ban3sTnbfVpNzRjcsf3dS6qyHAwAA5i0FFgD32uDgcH8O6/zhRJrXlmTp9P01+df+pvO8yzYu//c28gEAAPObAguAe2XwrOGTU8p7kvrwctfdt5SUPxhZv/z9Vl0BAAB7S4EFwF4ZXDl8WEl5S5O8uCSd6ftr8rlOt3n+8AUrvtJGPgAAYOFQYAGwh2oZWrn5hTX1rTU5fCerrraWlJdbdQUAAOwrCiwAdtvyM0Yf2unb/J6a+rSdH1Eu6yud37ps3YnfmN1kAADAQqbAAuAeDa4ePriOlTeXUl+SZMlODvlurfmt0Q0DF8x2NgAAYOFTYAFwtwZWjpya8fxlKfnJXRzyD90lnT+4fO3yLbMaDAAAWDQUWADs1NAZm46r/Z2/Ts3QzvbX5Kul1BePrBu6YrazAQAAi4sCC4ApTj31s8tuXXr7y2vJK1NzwF0OqLm9lPzZId2D33rBBY+/tYWIAADAIqPAAiBJcuqpn9jvtqUHvey2cvs5JTl8J4fUkvxDf+1/9SUbTvjerAcEAAAWLQUWABlYOXLqrck7SuqjdnHIf5RaXja8YeATsxoMAAAgCiyARW3FWaNPaUr+PKlP3MUhP0rKH+em5m+GRwYnZjUcAADAdgosgEVo6MxNj6idzqub1F9N0jd9f02akvrhpnb+aPOGge+0EBEAAOAOCiyARWTF2Zsf29TmNU3N2SXp7PSgkuHaLS8f3Tj4xVmOBwAAsFMKLIBFYMWZo8ubUl/TNM0vJknZ+WFfL0151fDGgY2zmQ0AAOCeKLAAFqjVq8/r+/7Y/X6llvoHTeqj7+bQ/y21vHrgMcv/Yc2a0sxaQAAAgN2kwAJYYFavPq9vy8SRq7eMl1en3G1x9a2kvmPprWP/91OfetotwxtmLSIAAMAeUWABLBBnnHHVgTd3bvmdLWP1ZSk5+m4O/XyteePQYwYutOIKAACYDxRYAPPc01ZfeZ9t49t+9+Zs/e0kR9zNoZem1DeMrBu6IklGrbgCAADmCQUWwDz11DOveMBYmThn2/jY85Ny8C4Oq0kuTKlv21FcAQAAzDcKLIB5ZvkzRn+yr1v/aCwTv1SSpbs4bKIkH+k0nXdctnH5v89qQAAAgH1MgQUwTwydsem49HX+uOnWZ9SkU3Z2UM3tNTm3NuVdmy8Y+K/ZzggAADATFFgAc9zg2cNPSlP+sCanJSk7La6SHyX1XX21+57LNp58/awGBAAAmGEKLIA5auCskTNKyjlp6vG7OqYmPyypf91Xun992bqTfzCb+QAAAGaLAgtgDlmzpnaGrx59Wkk5J6kDk/dg36nv1+SvuuN9777iwhNvnM2MAAAAs02BBTAHDK3c9LgmnRcNXz36zJLcZ1fFVU2+0anlzUcsPeL/rV173NgsxwQAAGiFAgugJSecdvnhfUu6zy41v1aTJ+3i3lY7/EdS3nbUkiM+rLgCAAAWGwUWwCw644yrDtzaueWZtdRfTbpDSfpy983VZ2rNn45uGPh4UnZ5PSEAAMBCpsACmAVDqzY9uta+596crb+c5H53d2xNmiSXJnnX6PqBTyquAACAxU6BBTBDTn7G5cdOdCd+LSnPrjWPuJsbsm9XvlCScyfGO2vdmB0AAOBOCiyAfej4M684ZGmn+8u15tfGu90nl5TOPZxyfUr9YJmoHx6+YMVXZiUkAADAPKPAAriXVq8+r++GsaOe3in51ZqJ02tyQEru7tZWt5Xko7XWDx25dMvw2rXP6s5eWgAAgPlHgQWwl05adflPdTPx/C0T5f8rJQ+8xxtV1fLpUvKhmua84fVDN81GRgAAgIVAgQWwBwbPGD4ifZ1fSfKr3dp93N2us5r0raR+sCmdf968fuCaWYgIAACw4CiwAO7BSasuvW83fU8ttTy9JquSevA9nDKe5JMlzd/vv+32j1900dO3zUZOAACAhUqBBTDN4HOH9683lqem5PQkJ3drjk3u8TMEa5LLSi0f7ls6tuHStU/90cwnBQAAWBzu8doXppj+51f//5gP3pfkRT3bL05ybktZ5qzlZ4w+tNPfPC0pp9Sak0qybDdP3ZLUf0rywZH1Q1fPZMZF5kWZ/Nnd4dxM/uzCXOe9AvOR9wrMR94rMF95r7CXrMACFqWnrb7yPmNj46en1NNrsjyp90ud/N2xG79BbqnJP5ZSPzz46MEr16wpzQzHBQAAWNQUWMAiUcvyMzf/fKdvcpXV7eNjx5eSpbt9dvLDJJem5NIyUdeNXjD0/SQZWTdjgQEAANhOgQUsWIMrhx+SdM5K6unJ6JOSHLzbq6xqbk/JpSXlgr6+zqWXfuzEa2c4LgAAALugwAIWjFNP/eyyW/e//WmpOTnJyUmOvcdbr9+pJuVfU5pL0+TSbUv3+/Rn1j7ltplLCwAAwO5SYAHz1po1tbPp3zY/otR6cqnlabeV2wdLzUF78BC3pJaRWurFnab51PDGFV+fsbAAAADsNQUWME/UMnTG8KPS6Xtck/q4UsrxI1eP/lwnWZIkKbu10mo8yWhqubSU7qVHLPn+1WvXPqs7k6kBAAC49xRYwJx0yikXH7TtgP0eW1KfmE55Yq2jT6jpHJPU7fev2r1LA2tyQ6n1klrKxWVJvWRk7dB1M5kbAACAfU+BBbRu8LnD++dHeXxq53El9fianDCW3P+O263X3bjp+p1uLsknk3Kpm68DAAAsDAosYNYNnj38oFI7T2hqfWJJeWJuqo9LcnBSd/+W61N9M7V+sqZcXJbW4eG1Q1v3aWBgrnhkksckeWCSpUl+nOQ/k3wmyU0t5gIAYIYpsIAZdcqqTx+1rRl/YunU49N0Tqil/myaLKt7eCngDjW5tpN8uqZ8IaX5Qvpz9YjCChay/iQvTvJ7SX5yF8dMJPlEkjcl+fws5QIAYBYpsIB9YnBwuL/v8P5jm4w/sjZ9P53SPDKlPH5bHX9kKemklqTUPbkUMEm2peaLNflc6ZTPNZ18bvPHBv5jZv4LgDnoIUk+luSx93Bcf5Izk5ye5G1JXpekmdFkAADMKgUWsEdOfsblx05MTByb0jmupj6qJMcmOS7J/bu1m6Sz/RMB9+zeVTUZK7V8vqZ+oZPyhTTdLxyx//e/5lMCYdF6aJLLM3m54O7qJHlNkqOTPH8mQgEA0A4FFrBTK06/7IG1v/x0aucRNfVRKeURSX56ott9UErJnZ8GuHdqckNJPpfUf+mk87kmzedGNgy6hw2QJPsnWZ+7llejSd6RycsEb0xyTJKzk/xBkvv1HPe8JF9K8hcznhQAgFmhwIJF7PgzrzhkSd/4o0vtPKqW5tjUclxNHlWSBzfJkiTbl1Ddm6oqSfKDJJ9J6hdSc0VKrhpdP6SsAnbld5P83LTZnyU5J1NvnPefSd6e5B+TfDLJz/Tse1OSjyS5buZiAgAwWxRYsMAt3f/I/ZYuXZb9Djwq+x94dI54wPKzDz/iMadOFlUTx6aW/pqa1MmS6l5WVRNJ/quWXJOar6Xma7WWqzs/br40MjI0ce//a4BF4NBMFlW9Lkzyqrs553+SrEzy70kO3D47MJP3wnrpvg4IAMDsU2DBPDa4cviwkvqwpnSOLbU8IKW5f6nl2JpybFIfUJP7lsmPmu/1tOTeFVU1ubYk19aar3ZK+UqtzbVL6pKvXrLxhO/di4cFSJIXJrlPz/ZEkt/ajfO+mckbuL+hZ/aiJH+cyVWgAADMYwosmMNOXL35yCXdPLhpuseUUo6ptR5TUx5cao6pJcckuX9NKaUm2b6KavLamsl/3tsL/2ryvZJyTU39ekm9JjVfa9L5+uYNA9+5lw8NsCvPmLZ9YZJv7+a55yb5o9z5/mZJkjOSfHCfJAMAoDUKLGjBKadcfNDEgf1Hp/bfr3a6R9am7+iUHF1TjynJg1Pz4JT8RMabAyY/B76k1snvO25Jda/vSnWn8dR8M51ck6Z8vSTXlFK+tt+2pV+76KIn/XjfPQ3APTo6yROnzf5pD86/LsmmJKf0zFZGgQUAMO8psGAfOWnVpfcdz5Kj+rs5sim5fy31qE5yZC25f02OKjVHJpP/PrbjHi2lJrUz+T09pdQ+bKeSpNYmY7f/ILffen36lxz4+YOWPXRjreUrndp89cdHL7v2C+c+fnzfPiPAXhnKXV8BL9/Dx7g8UwusFdsfs+78cAAA5gMFFuzC8WdecUhf6R62pJbDa6ce3aQ5qpRyVJocXUs9KilHJTk6JUel5qhuzZJOaprO5Pl3/GmpTv3T2D7upna4Kcl3knyrJt8upX4n6Xwnab511WUvfMktN//Xc2rt7jj2bzN5mQ3AXPOoadvfTvK/e/gYn5m2fUiSB2XyNRIAgHlKgcWC9LgXXbXkPtfdfuREHTu89pfDa1MO76QcXksOT5rDay2Hd0oOr8nhSfZPLQfUUg8vyeGpOTwl+0/eNzjplh1/ab/9Mr4d1/DtMPN/p/+jpPxnSb22qfnfkvK92qnXdmpzbX+z9H8A23l6AAAR8ElEQVQP2+/w769de9zY3Zz/qzOeEGDfeOS07W/uxWPs7JxHRoEFADCvKbCYcwafO7x/95a+Q5aMjy+rKYeV0ndoTfeQpnaWlVKWJc0hSQ5LkjSdQ5N62PbtO7+u33rYeCf7J53JFVAlqbmziJrc7lHqjF2+dw9uSs11KdmSkutrk++V1O+U0vl2U+p3lnb7v3X8Y4+/bs2a0sxqKoB2PGLa9u7evL3Xd5M0STrTHvdTexsKAID2KbDYJ9asqZ1LP3/Fof2dib7+pf3LmloPa5rustJ0ltVOs6ymHFJqWZZOc3ipZVlNDqkly0rNsqQcUlMPL8mymizLTVnalyZN+pIktdYkncle6c4lUJPK3LqlSU2akmxJsiUp16c216V0tqQ016fmutKpW+pEuT79ue6A227bctFFT992T495ycZZCA4wN9xn2vb39uIxJjL5Ony/u3lcAADmGQXWAnHqqZ/Y7+bOIQcmSae/OXhpf2dJ0x0rNeWwJGlKZ2lp6kElnaVNpx5UUpaWmoNqyZI0OTilWZKSg2stS5IcnKS/M1ky9afmkCT9STkkqX01WVaSvkwWTn0lWTZy9Wj6lyRJyUR3x72WSmpnsnCavOquJrXcuQ7qju7pztVPs7v4abdtS8n3a831Sa4rqVtqymQhVeqWWsr1tSnXdZY2W47Kli1r1z6re4+PCMDOHDxt+9a9fJzp501/XAAA5hkF1m568uorD/j8+qclSfqXHJRSOvmFX/zIT/Z1u8uSpJZyQJrsnySl01nWpPaV1L5SO8uSJKXuX5tyQJLUTl1WUvtKUzpNyaGT+7N/p2Zyf80hKemvSaekTO5P3a9u/+S6kh2FUkq2X0p3W5L+3NmbTJZIfXdsl5qklNTU7cVRveMG43fc02nazcbv2D91Mhs3JN/Xupm8yflNSbkpqTemTG6XJjdO3ZebSnJTSbmpUzs3bd2vc+Nn1j7ltjbDAywiB03bvn0vH2f667YCCwBgnpsn/UP7BlaNvLnUvLbtHIvRxPjWdCduTXfitkxM3JruxK13zCbGb8nE+M2ZGN+aibGtk997vsbHb053Ym//Ah8AAABmlF5mN1mBtZtKLWOz8XFzC0HTHcvE+M0ZH785E2Nb0zTb7pyN3ZyJ8ZvTdMfSNGOZGNt+3PjNabo7jpssnpruPd4eCgAAAFgEFFi7qzTbUudfMToxcUtSmzTNRLoTt6XWbs/3W1Nrk+74Lam1uePYye81E+O3pGb6/pqJ8a1JdnzPHd/Hx27evn1zW/+5AAAAMB/Mv4KhZQqs3VRSxqavv6q1SXfiliSTq4663W1pmrE03bE7ZjtWH+1YTdQ024/bvu+O47avSLrjuO5Yujs5t3f/jufq7mQ/AAAAwEKhwNpNff3jH0iz//okOeHRV/1xKal9pdtdsqQ7cc9nd5LJ+7P3fAcAer3lLW9Z/aMf/ejwHdsPf/jDv/HCF75wZE8eY2JiovO6173uBbXeuWz6xBNPvPL000//cu9x55xzzovudeC78ba3ve3cmXx8AGB+O+ecc9qOMO9YsgYAzBUfS3J2z/amJCft4WP8RJL/njb7xSQXT5u9bw8fd0+9eIYfHwBgUbECCwCYK66Ztv3wvXiMh+3G4yYKJgCAeaXTdgAAgO2+Om37mCT338PHeMq07a1JvrPXiQAAmBMUWADAXDGcZPpnppy4h48x/fidPSYAAAAA7LXPZrJw2vG1bg/OPTrJ+LTzn7evAwIAAACwuJ2TqQXUeJIH7+a5r5927liSI2YgIwAAAACL2GFJbszUIurju3Hew5LcMu2898xQRgAAAAAWuddkahFVk7w9SdnF8Q9M8qVpx9+a5AEznhQAAACARemA3LWQqpm8IftpSY5KsjTJw5O8Msl1Ozn2FbOeGgAAAIBF5WFJvpe7FlO78/X32fVqLQAAAADYZ45NcnV2v7hqkrwtSV8bYQEAAABYnPqTvDTJf2bXxdVEkguSPKGljAAAzDDL6wGA+eJRSR6byZuzL01ycyaLrSsz+cmFAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAsGdK2wEWuYcmeXSSn0hycJKxJDcm+Y8k/7793wHYcyXJQ5L8XJIHJDk8ybZMvq5+O8nnk/yorXAsWI9M8pgkD0yyNMmPk/xnks8kuanFXDCd10iAdukCmBcOTfKaJF9PUu/mq0ny5STvSPLYVpLC3Xttdv6z+9I2Q7GoHZHkBUnOS/LD3P1rbDfJ55I8P8n+bYRlwehP8ttJvpFd/7yNJ9mQ5BdaygiJ10gWPu9Nmet0Acwrv5bk+7n7H9adfX20jbBwN34yyW3xJoG54ReSbMzk31zt6etrzeQKmeNnPTULwUOSfCG7/7PWTfKWJJ0WsrJ4eY1kMfDelLlOF8C80ZfkPdm7Nw1+aJmLhrPrn1dvEphtr8vev772Fgu/PNvBmdcemuS72buft79rIS+Ll9dIFgPvTZmrdAH7UH/bARaJczO5BHu6LyVZn8nlgddtn90nyc8meVKSk5LsNxsBYQ+8IMlg2yHgHvwwyeXbv/47yfWZ/J13TJIVSX4pUy+L6ST50PbjNs1mUOal/TP5+/uB0+ajmVzu//lM3rvimCRnJ/mDJPfrOe55mXwP8BcznhR2zmskC4n3psxlugDmlZfkri3qfyc5YzfOPTjJbyRZM0PZYE/dL1Pvm/Gx+Fsu2rdjdcFEknVJzsw9/wXNA5Jclrv+/H4tyZIZS8pC8arc9Wfn7dn1h+M8MJNvVHuPvyXJ0TOeFLxGsrB5b8pcpgtgXnlwJj+BqPcH9pokR7UZCu6F83Lnz/KPM/mHMm8SaNv/3969hlpWlnEA/ys6ZDlNlJmajZdRSbOEMlNRSIMiIkhKoz6UCBUVppVSQtm36GL4oSEDKRSSMMwJFVPMGjSVvJXaRWs0yLyQmrfRdEbHPrxzmL3W3uew9z57nbXWOb8fbJi1Zr3veQYe3vPMs9d61zlJLklyyITjVqXcMVPP4ZNnGh3LzZokT6SaM1eNMW5dStNqcNz6hmKEQdZIljO1KV2lF0DvbEg1YZ9JSWToow+nms9nbD+vSKBti3kc/qCUN8QN5vDPZhEUy9ZZqebL1pTHrsZxbm3sliRvaCBGGGSNZLlSm9JlegH0yrqU118OJu1XWo0Iprc6yYPZkcu3p2xImCgS6L+Nqebw3a1GQ9fdkmq+/GqCsXtluBlw6ozjg1nbGGsk3aM2pcv0Auid76SasE+muhkm9Mn67Mjll5O8a+DvFAn03WB+v5KySTGMsleGC9JTJpzj2tr4SRpg0AZrJF2kNqXL9AIasnPbASxjn64dX5rkhTYCgUU6JmUDwjnrk9zRUizQhJdqx1tbiYI+OCHDG7XfOOEc9etPHDEndIk1kq5Rm9J1egEN0cBqxsEZfrPQFW0EAou0a5ILs2OteCjlbUawnBxYO36klSjog8Nqx//K5PlyS+14dZJ9p44ImmeNpEvUpnSdXkCDFrOpI/N794hzf6gdr01ydErRunOSx1IW4FtS3lIEXfD1JG8bOD4jybMtxQJN2D3lDphBN7cRCL1waO34/inmGDXm0JS9XKBrrJF0jdqUrtMLoHfOT/WZ1wcG/u6EJL/P8LPZc58Xk1y3/Tpo01tTbnWdy80r57nOPgP02RcynMPWX+ZzT6q5ctEUc+ySsl/L4Dynzyg+mDVrJF2iNqUP9ALonctSTcSNKQXr+Rne/HWhz+UpjxbAUtspyQ3ZkYvPJdlvnmsVCfTVHimbEQ/m712xHxHzeyjVfPn2lPM8WpvnWzOJDmbLGkmXqE3pC70Aeuc3qSbfz5P8NOMn6+DnniSvX9rwIZ9LNQ/PXuBaRQJ9tSHD+Vt/VAYGPZ1qvky778oDtXm+P5PoYLaskXSJ2pS+0Augd25PNfGeqB1vSnlzxkEpr9Nck+TIlG9yN2c4cX8dG+6zdPZO8lR25N/dWXi/PEUCfXROhnP3x61GRB+8lGrOnDXlPH+pzXPBTKKD2bFG0iVqU/pEL4DeuTfzd1EvSbJqgbEHJPn7iHGnNhcuVFyeHXm3LWWDwYUoEuibj2X4Fu67kuzWZlB03q4ZXu++NOVcd9bmuWgG8cGsWCPpGrUpfaIXwIKOTdnMb6k+j44R058yOmFvzHjd0wNT3qYxOPbeMcfSD13M2yQ5KZPfFaBIWFm6mrvjel+qG8C+krKv0f4z/jksT03dgeXOFrrCGknXqE3pG70AFnRc5u9wNvF5aoyY5nuzwBET/Lu+OWL8UROMp9u6mLevTXWD4keTvG6McYqElaWLuTuuYzN8a/bjqb6OGxbS1B5Y580kOlgcayRdozalj/QCGqSL14zNI87dkXL79bh+MuLce6eKBsbz3ST7DBx/ObNtHkCbjkxydZLXDJx7Osn7U+6GgXHUf7+/esp56o9iPTvlPDAr1ki6SG1KH+kFNGihze/64ukkNy3hzxunyPzviHM3TvhzHk75hvbAgXOTdG3ptq7l7XEpb3eZc13KGzOgrmu5O44jklybsknmnM1JPpiyFxGM68lU/zO19xRz7JLkjSPmhbZYI+kitSl9pRdA75yb4Vv+zpxinutrc1wzqwCh5nfZkWf/S3krxrjcpk2XHZ7ksVRz9Pn4Fovp/DLVXLp+ijn2y/C6+YFZBQgTskbSVWpT+kovoEEeIWzGX0ecm+YugvqYNSOvgsVbPfDnVyX5R8bfo6juhyOu2bepwGEBh6b88t9j4NyLKRvCbmwjIHrvb7XjSf5DNWfdGPPCUrBG0mVqU/pKL6BBGljN+POIc7tPMc/q2rFnvgHGc0iS3ybZc+Dc1iQnpzwqA9OoF6VrM/ljhMfWjjcneXDqiGA61kiAZugFNEgDqxn3Jfl37dx+U8xTH/P4dOEArCgHpTx6sNfAuZeTfDLJla1ExHIx90jLoOMnnKN+/ag5oUnWSIDm6AU0aDls4t5FryTZkOT0gXOTFrh7Z/gxgz8uJihYwLVJNk059uO14ztTbvMe9PyUc8OkDki5q2Bwo+1tST6V5LJWImI5eSTJrUneM3DuE0l+Meb4vZKcWDu3YQZxwbiskfSF2pS+0gugl47P8LPWb59g/DdGjH/HjGOEWbBRJl3xliT/TDUftyU5tcWYWH6+lmqObU3JvXHUN3bdkur+Q9AkayQrhdqUtukF0Es3pJp0GzPeY5v7J3mmNva2RiKExVMk0AX7ZHiD121JPtNmUCxLr0vyZKq5dtUY49Ylea427oKGYoQ6ayQridqULtALoHeOzvACenGSXRcYszblbUT1cR9qNFKYniKBtu2Z0evmF9sMimXtnAzn2/eS7DTP9W9Ock/t+udTfYwLmmKNZKVRm9IFegENmK/QYnZ+lOTztXP3JTkvyXVJHk5J4oOTfDTJmRl+48CFST7bbJgwtfrmw6cnWd9GIKxYlyY5pXbuqSQ3LWLO05L8ZxHjWd52S9kL6/Da+Y0pv99vS8nBtUlOSvLVJG+qXXv29muhadZIVhq1KV2hF0DvrEpydYa7qON+rtk+B3SVb7lo21WZfo2d77P/Uv4D6KV1KYXnNPl1cXyJyNKxRrLSqE3pCr2AGRvnGUwWZ0uSj6R0XyexLckPUm4X3DLroACARbk/yXFJ7ppgzNyjhqdl+A4BAGB50Qug145JckXKG4vm67I+m+SSJIe1FCNMyrdctM3dBbRpl5R1b1Pmz6eXklyZ5KiWYmRls0ay0qhN6SK9gBlw+3o71qRs6nbw9j+/kOTxlOL31pSkBgD65bAk70zZnH1VSiG6KcnNKW8uBABWNr0AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAACADvo/fJbpgLcKdzYAAAAASUVORK5CYII=\n", "text/plain": [ "" ] }, "execution_count": 62, "metadata": { "image/png": { "height": "500", "width": "500" } }, "output_type": "execute_result" } ], "source": [ "Image('images/logistic_function.png', height=\"500\", width=\"500\")" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "$$\\sigma(z) = \\frac{1}{1+e^{-z}} $$\n", "\n", "Where the $z$ in previous function has the value of $W^Tx+b$ and the output $c$ of this function is the probability of being in that class that is $P(Y=c \\mid X=x)$, given $x$ parametrized by $W$ and $b$.\n", "\n", "**Phase 1:**: build the graph.\n", "\n", "*Step 1:* define two placeholder variables, one for the input $x$ the other for the (true) output $y$." ] }, { "cell_type": "code", "execution_count": 63, "metadata": { "collapsed": true }, "outputs": [], "source": [ "x = tf.placeholder(tf.float32, shape=[None, IMAGE_PIXELS])\n", "y = tf.placeholder(tf.int32, shape=[None, NUM_CLASSES])" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "*Step 2:* weights and biases\n", "\n", "The first variable that must be optimized is called `weights` and is defined here as a TensorFlow variable that must be initialized with zeros and whose shape is `[IMAGE_PIXELS, NUM_CLASSES]`, so it is a 2-dimensional tensor (or matrix) with `IMAGE_PIXELS` rows and `NUM_CLASSES` columns.\n", "\n", "The second variable that must be optimized is called `biases` and is defined as a 1-dimensional tensor (or vector) of length `NUM_CLASSES`." ] }, { "cell_type": "code", "execution_count": 64, "metadata": { "collapsed": true }, "outputs": [], "source": [ "weights = tf.Variable(tf.zeros([IMAGE_PIXELS, NUM_CLASSES]))\n", "biases = tf.Variable(tf.zeros([NUM_CLASSES]))" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "*Step 3: * build the model\n", "\n", "This simple mathematical model multiplies the images in the placeholder variable `x` with the `weights` and then adds the `biases`.\n", "\n", "The result is a matrix of shape `[num_images, NUM_CLASSES]` because `x` has shape `[num_images, IMAGE_PIXELS]` and `weights` has shape `[IMAGE_PIXELS, NUM_CLASSES]`, so the multiplication of those two matrices is a matrix with shape `[num_images, NUM_CLASSES]` and then the `biases` vector is added to each row of that matrix." ] }, { "cell_type": "code", "execution_count": 65, "metadata": { "collapsed": true }, "outputs": [], "source": [ "logits = tf.matmul(x, weights) + biases" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "Now `logits` is a matrix with `num_images` rows and `num_classes` columns, where the element of the $i$'th row and $j$'th column is an estimate of how likely the $i$'th input image is to be of the $j$'th class.\n", "\n", "However, these estimates are a bit rough and difficult to interpret because the numbers may be very small or large, so we want to normalize them so that each row of the `logits` matrix sums to one, and each element is limited between zero and one. This is calculated using the so-called softmax function and the result is stored in `y_pred`." ] }, { "cell_type": "code", "execution_count": 66, "metadata": { "collapsed": true }, "outputs": [], "source": [ "y_pred = tf.nn.softmax(logits)" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "The predicted class can be calculated from the `y_pred` matrix by taking the index of the largest element in each row." ] }, { "cell_type": "code", "execution_count": 67, "metadata": { "collapsed": true }, "outputs": [], "source": [ "y_pred_cls = tf.argmax(y_pred, axis=1)" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "*Step 3:* define loss function\n", "\n", "To make the model better at classifying the input images, we must somehow change the variables for `weights` and `biases`. To do this we first need to know how well the model currently performs by comparing the predicted output of the model `y_pred` to the desired output `y`.\n", "\n", "The softmax is a generalization of the logistic function, it is used to calculate the probability associated to each class and it is usefull in multiclass classification problems. A softmax regression has two steps: first we add up the evidence of our input being in certain classes, and then we convert that evidence into probabilities. Here a class is one of the 10 possibile digits. \n", "\n", "Next we define the cost function, cross entropy in this case. Cross-entropy gives us a way to express how different two probability distributions are. The more different the distributions p and q are, the more the cross-entropy of p with respect to q will be bigger than the entropy of p. Similarly, the more different p is from q, the more the cross-entropy of q with respect to p will be bigger than the entropy of q. If the distributions are the same, this difference will be zero. As the difference grows, it will get bigger.\n", "\n", "The goal of optimization is therefore to minimize the cross-entropy so it gets as close to zero as possible by changing the `weights` and `biases` of the model.\n", "\n", "TensorFlow has a built-in function for calculating the cross-entropy. Note that it uses the values of the `logits` because it also calculates the softmax internally." ] }, { "cell_type": "code", "execution_count": 68, "metadata": { "collapsed": true }, "outputs": [], "source": [ "cross_entropy = tf.nn.softmax_cross_entropy_with_logits_v2(logits=logits,\n", " labels=y)" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "We have now calculated the cross-entropy for each of the image classifications so we have a measure of how well the model performs on each image individually. But in order to use the cross-entropy to guide the optimization of the model's variables we need a single scalar value, so we simply take the average of the cross-entropy for all the image classifications." ] }, { "cell_type": "code", "execution_count": 69, "metadata": { "collapsed": true }, "outputs": [], "source": [ "cost = tf.reduce_mean(cross_entropy)" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "*Step 4:* \n", "\n", "Now that we have a cost measure that must be minimized, we can then create an optimizer. In this case it is the basic form of Gradient Descent where the learning rate is set at $0.5$." ] }, { "cell_type": "code", "execution_count": 70, "metadata": { "collapsed": true }, "outputs": [], "source": [ "optimizer = tf.train.GradientDescentOptimizer(learning_rate=0.5).minimize(cost)" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "We need a few more performance measures to display the progress to the user.\n", "\n", "This is a vector of booleans whether the predicted class equals the true class of each image. \n", "\n", "This calculates the classification accuracy by first type-casting the vector of booleans to floats, so that False becomes 0 and True becomes 1, and then calculating the average of these numbers." ] }, { "cell_type": "code", "execution_count": 71, "metadata": { "collapsed": true }, "outputs": [], "source": [ "correct_prediction = tf.equal(tf.argmax(tf.nn.softmax(logits), 1), \n", " tf.argmax(y, 1))\n", "accuracy = tf.reduce_mean(tf.cast(correct_prediction, tf.float32))" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "**Phase 2:** train the model\n", "\n", "*Step 1:* initialize variables\n", "\n", "There are 50.000 images in the training-set. It takes a long time to calculate the gradient of the model using all these images. We therefore use Stochastic Gradient Descent which only uses a small batch of images in each iteration of the optimizer." ] }, { "cell_type": "code", "execution_count": 72, "metadata": { "collapsed": true }, "outputs": [], "source": [ "NUM_ITERATIONS = 1000\n", "BATCH_SIZE = 100\n", "\n", "session = tf.Session()\n", "session.run(tf.global_variables_initializer())" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "*Step 2:* run optimizer\n", "\n", "Function for performing a number of optimization iterations so as to gradually improve the `weights` and `biases` of the model. In each iteration, a new batch of data is selected from the training-set and then TensorFlow executes the optimizer using those training samples." ] }, { "cell_type": "code", "execution_count": 73, "metadata": { "collapsed": true }, "outputs": [], "source": [ "for i in range(NUM_ITERATIONS):\n", " x_batch, y_batch = mnist.train.next_batch(BATCH_SIZE)\n", "\n", " feed_dict_train = {x: x_batch,\n", " y: y_batch}\n", "\n", " session.run(optimizer, feed_dict=feed_dict_train)" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "*Step 3:* verify model accuracy\n", "\n", "Dict test contains test data that the model has not seen during the training" ] }, { "cell_type": "code", "execution_count": 74, "metadata": { "collapsed": true }, "outputs": [], "source": [ "feed_dict_test = {x: mnist.test.images,\n", " y: mnist.test.labels}" ] }, { "cell_type": "code", "execution_count": 75, "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "Accuracy on test set: 92.0%\n" ] } ], "source": [ "acc = session.run(accuracy, feed_dict=feed_dict_test)\n", " \n", "print(\"Accuracy on test set: {0:.1%}\".format(acc))" ] }, { "cell_type": "code", "execution_count": 76, "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "[[ 953 0 1 2 0 10 7 3 4 0]\n", " [ 0 1105 2 2 1 2 4 2 17 0]\n", " [ 9 7 891 18 11 9 14 12 51 10]\n", " [ 3 1 13 908 0 39 3 8 23 12]\n", " [ 1 1 2 1 900 0 12 2 10 53]\n", " [ 8 3 3 20 7 799 12 4 29 7]\n", " [ 9 3 3 2 13 22 899 3 4 0]\n", " [ 2 9 19 8 6 1 0 932 3 48]\n", " [ 4 5 4 14 9 29 9 4 885 11]\n", " [ 10 6 0 6 26 11 1 13 9 927]]\n" ] } ], "source": [ "cls_true = np.argmax(mnist.test.labels, axis=1)\n", "\n", "cls_pred = session.run(y_pred_cls, feed_dict=feed_dict_test)\n", "\n", "cm = confusion_matrix(y_true=cls_true,\n", " y_pred=cls_pred)\n", "\n", "print(cm)" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "## 4 Eager execution" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "When using TensorFlow, the first step is to write a python program that assembles the graph, second the graph is run by the C++ runtime. In a sense TensorFlow is a declarative program, because the specification of the computaion is seprated by the execution. \n", "\n", "From another perspective TensorFlow is both a programming language and a compiler of machine learning models: its input is a description of a machine learning model (the graph) and rewrites it into an optimized and executable form.\n", "\n", "The pros of this approach are:\n", " 1. optimization reduces execution time\n", " 1. the graph is an intermediate representation that simplify deployment\n", " 1. the graph can be rewritten with changes to device placement or quantized weights\n", "The cons are:\n", " 1. difficoult to debug, because errors are reported after graph construction and because `print` and `pdb` are not effective\n", " 1. a TensorFlow program is written by metaprogramming and thus can be difficult\n", " 1. it is not possible to use Python native control flow statements and mix with custom data structure\n", " \n", "For this reason since TensorFlow 1.5 **Eager** execution is available. Eager is:\n", "\n", "> A NumPy-like library for numerical computation with support for GPU acceleration and automatic differentiation. A flexible platform for machine learning research and experimentation (see [user guide](https://github.com/tensorflow/tensorflow/blob/master/tensorflow/contrib/eager/python/g3doc/guide.md))." ] }, { "cell_type": "code", "execution_count": 4, "metadata": { "collapsed": true }, "outputs": [], "source": [ "import tensorflow.contrib.eager as tfe\n", "tfe.enable_eager_execution()" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "The pros of eager are:\n", " - is compatible with pdb\n", " - errors are reported immediately\n", " - allows use of python control flow and structures" ] }, { "cell_type": "code", "execution_count": 5, "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "New value is: 1\n", "New value is: 2\n", "New value is: 3\n", "New value is: 4\n", "New value is: 5\n", "New value is: 6\n", "New value is: 7\n", "New value is: 8\n", "New value is: 9\n", "New value is: 10\n" ] } ], "source": [ "i = tf.constant(0)\n", "while i < 10:\n", " i = tf.add(i, 1)\n", " print('New value is: {:d}'.format(i))" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "For this reasons there is no longer need for placeholders, sessions, control dependencies, \"lazy loading\" and scopes. \n", "\n", "For example in this case there is no need to declare a placeholder for x and no need to create a session." ] }, { "cell_type": "code", "execution_count": 6, "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "tf.Tensor([[9.]], shape=(1, 1), dtype=float32)\n" ] } ], "source": [ "x = [[3.]]\n", "\n", "m = tf.matmul(x, x)\n", "print(m)" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "The most interesting thing is to note that tensors act like numpy arrays, for example:" ] }, { "cell_type": "code", "execution_count": 7, "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "True\n" ] } ], "source": [ "x = tf.constant([1.1, 2.2, 3.3])\n", "print(type(x.numpy()) == np.ndarray)" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "and are compatible with numpy functions:" ] }, { "cell_type": "code", "execution_count": 8, "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "[ 1.21 4.84 10.889999]\n" ] } ], "source": [ "print(np.square(x))" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "tensors are also iterable:" ] }, { "cell_type": "code", "execution_count": 15, "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "1.1\n", "2.2\n", "3.3\n" ] } ], "source": [ "for i in x:\n", " print('{:.1f}'.format(i))" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "Automated differentiation is built into eager execution, in practice operation are traced on a *tape* and played back to compute gradients. For example:" ] }, { "cell_type": "code", "execution_count": 19, "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "tf.Tensor(9.0, shape=(), dtype=float32)\n", "[]\n" ] } ], "source": [ "def square(x):\n", " return tf.multiply(x, x)\n", "\n", "grad = tfe.gradients_function(square)\n", "\n", "print(square(3.))\n", "print(grad(3.))" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "The API for automated differentiation works even when eager execution is not enabled. \n", "\n", "Another way to think about TensorFlow is a collection of operations and a mean to execute computations composed of such operations. Session provide is one method, with eager enabled Python is the method of execution.\n", "\n", "Use eager if you want flexibility when developing a new model or if you want to explore TensorFlow API. At the moment eager is under active development, not all TensorFlow API are supported. With eager execution enabled, computation do not run automatically on GPUs. The user must explicitly specify if the operation must run on GPUs with `tf.device`." ] }, { "cell_type": "markdown", "metadata": { "collapsed": true }, "source": [ "---\n", "\n", "Visit [www.add-for.com]() for more tutorials and updates.\n", "\n", "This work is licensed under a Creative Commons Attribution-ShareAlike 4.0 International License." ] } ], "metadata": { "kernelspec": { "display_name": "Python [conda env:addfor_tutorials]", "language": "python", "name": "conda-env-addfor_tutorials-py" }, "language_info": { "codemirror_mode": { "name": "ipython", "version": 3 }, "file_extension": ".py", "mimetype": "text/x-python", "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", "version": "3.6.4" } }, "nbformat": 4, "nbformat_minor": 1 }