{ "cells": [ { "cell_type": "markdown", "metadata": {}, "source": [ "# Merging and concatenating data frames\n", "\n", "[**Data set download**](https://s3.amazonaws.com/bebi103.caltech.edu/data/frog_strikes.zip)\n", "\n", "
" ] }, { "cell_type": "code", "execution_count": 1, "metadata": {}, "outputs": [ { "data": { "text/html": [ "\n", "
\n", " \n", " Loading BokehJS ...\n", "
" ] }, "metadata": {}, "output_type": "display_data" }, { "data": { "application/javascript": [ "\n", "(function(root) {\n", " function now() {\n", " return new Date();\n", " }\n", "\n", " var force = true;\n", "\n", " if (typeof root._bokeh_onload_callbacks === \"undefined\" || force === true) {\n", " root._bokeh_onload_callbacks = [];\n", " root._bokeh_is_loading = undefined;\n", " }\n", "\n", " var JS_MIME_TYPE = 'application/javascript';\n", " var HTML_MIME_TYPE = 'text/html';\n", " var EXEC_MIME_TYPE = 'application/vnd.bokehjs_exec.v0+json';\n", " var CLASS_NAME = 'output_bokeh rendered_html';\n", "\n", " /**\n", " * Render data to the DOM node\n", " */\n", " function render(props, node) {\n", " var script = document.createElement(\"script\");\n", " node.appendChild(script);\n", " }\n", "\n", " /**\n", " * Handle when an output is cleared or removed\n", " */\n", " function handleClearOutput(event, handle) {\n", " var cell = handle.cell;\n", "\n", " var id = cell.output_area._bokeh_element_id;\n", " var server_id = cell.output_area._bokeh_server_id;\n", " // Clean up Bokeh references\n", " if (id != null && id in Bokeh.index) {\n", " Bokeh.index[id].model.document.clear();\n", " delete Bokeh.index[id];\n", " }\n", "\n", " if (server_id !== undefined) {\n", " // Clean up Bokeh references\n", " var cmd = \"from bokeh.io.state import curstate; print(curstate().uuid_to_server['\" + server_id + \"'].get_sessions()[0].document.roots[0]._id)\";\n", " cell.notebook.kernel.execute(cmd, {\n", " iopub: {\n", " output: function(msg) {\n", " var id = msg.content.text.trim();\n", " if (id in Bokeh.index) {\n", " Bokeh.index[id].model.document.clear();\n", " delete Bokeh.index[id];\n", " }\n", " }\n", " }\n", " });\n", " // Destroy server and session\n", " var cmd = \"import bokeh.io.notebook as ion; ion.destroy_server('\" + server_id + \"')\";\n", " cell.notebook.kernel.execute(cmd);\n", " }\n", " }\n", "\n", " /**\n", " * Handle when a new output is added\n", " */\n", " function handleAddOutput(event, handle) {\n", " var output_area = handle.output_area;\n", " var output = handle.output;\n", "\n", " // limit handleAddOutput to display_data with EXEC_MIME_TYPE content only\n", " if ((output.output_type != \"display_data\") || (!output.data.hasOwnProperty(EXEC_MIME_TYPE))) {\n", " return\n", " }\n", "\n", " var toinsert = output_area.element.find(\".\" + CLASS_NAME.split(' ')[0]);\n", "\n", " if (output.metadata[EXEC_MIME_TYPE][\"id\"] !== undefined) {\n", " toinsert[toinsert.length - 1].firstChild.textContent = output.data[JS_MIME_TYPE];\n", " // store reference to embed id on output_area\n", " output_area._bokeh_element_id = output.metadata[EXEC_MIME_TYPE][\"id\"];\n", " }\n", " if (output.metadata[EXEC_MIME_TYPE][\"server_id\"] !== undefined) {\n", " var bk_div = document.createElement(\"div\");\n", " bk_div.innerHTML = output.data[HTML_MIME_TYPE];\n", " var script_attrs = bk_div.children[0].attributes;\n", " for (var i = 0; i < script_attrs.length; i++) {\n", " toinsert[toinsert.length - 1].firstChild.setAttribute(script_attrs[i].name, script_attrs[i].value);\n", " toinsert[toinsert.length - 1].firstChild.textContent = bk_div.children[0].textContent\n", " }\n", " // store reference to server id on output_area\n", " output_area._bokeh_server_id = output.metadata[EXEC_MIME_TYPE][\"server_id\"];\n", " }\n", " }\n", "\n", " function register_renderer(events, OutputArea) {\n", "\n", " function append_mime(data, metadata, element) {\n", " // create a DOM node to render to\n", " var toinsert = this.create_output_subarea(\n", " metadata,\n", " CLASS_NAME,\n", " EXEC_MIME_TYPE\n", " );\n", " this.keyboard_manager.register_events(toinsert);\n", " // Render to node\n", " var props = {data: data, metadata: metadata[EXEC_MIME_TYPE]};\n", " render(props, toinsert[toinsert.length - 1]);\n", " element.append(toinsert);\n", " return toinsert\n", " }\n", "\n", " /* Handle when an output is cleared or removed */\n", " events.on('clear_output.CodeCell', handleClearOutput);\n", " events.on('delete.Cell', handleClearOutput);\n", "\n", " /* Handle when a new output is added */\n", " events.on('output_added.OutputArea', handleAddOutput);\n", "\n", " /**\n", " * Register the mime type and append_mime function with output_area\n", " */\n", " OutputArea.prototype.register_mime_type(EXEC_MIME_TYPE, append_mime, {\n", " /* Is output safe? */\n", " safe: true,\n", " /* Index of renderer in `output_area.display_order` */\n", " index: 0\n", " });\n", " }\n", "\n", " // register the mime type if in Jupyter Notebook environment and previously unregistered\n", " if (root.Jupyter !== undefined) {\n", " var events = require('base/js/events');\n", " var OutputArea = require('notebook/js/outputarea').OutputArea;\n", "\n", " if (OutputArea.prototype.mime_types().indexOf(EXEC_MIME_TYPE) == -1) {\n", " register_renderer(events, OutputArea);\n", " }\n", " }\n", "\n", " \n", " if (typeof (root._bokeh_timeout) === \"undefined\" || force === true) {\n", " root._bokeh_timeout = Date.now() + 5000;\n", " root._bokeh_failed_load = false;\n", " }\n", "\n", " var NB_LOAD_WARNING = {'data': {'text/html':\n", " \"
\\n\"+\n", " \"

\\n\"+\n", " \"BokehJS does not appear to have successfully loaded. If loading BokehJS from CDN, this \\n\"+\n", " \"may be due to a slow or bad network connection. Possible fixes:\\n\"+\n", " \"

\\n\"+\n", " \"\\n\"+\n", " \"\\n\"+\n", " \"from bokeh.resources import INLINE\\n\"+\n", " \"output_notebook(resources=INLINE)\\n\"+\n", " \"\\n\"+\n", " \"
\"}};\n", "\n", " function display_loaded() {\n", " var el = document.getElementById(\"1001\");\n", " if (el != null) {\n", " el.textContent = \"BokehJS is loading...\";\n", " }\n", " if (root.Bokeh !== undefined) {\n", " if (el != null) {\n", " el.textContent = \"BokehJS \" + root.Bokeh.version + \" successfully loaded.\";\n", " }\n", " } else if (Date.now() < root._bokeh_timeout) {\n", " setTimeout(display_loaded, 100)\n", " }\n", " }\n", "\n", "\n", " function run_callbacks() {\n", " try {\n", " root._bokeh_onload_callbacks.forEach(function(callback) {\n", " if (callback != null)\n", " callback();\n", " });\n", " } finally {\n", " delete root._bokeh_onload_callbacks\n", " }\n", " console.debug(\"Bokeh: all callbacks have finished\");\n", " }\n", "\n", " function load_libs(css_urls, js_urls, callback) {\n", " if (css_urls == null) css_urls = [];\n", " if (js_urls == null) js_urls = [];\n", "\n", " root._bokeh_onload_callbacks.push(callback);\n", " if (root._bokeh_is_loading > 0) {\n", " console.debug(\"Bokeh: BokehJS is being loaded, scheduling callback at\", now());\n", " return null;\n", " }\n", " if (js_urls == null || js_urls.length === 0) {\n", " run_callbacks();\n", " return null;\n", " }\n", " console.debug(\"Bokeh: BokehJS not loaded, scheduling load and callback at\", now());\n", " root._bokeh_is_loading = css_urls.length + js_urls.length;\n", "\n", " function on_load() {\n", " root._bokeh_is_loading--;\n", " if (root._bokeh_is_loading === 0) {\n", " console.debug(\"Bokeh: all BokehJS libraries/stylesheets loaded\");\n", " run_callbacks()\n", " }\n", " }\n", "\n", " function on_error() {\n", " console.error(\"failed to load \" + url);\n", " }\n", "\n", " for (var i = 0; i < css_urls.length; i++) {\n", " var url = css_urls[i];\n", " const element = document.createElement(\"link\");\n", " element.onload = on_load;\n", " element.onerror = on_error;\n", " element.rel = \"stylesheet\";\n", " element.type = \"text/css\";\n", " element.href = url;\n", " console.debug(\"Bokeh: injecting link tag for BokehJS stylesheet: \", url);\n", " document.body.appendChild(element);\n", " }\n", "\n", " const hashes = {\"https://cdn.bokeh.org/bokeh/release/bokeh-2.2.1.min.js\": \"qkRvDQVAIfzsJo40iRBbxt6sttt0hv4lh74DG7OK4MCHv4C5oohXYoHUM5W11uqS\", \"https://cdn.bokeh.org/bokeh/release/bokeh-widgets-2.2.1.min.js\": \"Sb7Mr06a9TNlet/GEBeKaf5xH3eb6AlCzwjtU82wNPyDrnfoiVl26qnvlKjmcAd+\", \"https://cdn.bokeh.org/bokeh/release/bokeh-tables-2.2.1.min.js\": \"HaJ15vgfmcfRtB4c4YBOI4f1MUujukqInOWVqZJZZGK7Q+ivud0OKGSTn/Vm2iso\"};\n", "\n", " for (var i = 0; i < js_urls.length; i++) {\n", " var url = js_urls[i];\n", " var element = document.createElement('script');\n", " element.onload = on_load;\n", " element.onerror = on_error;\n", " element.async = false;\n", " element.src = url;\n", " if (url in hashes) {\n", " element.crossOrigin = \"anonymous\";\n", " element.integrity = \"sha384-\" + hashes[url];\n", " }\n", " console.debug(\"Bokeh: injecting script tag for BokehJS library: \", url);\n", " document.head.appendChild(element);\n", " }\n", " };\n", "\n", " function inject_raw_css(css) {\n", " const element = document.createElement(\"style\");\n", " element.appendChild(document.createTextNode(css));\n", " document.body.appendChild(element);\n", " }\n", "\n", " \n", " var js_urls = [\"https://cdn.bokeh.org/bokeh/release/bokeh-2.2.1.min.js\", \"https://cdn.bokeh.org/bokeh/release/bokeh-widgets-2.2.1.min.js\", \"https://cdn.bokeh.org/bokeh/release/bokeh-tables-2.2.1.min.js\"];\n", " var css_urls = [];\n", " \n", "\n", " var inline_js = [\n", " function(Bokeh) {\n", " Bokeh.set_log_level(\"info\");\n", " },\n", " function(Bokeh) {\n", " \n", " \n", " }\n", " ];\n", "\n", " function run_inline_js() {\n", " \n", " if (root.Bokeh !== undefined || force === true) {\n", " \n", " for (var i = 0; i < inline_js.length; i++) {\n", " inline_js[i].call(root, root.Bokeh);\n", " }\n", " if (force === true) {\n", " display_loaded();\n", " }} else if (Date.now() < root._bokeh_timeout) {\n", " setTimeout(run_inline_js, 100);\n", " } else if (!root._bokeh_failed_load) {\n", " console.log(\"Bokeh: BokehJS failed to load within specified timeout.\");\n", " root._bokeh_failed_load = true;\n", " } else if (force !== true) {\n", " var cell = $(document.getElementById(\"1001\")).parents('.cell').data().cell;\n", " cell.output_area.append_execute_result(NB_LOAD_WARNING)\n", " }\n", "\n", " }\n", "\n", " if (root._bokeh_is_loading === 0) {\n", " console.debug(\"Bokeh: BokehJS loaded, going straight to plotting\");\n", " run_inline_js();\n", " } else {\n", " load_libs(css_urls, js_urls, function() {\n", " console.debug(\"Bokeh: BokehJS plotting callback run at\", now());\n", " run_inline_js();\n", " });\n", " }\n", "}(window));" ], "application/vnd.bokehjs_load.v0+json": "\n(function(root) {\n function now() {\n return new Date();\n }\n\n var force = true;\n\n if (typeof root._bokeh_onload_callbacks === \"undefined\" || force === true) {\n root._bokeh_onload_callbacks = [];\n root._bokeh_is_loading = undefined;\n }\n\n \n\n \n if (typeof (root._bokeh_timeout) === \"undefined\" || force === true) {\n root._bokeh_timeout = Date.now() + 5000;\n root._bokeh_failed_load = false;\n }\n\n var NB_LOAD_WARNING = {'data': {'text/html':\n \"
\\n\"+\n \"

\\n\"+\n \"BokehJS does not appear to have successfully loaded. If loading BokehJS from CDN, this \\n\"+\n \"may be due to a slow or bad network connection. Possible fixes:\\n\"+\n \"

\\n\"+\n \"\\n\"+\n \"\\n\"+\n \"from bokeh.resources import INLINE\\n\"+\n \"output_notebook(resources=INLINE)\\n\"+\n \"\\n\"+\n \"
\"}};\n\n function display_loaded() {\n var el = document.getElementById(\"1001\");\n if (el != null) {\n el.textContent = \"BokehJS is loading...\";\n }\n if (root.Bokeh !== undefined) {\n if (el != null) {\n el.textContent = \"BokehJS \" + root.Bokeh.version + \" successfully loaded.\";\n }\n } else if (Date.now() < root._bokeh_timeout) {\n setTimeout(display_loaded, 100)\n }\n }\n\n\n function run_callbacks() {\n try {\n root._bokeh_onload_callbacks.forEach(function(callback) {\n if (callback != null)\n callback();\n });\n } finally {\n delete root._bokeh_onload_callbacks\n }\n console.debug(\"Bokeh: all callbacks have finished\");\n }\n\n function load_libs(css_urls, js_urls, callback) {\n if (css_urls == null) css_urls = [];\n if (js_urls == null) js_urls = [];\n\n root._bokeh_onload_callbacks.push(callback);\n if (root._bokeh_is_loading > 0) {\n console.debug(\"Bokeh: BokehJS is being loaded, scheduling callback at\", now());\n return null;\n }\n if (js_urls == null || js_urls.length === 0) {\n run_callbacks();\n return null;\n }\n console.debug(\"Bokeh: BokehJS not loaded, scheduling load and callback at\", now());\n root._bokeh_is_loading = css_urls.length + js_urls.length;\n\n function on_load() {\n root._bokeh_is_loading--;\n if (root._bokeh_is_loading === 0) {\n console.debug(\"Bokeh: all BokehJS libraries/stylesheets loaded\");\n run_callbacks()\n }\n }\n\n function on_error() {\n console.error(\"failed to load \" + url);\n }\n\n for (var i = 0; i < css_urls.length; i++) {\n var url = css_urls[i];\n const element = document.createElement(\"link\");\n element.onload = on_load;\n element.onerror = on_error;\n element.rel = \"stylesheet\";\n element.type = \"text/css\";\n element.href = url;\n console.debug(\"Bokeh: injecting link tag for BokehJS stylesheet: \", url);\n document.body.appendChild(element);\n }\n\n const hashes = {\"https://cdn.bokeh.org/bokeh/release/bokeh-2.2.1.min.js\": \"qkRvDQVAIfzsJo40iRBbxt6sttt0hv4lh74DG7OK4MCHv4C5oohXYoHUM5W11uqS\", \"https://cdn.bokeh.org/bokeh/release/bokeh-widgets-2.2.1.min.js\": \"Sb7Mr06a9TNlet/GEBeKaf5xH3eb6AlCzwjtU82wNPyDrnfoiVl26qnvlKjmcAd+\", \"https://cdn.bokeh.org/bokeh/release/bokeh-tables-2.2.1.min.js\": \"HaJ15vgfmcfRtB4c4YBOI4f1MUujukqInOWVqZJZZGK7Q+ivud0OKGSTn/Vm2iso\"};\n\n for (var i = 0; i < js_urls.length; i++) {\n var url = js_urls[i];\n var element = document.createElement('script');\n element.onload = on_load;\n element.onerror = on_error;\n element.async = false;\n element.src = url;\n if (url in hashes) {\n element.crossOrigin = \"anonymous\";\n element.integrity = \"sha384-\" + hashes[url];\n }\n console.debug(\"Bokeh: injecting script tag for BokehJS library: \", url);\n document.head.appendChild(element);\n }\n };\n\n function inject_raw_css(css) {\n const element = document.createElement(\"style\");\n element.appendChild(document.createTextNode(css));\n document.body.appendChild(element);\n }\n\n \n var js_urls = [\"https://cdn.bokeh.org/bokeh/release/bokeh-2.2.1.min.js\", \"https://cdn.bokeh.org/bokeh/release/bokeh-widgets-2.2.1.min.js\", \"https://cdn.bokeh.org/bokeh/release/bokeh-tables-2.2.1.min.js\"];\n var css_urls = [];\n \n\n var inline_js = [\n function(Bokeh) {\n Bokeh.set_log_level(\"info\");\n },\n function(Bokeh) {\n \n \n }\n ];\n\n function run_inline_js() {\n \n if (root.Bokeh !== undefined || force === true) {\n \n for (var i = 0; i < inline_js.length; i++) {\n inline_js[i].call(root, root.Bokeh);\n }\n if (force === true) {\n display_loaded();\n }} else if (Date.now() < root._bokeh_timeout) {\n setTimeout(run_inline_js, 100);\n } else if (!root._bokeh_failed_load) {\n console.log(\"Bokeh: BokehJS failed to load within specified timeout.\");\n root._bokeh_failed_load = true;\n } else if (force !== true) {\n var cell = $(document.getElementById(\"1001\")).parents('.cell').data().cell;\n cell.output_area.append_execute_result(NB_LOAD_WARNING)\n }\n\n }\n\n if (root._bokeh_is_loading === 0) {\n console.debug(\"Bokeh: BokehJS loaded, going straight to plotting\");\n run_inline_js();\n } else {\n load_libs(css_urls, js_urls, function() {\n console.debug(\"Bokeh: BokehJS plotting callback run at\", now());\n run_inline_js();\n });\n }\n}(window));" }, "metadata": {}, "output_type": "display_data" } ], "source": [ "# Colab setup ------------------\n", "import os, sys, subprocess\n", "if \"google.colab\" in sys.modules:\n", " cmd = \"pip install --upgrade iqplot watermark\"\n", " process = subprocess.Popen(cmd.split(), stdout=subprocess.PIPE, stderr=subprocess.PIPE)\n", " stdout, stderr = process.communicate()\n", " data_path = \"https://s3.amazonaws.com/bebi103.caltech.edu/data/\"\n", "else:\n", " data_path = \"../data/\"\n", "# ------------------------------\n", "\n", "import numpy as np\n", "import pandas as pd\n", "\n", "import iqplot\n", "\n", "import bokeh.io\n", "bokeh.io.output_notebook()" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "
" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "It often happens that experiments consist of multiple data files that need to be brought together into a single data frame to work with in exploratory data analysis and subsequent analyses. Through its concatenation and merging capabilities, Pandas provides powerful tools for handling this sort of data." ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "## The frog tongue strike data set\n", "\n", "As usual, we will work with a real data set to learn about concatenation and merging of data frames. The data set we will use comes from a fun paper about the adhesive properties of frog tongues. The reference is [Kleinteich and Gorb, Tongue adhesion in the horned frog *Ceratophrys sp.*, *Sci. Rep.*, **4**, 5225, 2014](https://dx.doi.org/10.1038%2Fsrep05225). You might also want to check out a *New York Times* feature on the paper [here](http://www.nytimes.com/2014/08/25/science/a-frog-thats-a-living-breathing-pac-man.html).\n", "\n", "In this paper, the authors investigated various properties of the adhesive characteristics of the tongues of horned frogs when they strike prey. The authors had a striking pad connected to a cantilever to measure forces. They also used high speed cameras to capture the strike and record relevant data.\n", "\n", "To get an idea of the experimental set up, you can check out this movie, kindly sent to me by Thomas Kleinteich. If video does not play in your browser, you may download it [here](kleinteich_frog_strike.mp4).\n", "\n", "
\n", " \n", "\n", " \n", "
" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "### The data files\n", "\n", "I pulled data files from the [Kleinteich and Gorb paper](https://dx.doi.org/10.1038%2Fsrep05225). You can download the data files here: [https://s3.amazonaws.com/bebi103.caltech.edu/data/frog_strikes.zip](https://s3.amazonaws.com/bebi103.caltech.edu/data/frog_strikes.zip).\n", "\n", "There are four files, one for each of the four frogs, labeled with IDs I, II, III, and IV, that were studied. To see the format of the files, we can look at the content of the file for frog I. You can use\n", "\n", " head -n 20 ../data/frog_tongue_adhesion.csv\n", " \n", "from the command line. Here is the content of the first data file.\n", "\n", "```\n", "# These data are from Kleinteich and Gorb, Sci. Rep., 4, 5225, 2014.\n", "# Frog ID: I\n", "# Age: adult\n", "# Snout-vent-length (SVL): 63 mm\n", "# Body weight: 63.1 g\n", "# Species: Ceratophrys cranwelli crossed with Ceratophrys cornuta\n", "date,trial number,impact force (mN),impact time (ms),impact force / body weight,adhesive force (mN),time frog pulls on target (ms),adhesive force / body weight,adhesive impulse (N-s),total contact area (mm2),contact area without mucus (mm2),contact area with mucus / contact area without mucus,contact pressure (Pa),adhesive strength (Pa)\n", "2013_02_26,3,1205,46,1.95,-785,884,1.27,-0.290,387,70,0.82,3117,-2030\n", "2013_02_26,4,2527,44,4.08,-983,248,1.59,-0.181,101,94,0.07,24923,-9695\n", "2013_03_01,1,1745,34,2.82,-850,211,1.37,-0.157,83,79,0.05,21020,-10239\n", "2013_03_01,2,1556,41,2.51,-455,1025,0.74,-0.170,330,158,0.52,4718,-1381\n", "2013_03_01,3,493,36,0.80,-974,499,1.57,-0.423,245,216,0.12,2012,-3975\n", "2013_03_01,4,2276,31,3.68,-592,969,0.96,-0.176,341,106,0.69,6676,-1737\n", "2013_03_05,1,556,43,0.90,-512,835,0.83,-0.285,359,110,0.69,1550,-1427\n", "2013_03_05,2,1928,46,3.11,-804,508,1.30,-0.285,246,178,0.28,7832,-3266\n", "2013_03_05,3,2641,50,4.27,-690,491,1.12,-0.239,269,224,0.17,9824,-2568\n", "2013_03_05,4,1897,41,3.06,-462,839,0.75,-0.328,266,176,0.34,7122,-1733\n", "2013_03_12,1,1891,40,3.06,-766,1069,1.24,-0.380,408,33,0.92,4638,-1879\n", "2013_03_12,2,1545,48,2.50,-715,649,1.15,-0.298,141,112,0.21,10947,-5064\n", "2013_03_12,3,1307,29,2.11,-613,1845,0.99,-0.768,455,92,0.80,2874,-1348\n", "2013_03_12,4,1692,31,2.73,-677,917,1.09,-0.457,186,129,0.31,9089,-3636\n", "2013_03_12,5,1543,38,2.49,-528,750,0.85,-0.353,153,148,0.03,10095,-3453\n", "2013_03_15,1,1282,31,2.07,-452,785,0.73,-0.253,290,105,0.64,4419,-1557\n", "2013_03_15,2,775,34,1.25,-430,837,0.70,-0.276,257,124,0.52,3019,-1677\n", "2013_03_15,3,2032,60,3.28,-652,486,1.05,-0.257,147,134,0.09,13784,-4425\n", "2013_03_15,4,1240,34,2.00,-692,906,1.12,-0.317,364,260,0.28,3406,-1901\n", "2013_03_15,5,473,40,0.76,-536,1218,0.87,-0.382,259,168,0.35,1830,-2073\n", "```\n", "\n", "The first lines all begin with `#` signs, signifying that they are comments. They do give important information about the frog, though.\n", "\n", "The first line after the comments are the headers, giving the column names for the data frame we will load." ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "## Concatenating data frames\n", "\n", "We would like to have all of the data frames be together in one data frame so we can conveniently do things like make plots comparing the four frogs. Let's read in the data sets and make a list of data frames." ] }, { "cell_type": "code", "execution_count": 2, "metadata": {}, "outputs": [ { "data": { "text/html": [ "
\n", "\n", "\n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", "
datetrial numberimpact force (mN)impact time (ms)impact force / body weightadhesive force (mN)time frog pulls on target (ms)adhesive force / body weightadhesive impulse (N-s)total contact area (mm2)contact area without mucus (mm2)contact area with mucus / contact area without mucuscontact pressure (Pa)adhesive strength (Pa)
02013_02_2631205461.95-7858841.27-0.290387700.823117-2030
12013_02_2642527444.08-9832481.59-0.181101940.0724923-9695
22013_03_0111745342.82-8502111.37-0.15783790.0521020-10239
32013_03_0121556412.51-45510250.74-0.1703301580.524718-1381
42013_03_013493360.80-9744991.57-0.4232452160.122012-3975
\n", "
" ], "text/plain": [ " date trial number impact force (mN) impact time (ms) \\\n", "0 2013_02_26 3 1205 46 \n", "1 2013_02_26 4 2527 44 \n", "2 2013_03_01 1 1745 34 \n", "3 2013_03_01 2 1556 41 \n", "4 2013_03_01 3 493 36 \n", "\n", " impact force / body weight adhesive force (mN) \\\n", "0 1.95 -785 \n", "1 4.08 -983 \n", "2 2.82 -850 \n", "3 2.51 -455 \n", "4 0.80 -974 \n", "\n", " time frog pulls on target (ms) adhesive force / body weight \\\n", "0 884 1.27 \n", "1 248 1.59 \n", "2 211 1.37 \n", "3 1025 0.74 \n", "4 499 1.57 \n", "\n", " adhesive impulse (N-s) total contact area (mm2) \\\n", "0 -0.290 387 \n", "1 -0.181 101 \n", "2 -0.157 83 \n", "3 -0.170 330 \n", "4 -0.423 245 \n", "\n", " contact area without mucus (mm2) \\\n", "0 70 \n", "1 94 \n", "2 79 \n", "3 158 \n", "4 216 \n", "\n", " contact area with mucus / contact area without mucus \\\n", "0 0.82 \n", "1 0.07 \n", "2 0.05 \n", "3 0.52 \n", "4 0.12 \n", "\n", " contact pressure (Pa) adhesive strength (Pa) \n", "0 3117 -2030 \n", "1 24923 -9695 \n", "2 21020 -10239 \n", "3 4718 -1381 \n", "4 2012 -3975 " ] }, "execution_count": 2, "metadata": {}, "output_type": "execute_result" } ], "source": [ "# On a local machine, we would do this:\n", "# fnames = glob.glob('../data/frog_strikes_*.csv')\n", "#\n", "# But for Colab compatibility, we will do it by hand\n", "fnames = [\n", " os.path.join(data_path, f\"frog_strikes_{frog_id}.csv\")\n", " for frog_id in [\"I\", \"II\", \"III\", \"IV\"]\n", "]\n", "\n", "dfs = [pd.read_csv(f, comment=\"#\") for f in fnames]\n", "\n", "# Take a look at first data frame\n", "dfs[0].head()" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "We have successfully loaded in all of the data frames. They all have the same columns (as given by the CSV files) and they all have the same indexes (range indexes that were applied be default when loading from the CSV files). We do not really care about the indexes. So, we wish to tape the data frames together vertically. We can use the `pd.concat()` function to do this.\n", "\n", "Before we do that, though, we might notice a problem. We will not have information to tell us which frog is which. We might therefore like to add a column to each data frame that has the frog ID, and then concatenate them. We can parse the ID of the frog from the file name, as we can see by looking at the file names." ] }, { "cell_type": "code", "execution_count": 3, "metadata": {}, "outputs": [ { "data": { "text/plain": [ "['../data/frog_strikes_I.csv',\n", " '../data/frog_strikes_II.csv',\n", " '../data/frog_strikes_III.csv',\n", " '../data/frog_strikes_IV.csv']" ] }, "execution_count": 3, "metadata": {}, "output_type": "execute_result" } ], "source": [ "fnames" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "So, for each data frame/file name pair, we extract the Roman numeral and add a column to the data frame containing the frog ID." ] }, { "cell_type": "code", "execution_count": 4, "metadata": {}, "outputs": [ { "data": { "text/html": [ "
\n", "\n", "\n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", "
datetrial numberimpact force (mN)impact time (ms)impact force / body weightadhesive force (mN)time frog pulls on target (ms)adhesive force / body weightadhesive impulse (N-s)total contact area (mm2)contact area without mucus (mm2)contact area with mucus / contact area without mucuscontact pressure (Pa)adhesive strength (Pa)ID
02013_02_2631205461.95-7858841.27-0.290387700.823117-2030I
12013_02_2642527444.08-9832481.59-0.181101940.0724923-9695I
22013_03_0111745342.82-8502111.37-0.15783790.0521020-10239I
32013_03_0121556412.51-45510250.74-0.1703301580.524718-1381I
42013_03_013493360.80-9744991.57-0.4232452160.122012-3975I
\n", "
" ], "text/plain": [ " date trial number impact force (mN) impact time (ms) \\\n", "0 2013_02_26 3 1205 46 \n", "1 2013_02_26 4 2527 44 \n", "2 2013_03_01 1 1745 34 \n", "3 2013_03_01 2 1556 41 \n", "4 2013_03_01 3 493 36 \n", "\n", " impact force / body weight adhesive force (mN) \\\n", "0 1.95 -785 \n", "1 4.08 -983 \n", "2 2.82 -850 \n", "3 2.51 -455 \n", "4 0.80 -974 \n", "\n", " time frog pulls on target (ms) adhesive force / body weight \\\n", "0 884 1.27 \n", "1 248 1.59 \n", "2 211 1.37 \n", "3 1025 0.74 \n", "4 499 1.57 \n", "\n", " adhesive impulse (N-s) total contact area (mm2) \\\n", "0 -0.290 387 \n", "1 -0.181 101 \n", "2 -0.157 83 \n", "3 -0.170 330 \n", "4 -0.423 245 \n", "\n", " contact area without mucus (mm2) \\\n", "0 70 \n", "1 94 \n", "2 79 \n", "3 158 \n", "4 216 \n", "\n", " contact area with mucus / contact area without mucus \\\n", "0 0.82 \n", "1 0.07 \n", "2 0.05 \n", "3 0.52 \n", "4 0.12 \n", "\n", " contact pressure (Pa) adhesive strength (Pa) ID \n", "0 3117 -2030 I \n", "1 24923 -9695 I \n", "2 21020 -10239 I \n", "3 4718 -1381 I \n", "4 2012 -3975 I " ] }, "execution_count": 4, "metadata": {}, "output_type": "execute_result" } ], "source": [ "for i, f in enumerate(fnames):\n", " frog_id = f[f.rfind('_')+1:f.rfind('.')]\n", " dfs[i]['ID'] = frog_id\n", " \n", "# Take a look\n", "dfs[0].head()" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "Good! Now all data frames have an `'ID'` column, and we can concatenate. The `pd.concat()` function takes as input a list of data frames to be concatenated. Since we do not care about the index, we can use the `ignore_index=True` kwarg." ] }, { "cell_type": "code", "execution_count": 5, "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "Number of rows: 80 \n", "Unique IDs: ['I' 'II' 'III' 'IV']\n" ] } ], "source": [ "df = pd.concat(dfs, ignore_index=True)\n", "\n", "# Make sure we got them all\n", "print('Number of rows:', len(df), '\\nUnique IDs:', df['ID'].unique())" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "Check!" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "### More advanced concatenation\n", "\n", "When we concatenated, we updated each data frame with a fresh column. The `pd.concat()` function can handle some of this for you. If we instead passed a dictionary of data frames instead of a list, it applies the keys to each data frame that is concatenated using a multiindex. First, we'll read in the data frames as a dictionary of data frames instead of a list." ] }, { "cell_type": "code", "execution_count": 6, "metadata": {}, "outputs": [ { "data": { "text/plain": [ "dict_keys(['I', 'II', 'III', 'IV'])" ] }, "execution_count": 6, "metadata": {}, "output_type": "execute_result" } ], "source": [ "# Make dictionary of data frames\n", "dfs = {\n", " f[f.rfind(\"_\") + 1 : f.rfind(\".\")]: pd.read_csv(f, comment=\"#\")\n", " for i, f in enumerate(fnames)\n", "}\n", "\n", "# Verify that keys are in fact IDs\n", "dfs.keys()" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "Now, if we call `pd.concat()` with dictionary input, we get a new data frame with a multiindex." ] }, { "cell_type": "code", "execution_count": 7, "metadata": {}, "outputs": [ { "data": { "text/html": [ "
\n", "\n", "\n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", "
datetrial numberimpact force (mN)impact time (ms)impact force / body weightadhesive force (mN)time frog pulls on target (ms)adhesive force / body weightadhesive impulse (N-s)total contact area (mm2)contact area without mucus (mm2)contact area with mucus / contact area without mucuscontact pressure (Pa)adhesive strength (Pa)
I02013_02_2631205461.95-7858841.27-0.290387700.823117-2030
12013_02_2642527444.08-9832481.59-0.181101940.0724923-9695
22013_03_0111745342.82-8502111.37-0.15783790.0521020-10239
32013_03_0121556412.51-45510250.74-0.1703301580.524718-1381
42013_03_013493360.80-9744991.57-0.4232452160.122012-3975
\n", "
" ], "text/plain": [ " date trial number impact force (mN) impact time (ms) \\\n", "I 0 2013_02_26 3 1205 46 \n", " 1 2013_02_26 4 2527 44 \n", " 2 2013_03_01 1 1745 34 \n", " 3 2013_03_01 2 1556 41 \n", " 4 2013_03_01 3 493 36 \n", "\n", " impact force / body weight adhesive force (mN) \\\n", "I 0 1.95 -785 \n", " 1 4.08 -983 \n", " 2 2.82 -850 \n", " 3 2.51 -455 \n", " 4 0.80 -974 \n", "\n", " time frog pulls on target (ms) adhesive force / body weight \\\n", "I 0 884 1.27 \n", " 1 248 1.59 \n", " 2 211 1.37 \n", " 3 1025 0.74 \n", " 4 499 1.57 \n", "\n", " adhesive impulse (N-s) total contact area (mm2) \\\n", "I 0 -0.290 387 \n", " 1 -0.181 101 \n", " 2 -0.157 83 \n", " 3 -0.170 330 \n", " 4 -0.423 245 \n", "\n", " contact area without mucus (mm2) \\\n", "I 0 70 \n", " 1 94 \n", " 2 79 \n", " 3 158 \n", " 4 216 \n", "\n", " contact area with mucus / contact area without mucus \\\n", "I 0 0.82 \n", " 1 0.07 \n", " 2 0.05 \n", " 3 0.52 \n", " 4 0.12 \n", "\n", " contact pressure (Pa) adhesive strength (Pa) \n", "I 0 3117 -2030 \n", " 1 24923 -9695 \n", " 2 21020 -10239 \n", " 3 4718 -1381 \n", " 4 2012 -3975 " ] }, "execution_count": 7, "metadata": {}, "output_type": "execute_result" } ], "source": [ "df = pd.concat(dfs)\n", "\n", "# Take a look\n", "df.head()" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "We have a multiindex for the rows, with the high level index being the ID and the low level index being the original index of the data frame that was concatenated. It is useful to give these indexes names so we can conveniently refer to them. We can do that by setting the `df.index.names` property as\n", "\n", "```python\n", "df.index.names = ['ID', 'original index']\n", "```\n", "\n", "We can instead specify a `names` kwarg when we call `pd.concat()`. This kwarg specifies the names of the resulting multiindex from the concatenation." ] }, { "cell_type": "code", "execution_count": 8, "metadata": {}, "outputs": [ { "data": { "text/html": [ "
\n", "\n", "\n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", "
datetrial numberimpact force (mN)impact time (ms)impact force / body weightadhesive force (mN)time frog pulls on target (ms)adhesive force / body weightadhesive impulse (N-s)total contact area (mm2)contact area without mucus (mm2)contact area with mucus / contact area without mucuscontact pressure (Pa)adhesive strength (Pa)
IDoriginal index
I02013_02_2631205461.95-7858841.27-0.290387700.823117-2030
12013_02_2642527444.08-9832481.59-0.181101940.0724923-9695
22013_03_0111745342.82-8502111.37-0.15783790.0521020-10239
32013_03_0121556412.51-45510250.74-0.1703301580.524718-1381
42013_03_013493360.80-9744991.57-0.4232452160.122012-3975
\n", "
" ], "text/plain": [ " date trial number impact force (mN) \\\n", "ID original index \n", "I 0 2013_02_26 3 1205 \n", " 1 2013_02_26 4 2527 \n", " 2 2013_03_01 1 1745 \n", " 3 2013_03_01 2 1556 \n", " 4 2013_03_01 3 493 \n", "\n", " impact time (ms) impact force / body weight \\\n", "ID original index \n", "I 0 46 1.95 \n", " 1 44 4.08 \n", " 2 34 2.82 \n", " 3 41 2.51 \n", " 4 36 0.80 \n", "\n", " adhesive force (mN) time frog pulls on target (ms) \\\n", "ID original index \n", "I 0 -785 884 \n", " 1 -983 248 \n", " 2 -850 211 \n", " 3 -455 1025 \n", " 4 -974 499 \n", "\n", " adhesive force / body weight adhesive impulse (N-s) \\\n", "ID original index \n", "I 0 1.27 -0.290 \n", " 1 1.59 -0.181 \n", " 2 1.37 -0.157 \n", " 3 0.74 -0.170 \n", " 4 1.57 -0.423 \n", "\n", " total contact area (mm2) contact area without mucus (mm2) \\\n", "ID original index \n", "I 0 387 70 \n", " 1 101 94 \n", " 2 83 79 \n", " 3 330 158 \n", " 4 245 216 \n", "\n", " contact area with mucus / contact area without mucus \\\n", "ID original index \n", "I 0 0.82 \n", " 1 0.07 \n", " 2 0.05 \n", " 3 0.52 \n", " 4 0.12 \n", "\n", " contact pressure (Pa) adhesive strength (Pa) \n", "ID original index \n", "I 0 3117 -2030 \n", " 1 24923 -9695 \n", " 2 21020 -10239 \n", " 3 4718 -1381 \n", " 4 2012 -3975 " ] }, "execution_count": 8, "metadata": {}, "output_type": "execute_result" } ], "source": [ "df = pd.concat(dfs, names=['ID', 'original index'])\n", "\n", "# Take a look\n", "df.head()" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "We conveniently have labeled indexes, and we can now make `ID` a column in the data frame using the `reset_index()` method." ] }, { "cell_type": "code", "execution_count": 9, "metadata": {}, "outputs": [ { "data": { "text/html": [ "
\n", "\n", "\n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", "
IDoriginal indexdatetrial numberimpact force (mN)impact time (ms)impact force / body weightadhesive force (mN)time frog pulls on target (ms)adhesive force / body weightadhesive impulse (N-s)total contact area (mm2)contact area without mucus (mm2)contact area with mucus / contact area without mucuscontact pressure (Pa)adhesive strength (Pa)
0I02013_02_2631205461.95-7858841.27-0.290387700.823117-2030
1I12013_02_2642527444.08-9832481.59-0.181101940.0724923-9695
2I22013_03_0111745342.82-8502111.37-0.15783790.0521020-10239
3I32013_03_0121556412.51-45510250.74-0.1703301580.524718-1381
4I42013_03_013493360.80-9744991.57-0.4232452160.122012-3975
\n", "
" ], "text/plain": [ " ID original index date trial number impact force (mN) \\\n", "0 I 0 2013_02_26 3 1205 \n", "1 I 1 2013_02_26 4 2527 \n", "2 I 2 2013_03_01 1 1745 \n", "3 I 3 2013_03_01 2 1556 \n", "4 I 4 2013_03_01 3 493 \n", "\n", " impact time (ms) impact force / body weight adhesive force (mN) \\\n", "0 46 1.95 -785 \n", "1 44 4.08 -983 \n", "2 34 2.82 -850 \n", "3 41 2.51 -455 \n", "4 36 0.80 -974 \n", "\n", " time frog pulls on target (ms) adhesive force / body weight \\\n", "0 884 1.27 \n", "1 248 1.59 \n", "2 211 1.37 \n", "3 1025 0.74 \n", "4 499 1.57 \n", "\n", " adhesive impulse (N-s) total contact area (mm2) \\\n", "0 -0.290 387 \n", "1 -0.181 101 \n", "2 -0.157 83 \n", "3 -0.170 330 \n", "4 -0.423 245 \n", "\n", " contact area without mucus (mm2) \\\n", "0 70 \n", "1 94 \n", "2 79 \n", "3 158 \n", "4 216 \n", "\n", " contact area with mucus / contact area without mucus \\\n", "0 0.82 \n", "1 0.07 \n", "2 0.05 \n", "3 0.52 \n", "4 0.12 \n", "\n", " contact pressure (Pa) adhesive strength (Pa) \n", "0 3117 -2030 \n", "1 24923 -9695 \n", "2 21020 -10239 \n", "3 4718 -1381 \n", "4 2012 -3975 " ] }, "execution_count": 9, "metadata": {}, "output_type": "execute_result" } ], "source": [ "df = df.reset_index()\n", "\n", "df.head()" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "We now have a default range index for the data frame that we do not care about. Because the original index was not informative either, we can delete that column if we like, but it is not really a burden to have an unused column laying around in a data set this small. Nonetheless, let's blow it away." ] }, { "cell_type": "code", "execution_count": 10, "metadata": {}, "outputs": [ { "data": { "text/html": [ "
\n", "\n", "\n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", "
IDdatetrial numberimpact force (mN)impact time (ms)impact force / body weightadhesive force (mN)time frog pulls on target (ms)adhesive force / body weightadhesive impulse (N-s)total contact area (mm2)contact area without mucus (mm2)contact area with mucus / contact area without mucuscontact pressure (Pa)adhesive strength (Pa)
0I2013_02_2631205461.95-7858841.27-0.290387700.823117-2030
1I2013_02_2642527444.08-9832481.59-0.181101940.0724923-9695
2I2013_03_0111745342.82-8502111.37-0.15783790.0521020-10239
3I2013_03_0121556412.51-45510250.74-0.1703301580.524718-1381
4I2013_03_013493360.80-9744991.57-0.4232452160.122012-3975
\n", "
" ], "text/plain": [ " ID date trial number impact force (mN) impact time (ms) \\\n", "0 I 2013_02_26 3 1205 46 \n", "1 I 2013_02_26 4 2527 44 \n", "2 I 2013_03_01 1 1745 34 \n", "3 I 2013_03_01 2 1556 41 \n", "4 I 2013_03_01 3 493 36 \n", "\n", " impact force / body weight adhesive force (mN) \\\n", "0 1.95 -785 \n", "1 4.08 -983 \n", "2 2.82 -850 \n", "3 2.51 -455 \n", "4 0.80 -974 \n", "\n", " time frog pulls on target (ms) adhesive force / body weight \\\n", "0 884 1.27 \n", "1 248 1.59 \n", "2 211 1.37 \n", "3 1025 0.74 \n", "4 499 1.57 \n", "\n", " adhesive impulse (N-s) total contact area (mm2) \\\n", "0 -0.290 387 \n", "1 -0.181 101 \n", "2 -0.157 83 \n", "3 -0.170 330 \n", "4 -0.423 245 \n", "\n", " contact area without mucus (mm2) \\\n", "0 70 \n", "1 94 \n", "2 79 \n", "3 158 \n", "4 216 \n", "\n", " contact area with mucus / contact area without mucus \\\n", "0 0.82 \n", "1 0.07 \n", "2 0.05 \n", "3 0.52 \n", "4 0.12 \n", "\n", " contact pressure (Pa) adhesive strength (Pa) \n", "0 3117 -2030 \n", "1 24923 -9695 \n", "2 21020 -10239 \n", "3 4718 -1381 \n", "4 2012 -3975 " ] }, "execution_count": 10, "metadata": {}, "output_type": "execute_result" } ], "source": [ "del df['original index']\n", "\n", "df.head()" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "We now have a nice, tidy data frame!" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "## Creating a DataFrame from scratch\n", "\n", "Looking back at the [headers of the original data files](#The-data-files), we see that there is information present in the header that we would like to have in our data frame. For example, it would be nice to know if each strike came from an adult or juvenile. Or what the snout-vent length was. Going toward the goal of including this in our data frame, we will first construct a new data frame containing information about each frog." ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "### Data frames from dictionaries\n", "\n", "One way do create this new data frame is to first construct a dictionary with the respective fields. Since these data sets are small, we can look at the files and make the dictionary by hand." ] }, { "cell_type": "code", "execution_count": 11, "metadata": {}, "outputs": [], "source": [ "data_dict = {\n", " \"ID\": [\"I\", \"II\", \"III\", \"IV\"],\n", " \"age\": [\"adult\", \"adult\", \"juvenile\", \"juvenile\"],\n", " \"SVL (mm)\": [63, 70, 28, 31],\n", " \"body weight (g)\": [63.1, 72.7, 12.7, 12.7],\n", " \"species\": [\"cross\", \"cross\", \"cranwelli\", \"cranwelli\"],\n", "}" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "Now that we have this dictionary, we can convert it into a `DataFrame` by instantiating a `pd.DataFrame` class with it, using the `data` kwarg." ] }, { "cell_type": "code", "execution_count": 12, "metadata": {}, "outputs": [ { "data": { "text/html": [ "
\n", "\n", "\n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", "
IDageSVL (mm)body weight (g)species
0Iadult6363.1cross
1IIadult7072.7cross
2IIIjuvenile2812.7cranwelli
3IVjuvenile3112.7cranwelli
\n", "
" ], "text/plain": [ " ID age SVL (mm) body weight (g) species\n", "0 I adult 63 63.1 cross\n", "1 II adult 70 72.7 cross\n", "2 III juvenile 28 12.7 cranwelli\n", "3 IV juvenile 31 12.7 cranwelli" ] }, "execution_count": 12, "metadata": {}, "output_type": "execute_result" } ], "source": [ "# Make it into a DataFrame\n", "df_frog_info = pd.DataFrame(data=data_dict)\n", "\n", "# Take a look\n", "df_frog_info" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "Nice!" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "### Data frames from numpy arrays\n", "\n", "Sometimes the data sets are not small enough to construct a dictionary by hand. Oftentimes, we have a two-dimensional array of data that we want to make into a `DataFrame`. As an example, let's say we have a Numpy array where the first column is snout vent length and the second is weight." ] }, { "cell_type": "code", "execution_count": 13, "metadata": {}, "outputs": [ { "data": { "text/plain": [ "array([[63. , 63.1],\n", " [70. , 72.7],\n", " [28. , 12.7],\n", " [31. , 12.7]])" ] }, "execution_count": 13, "metadata": {}, "output_type": "execute_result" } ], "source": [ "data = np.array([[63, 70, 28, 31], [63.1, 72.7, 12.7, 12.7]]).transpose()\n", "\n", "# Verify that it's what we think it is\n", "data" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "To make this into a `DataFrame`, we again create `pd.DataFrame` instance, but this time we also specify the `column` keyword argument." ] }, { "cell_type": "code", "execution_count": 14, "metadata": {}, "outputs": [ { "data": { "text/html": [ "
\n", "\n", "\n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", "
SVL (mm)weight (g)
063.063.1
170.072.7
228.012.7
331.012.7
\n", "
" ], "text/plain": [ " SVL (mm) weight (g)\n", "0 63.0 63.1\n", "1 70.0 72.7\n", "2 28.0 12.7\n", "3 31.0 12.7" ] }, "execution_count": 14, "metadata": {}, "output_type": "execute_result" } ], "source": [ "df_demo = pd.DataFrame(data=data, columns=[\"SVL (mm)\", \"weight (g)\"])\n", "\n", "# Take a look\n", "df_demo" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "That also works. Generally, any two-dimensional Numpy array can be converted into a `DataFrame` in this way. You just need to supply column names." ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "### Programmatically creating a data frame\n", "\n", "Hand-entering data should be minimized. The information about each frog were hand-entered once by the experimenter. We should not hand-enter them again. We therefore should parse the comment lines of input files to get the pertinent information.\n", "\n", "Note, though, that in the case of a single experiment with only four data sets, hand entering might be faster and indeed less error prone than doing it programmatically. We should definitely do it programmatically if we have a large number of data files or will ever do an experiment with the same file format again.\n", "\n", "So, let's programmatically parse the files. We start by writing a function to parse the metadata from a single file. Recall that the comment lines look like this:\n", "\n", "```\n", "# These data are from Kleinteich and Gorb, Sci. Rep., 4, 5225, 2014.\n", "# Frog ID: I\n", "# Age: adult\n", "# Snout-vent-length (SVL): 63 mm\n", "# Body weight: 63.1 g\n", "# Species: Ceratophrys cranwelli crossed with Ceratophrys cornuta\n", "```\n", "\n", "(The function below will not work with Colab because `open()` does not work for files specified by a URL.)" ] }, { "cell_type": "code", "execution_count": 15, "metadata": {}, "outputs": [], "source": [ "def parse_frog_metadata(fname):\n", " with open(fname, 'r') as f:\n", " # Citation line, ignore.\n", " f.readline()\n", " \n", " # Frog ID\n", " line = f.readline()\n", " frog_id = line[line.find(':')+1:].strip()\n", " \n", " # Age\n", " line = f.readline()\n", " age = line[line.find(':')+1:].strip()\n", " \n", " # SVL, assume units given as mm\n", " line = f.readline()\n", " svl = line[line.find(':')+1:line.rfind(' ')].strip()\n", " \n", " # Body weight, assume units given as g\n", " line = f.readline()\n", " body_weight = line[line.find(':')+1:line.rfind(' ')].strip()\n", "\n", " \n", " # Species (either cranwelli or cross)\n", " line = f.readline()\n", " species = line[line.find(':')+1:].strip()\n", " if 'cross' in species:\n", " species = 'cross'\n", " else:\n", " species = 'cranwelli'\n", "\n", " return frog_id, age, svl, body_weight, species" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "Let's take it for a spin." ] }, { "cell_type": "code", "execution_count": 16, "metadata": {}, "outputs": [ { "data": { "text/plain": [ "('I', 'adult', '63', '63.1', 'cross')" ] }, "execution_count": 16, "metadata": {}, "output_type": "execute_result" } ], "source": [ "parse_frog_metadata(os.path.join(data_path, 'frog_strikes_I.csv'))" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "Looks good! Now we can create a list of tuples to use as data for making a data frame." ] }, { "cell_type": "code", "execution_count": 17, "metadata": {}, "outputs": [ { "data": { "text/plain": [ "[('I', 'adult', '63', '63.1', 'cross'),\n", " ('II', 'adult', '70', '72.7', 'cross'),\n", " ('III', 'juvenile', '28', '12.7', 'cranwelli'),\n", " ('IV', 'juvenile', '31', '12.7', 'cranwelli')]" ] }, "execution_count": 17, "metadata": {}, "output_type": "execute_result" } ], "source": [ "data = [parse_frog_metadata(f) for f in fnames]\n", " \n", "# Take a look\n", "data" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "We now input this list of tuples, plus the column names, into `pd.DataFrame()`, and we've got our data frame." ] }, { "cell_type": "code", "execution_count": 18, "metadata": {}, "outputs": [ { "data": { "text/html": [ "
\n", "\n", "\n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", "
IDageSVL (mm)body weight (g)species
0Iadult6363.1cross
1IIadult7072.7cross
2IIIjuvenile2812.7cranwelli
3IVjuvenile3112.7cranwelli
\n", "
" ], "text/plain": [ " ID age SVL (mm) body weight (g) species\n", "0 I adult 63 63.1 cross\n", "1 II adult 70 72.7 cross\n", "2 III juvenile 28 12.7 cranwelli\n", "3 IV juvenile 31 12.7 cranwelli" ] }, "execution_count": 18, "metadata": {}, "output_type": "execute_result" } ], "source": [ "df_frog_info = pd.DataFrame(\n", " data=data, \n", " columns=[\"ID\", \"age\", \"SVL (mm)\", \"body weight (g)\", \"species\"]\n", ")\n", "\n", "# Take a look\n", "df_frog_info" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "## Merging DataFrames\n", "\n", "Our ultimate goal is to add the information about the frogs into our main data frame, `df`, that we have been working with. " ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "### Brute force merge\n", "\n", "We can do it using tools we have already learned. For each row in the `DataFrame`, we can add the relevant value in each column. Because this will not be the final way I recommend doing this, I will do these operations on a copy of `df` using the `copy()` method." ] }, { "cell_type": "code", "execution_count": 19, "metadata": {}, "outputs": [ { "data": { "text/html": [ "
\n", "\n", "\n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", "
IDdatetrial numberimpact force (mN)impact time (ms)impact force / body weightadhesive force (mN)time frog pulls on target (ms)adhesive force / body weightadhesive impulse (N-s)total contact area (mm2)contact area without mucus (mm2)contact area with mucus / contact area without mucuscontact pressure (Pa)adhesive strength (Pa)ageSVL (mm)body weight (g)species
0I2013_02_2631205461.95-7858841.27-0.290387700.823117-2030adult6363.1cross
1I2013_02_2642527444.08-9832481.59-0.181101940.0724923-9695adult6363.1cross
2I2013_03_0111745342.82-8502111.37-0.15783790.0521020-10239adult6363.1cross
3I2013_03_0121556412.51-45510250.74-0.1703301580.524718-1381adult6363.1cross
4I2013_03_013493360.80-9744991.57-0.4232452160.122012-3975adult6363.1cross
\n", "
" ], "text/plain": [ " ID date trial number impact force (mN) impact time (ms) \\\n", "0 I 2013_02_26 3 1205 46 \n", "1 I 2013_02_26 4 2527 44 \n", "2 I 2013_03_01 1 1745 34 \n", "3 I 2013_03_01 2 1556 41 \n", "4 I 2013_03_01 3 493 36 \n", "\n", " impact force / body weight adhesive force (mN) \\\n", "0 1.95 -785 \n", "1 4.08 -983 \n", "2 2.82 -850 \n", "3 2.51 -455 \n", "4 0.80 -974 \n", "\n", " time frog pulls on target (ms) adhesive force / body weight \\\n", "0 884 1.27 \n", "1 248 1.59 \n", "2 211 1.37 \n", "3 1025 0.74 \n", "4 499 1.57 \n", "\n", " adhesive impulse (N-s) total contact area (mm2) \\\n", "0 -0.290 387 \n", "1 -0.181 101 \n", "2 -0.157 83 \n", "3 -0.170 330 \n", "4 -0.423 245 \n", "\n", " contact area without mucus (mm2) \\\n", "0 70 \n", "1 94 \n", "2 79 \n", "3 158 \n", "4 216 \n", "\n", " contact area with mucus / contact area without mucus \\\n", "0 0.82 \n", "1 0.07 \n", "2 0.05 \n", "3 0.52 \n", "4 0.12 \n", "\n", " contact pressure (Pa) adhesive strength (Pa) age SVL (mm) \\\n", "0 3117 -2030 adult 63 \n", "1 24923 -9695 adult 63 \n", "2 21020 -10239 adult 63 \n", "3 4718 -1381 adult 63 \n", "4 2012 -3975 adult 63 \n", "\n", " body weight (g) species \n", "0 63.1 cross \n", "1 63.1 cross \n", "2 63.1 cross \n", "3 63.1 cross \n", "4 63.1 cross " ] }, "execution_count": 19, "metadata": {}, "output_type": "execute_result" } ], "source": [ "# Make a copy of df\n", "df_copy = df.copy()\n", "\n", "# Build each column\n", "for col in df_frog_info.columns[df_frog_info.columns != 'ID']:\n", " # Make a new column with empty values\n", " df_copy[col] = np.empty(len(df_copy))\n", " \n", " # Add in each entry, row by row\n", " for i, r in df_copy.iterrows():\n", " ind = df_frog_info['ID'] == r['ID']\n", " df_copy.loc[i, col] = df_frog_info.loc[ind, col].iloc[0]\n", " \n", "# Take a look at the updated DataFrame\n", "df_copy.head()" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "Note that I used the `iterrows()` method of the `df_copy` data frame. This iterator gives an index (which I called `i`) and a row of a data frame (which I called `r`). This method, and the analogous one for iterating over columns, `iteritems()`, can be useful.\n", "\n", "But this approach seems rather clunky. " ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "### Using pd.merge()\n", "\n", "There is a much better way to do it is to use Pandas's [built-in merge() method](https://pandas.pydata.org/pandas-docs/stable/merging.html#database-style-dataframe-joining-merging). Called with all the default keyword arguments, this function finds a common columns between two `DataFrame`s (in this case, the `ID` column), and then uses those columns to merge them, filling in values that match in the common column. This is exactly what we want." ] }, { "cell_type": "code", "execution_count": 20, "metadata": {}, "outputs": [ { "data": { "text/html": [ "
\n", "\n", "\n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", "
IDdatetrial numberimpact force (mN)impact time (ms)impact force / body weightadhesive force (mN)time frog pulls on target (ms)adhesive force / body weightadhesive impulse (N-s)total contact area (mm2)contact area without mucus (mm2)contact area with mucus / contact area without mucuscontact pressure (Pa)adhesive strength (Pa)ageSVL (mm)body weight (g)species
0I2013_02_2631205461.95-7858841.27-0.290387700.823117-2030adult6363.1cross
1I2013_02_2642527444.08-9832481.59-0.181101940.0724923-9695adult6363.1cross
2I2013_03_0111745342.82-8502111.37-0.15783790.0521020-10239adult6363.1cross
3I2013_03_0121556412.51-45510250.74-0.1703301580.524718-1381adult6363.1cross
4I2013_03_013493360.80-9744991.57-0.4232452160.122012-3975adult6363.1cross
\n", "
" ], "text/plain": [ " ID date trial number impact force (mN) impact time (ms) \\\n", "0 I 2013_02_26 3 1205 46 \n", "1 I 2013_02_26 4 2527 44 \n", "2 I 2013_03_01 1 1745 34 \n", "3 I 2013_03_01 2 1556 41 \n", "4 I 2013_03_01 3 493 36 \n", "\n", " impact force / body weight adhesive force (mN) \\\n", "0 1.95 -785 \n", "1 4.08 -983 \n", "2 2.82 -850 \n", "3 2.51 -455 \n", "4 0.80 -974 \n", "\n", " time frog pulls on target (ms) adhesive force / body weight \\\n", "0 884 1.27 \n", "1 248 1.59 \n", "2 211 1.37 \n", "3 1025 0.74 \n", "4 499 1.57 \n", "\n", " adhesive impulse (N-s) total contact area (mm2) \\\n", "0 -0.290 387 \n", "1 -0.181 101 \n", "2 -0.157 83 \n", "3 -0.170 330 \n", "4 -0.423 245 \n", "\n", " contact area without mucus (mm2) \\\n", "0 70 \n", "1 94 \n", "2 79 \n", "3 158 \n", "4 216 \n", "\n", " contact area with mucus / contact area without mucus \\\n", "0 0.82 \n", "1 0.07 \n", "2 0.05 \n", "3 0.52 \n", "4 0.12 \n", "\n", " contact pressure (Pa) adhesive strength (Pa) age SVL (mm) \\\n", "0 3117 -2030 adult 63 \n", "1 24923 -9695 adult 63 \n", "2 21020 -10239 adult 63 \n", "3 4718 -1381 adult 63 \n", "4 2012 -3975 adult 63 \n", "\n", " body weight (g) species \n", "0 63.1 cross \n", "1 63.1 cross \n", "2 63.1 cross \n", "3 63.1 cross \n", "4 63.1 cross " ] }, "execution_count": 20, "metadata": {}, "output_type": "execute_result" } ], "source": [ "df = df.merge(df_frog_info)\n", "\n", "# Check it out!\n", "df.head()" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "Boo-yah! Note that the entries for the added columns were repeated appropriately, e.g., body weight column had 63 for every row corresponding to frog I. \n", "\n", "I think this example of merging `DataFrame`s highlights the power of using them in your data analysis. Note also that there are plenty of options for how merges are done, and you should consult the [Pandas documentation](https://pandas.pydata.org/pandas-docs/stable/user_guide/merging.html#database-style-dataframe-or-named-series-joining-merging).\n", "\n", "This example also brings up an important point. When you have to perform operations on data frames, you can often \"brute force\" it with loops, etc. But if what you are trying to do seems like something a data analyst would frequently encounter, there is a good chance it's already built-in to Pandas, and you should ask Google how to do it." ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "## At long last, a plot!\n", "\n", "While the purpose of this part of the lesson was to learn how to concatenate and merge data frames, going through all of that wrangling effort would somehow be unsatisfying if we we didn't generate a plot. Let's compare the impact force on a per-mass basis for each frog." ] }, { "cell_type": "code", "execution_count": 21, "metadata": {}, "outputs": [ { "data": { "text/html": [ "\n", "\n", "\n", "\n", "\n", "\n", "
\n" ] }, "metadata": {}, "output_type": "display_data" }, { "data": { "application/javascript": [ "(function(root) {\n", " function embed_document(root) {\n", " \n", " var docs_json = {\"a97c334c-0eed-479d-a81a-b001d35eea56\":{\"roots\":{\"references\":[{\"attributes\":{\"below\":[{\"id\":\"1011\"}],\"center\":[{\"id\":\"1014\"},{\"id\":\"1017\"}],\"frame_height\":275,\"frame_width\":375,\"left\":[{\"id\":\"1015\"}],\"renderers\":[{\"id\":\"1038\"}],\"title\":{\"id\":\"1041\"},\"toolbar\":{\"id\":\"1025\"},\"x_range\":{\"id\":\"1004\"},\"x_scale\":{\"id\":\"1007\"},\"y_range\":{\"id\":\"1002\"},\"y_scale\":{\"id\":\"1009\"}},\"id\":\"1003\",\"subtype\":\"Figure\",\"type\":\"Plot\"},{\"attributes\":{\"axis_label\":\"impact force / body weight (mN/g)\",\"formatter\":{\"id\":\"1044\"},\"ticker\":{\"id\":\"1012\"}},\"id\":\"1011\",\"type\":\"LinearAxis\"},{\"attributes\":{\"data_source\":{\"id\":\"1033\"},\"glyph\":{\"id\":\"1036\"},\"hover_glyph\":null,\"muted_glyph\":null,\"nonselection_glyph\":{\"id\":\"1037\"},\"selection_glyph\":null,\"view\":{\"id\":\"1039\"}},\"id\":\"1038\",\"type\":\"GlyphRenderer\"},{\"attributes\":{\"source\":{\"id\":\"1033\"}},\"id\":\"1039\",\"type\":\"CDSView\"},{\"attributes\":{},\"id\":\"1016\",\"type\":\"CategoricalTicker\"},{\"attributes\":{},\"id\":\"1042\",\"type\":\"CategoricalTickFormatter\"},{\"attributes\":{\"factors\":[\"adult\",\"juvenile\"],\"palette\":[\"#1f77b3\",\"#ff7e0e\",\"#2ba02b\",\"#d62628\",\"#9367bc\",\"#8c564b\",\"#e277c1\",\"#7e7e7e\",\"#bcbc21\",\"#16bdcf\",\"#3a0182\",\"#004201\",\"#0fffa8\",\"#5d003f\",\"#bcbcff\",\"#d8afa1\",\"#b80080\",\"#004d52\",\"#6b6400\",\"#7c0100\",\"#6026ff\",\"#ffff9a\",\"#564964\",\"#8cb893\",\"#93fbff\",\"#018267\",\"#90ff00\",\"#8200a0\",\"#ac8944\",\"#5b3400\",\"#ffbff2\",\"#ff6e75\",\"#798cff\",\"#dd00ff\",\"#505646\",\"#004489\",\"#ffbf60\",\"#ff018c\",\"#bdc8cf\",\"#af97b5\",\"#b65600\",\"#017000\",\"#cd87ff\",\"#1cd646\",\"#bfebc3\",\"#7997b5\",\"#a56089\",\"#6e8956\",\"#bc7c75\",\"#8a2844\",\"#00acff\",\"#8ed4ff\",\"#4b6d77\",\"#00d4b1\",\"#9300f2\",\"#8a9500\",\"#5d5b9e\",\"#fddfba\",\"#00939e\",\"#ffdb00\",\"#00aa79\",\"#520067\",\"#000091\",\"#0a5d3d\",\"#a5e275\",\"#623b41\",\"#c6c689\",\"#ff9eb5\",\"#cd4f6b\",\"#ff07d6\",\"#8a3a05\",\"#7e3d70\",\"#ff4901\",\"#602ba5\",\"#1c00ff\",\"#e6dfff\",\"#aa3baf\",\"#d89c00\",\"#a3a39e\",\"#3f69ff\",\"#46490c\",\"#7b6985\",\"#6b978c\",\"#ff9a75\",\"#835bff\",\"#7c6b46\",\"#80b654\",\"#bc0049\",\"#fd93ff\",\"#5d0018\",\"#89d1d1\",\"#9c8cd3\",\"#da6d42\",\"#8a5700\",\"#3b5069\",\"#4b6b3b\",\"#edcfd8\",\"#cfedff\",\"#aa1500\",\"#dfff4f\",\"#ff2a56\",\"#d1499e\",\"#707cb8\",\"#598000\",\"#00e4fd\",\"#774b95\",\"#67d48c\",\"#3d3a72\",\"#ac413f\",\"#d6a166\",\"#c169cd\",\"#69595d\",\"#87aced\",\"#a0a569\",\"#d1aae6\",\"#870062\",\"#00fddb\",\"#672818\",\"#b342ff\",\"#0e59c4\",\"#168742\",\"#90d300\",\"#cd7900\",\"#f959ff\",\"#5b7466\",\"#8eaeb3\",\"#9c7c8c\",\"#4600c6\",\"#6b4d2d\",\"#a56d46\",\"#9e8972\",\"#a8afca\",\"#cd8ca7\",\"#00fd64\",\"#917900\",\"#ff62a1\",\"#f4ffd8\",\"#018cf0\",\"#13aca0\",\"#5b2d59\",\"#89859e\",\"#cfccba\",\"#d4afc4\",\"#dbdd6d\",\"#cffff4\",\"#006485\",\"#006962\",\"#a84167\",\"#2d97c4\",\"#a874ff\",\"#26ba5d\",\"#57b600\",\"#caffa7\",\"#a379aa\",\"#ffbc93\",\"#89e2c1\",\"#0fc8ff\",\"#d400c4\",\"#626d89\",\"#69858e\",\"#4b4d52\",\"#aa6067\",\"#79b5d4\",\"#2b5916\",\"#9a0024\",\"#bdd1f2\",\"#896e67\",\"#69a56b\",\"#855467\",\"#aecdba\",\"#87997e\",\"#cadb00\",\"#9a0390\",\"#ebbc1a\",\"#eb9cd1\",\"#70006e\",\"#b1a131\",\"#ca6b93\",\"#4146a3\",\"#e48c89\",\"#d44400\",\"#c68aca\",\"#b69597\",\"#d41f75\",\"#724bcc\",\"#674d00\",\"#672138\",\"#38564f\",\"#6ebaaa\",\"#853a31\",\"#a5d397\",\"#b8af8e\",\"#d8e4df\",\"#aa00df\",\"#cac1db\",\"#ffdf8c\",\"#e2524d\",\"#66696e\",\"#ff001c\",\"#522d72\",\"#4d906b\",\"#a86d11\",\"#ff9e26\",\"#5ea3af\",\"#c88556\",\"#915997\",\"#a3a1ff\",\"#fdbaba\",\"#242a87\",\"#dbe6a8\",\"#97f2a7\",\"#6793d6\",\"#ba5b3f\",\"#3a5d91\",\"#364f2f\",\"#267c95\",\"#89959a\",\"#cfb356\",\"#004664\",\"#5e5d2f\",\"#8e8e41\",\"#ac3f13\",\"#69953b\",\"#a13d85\",\"#bfb6ba\",\"#acc667\",\"#6469cf\",\"#91af00\",\"#2be2da\",\"#016e36\",\"#ff7952\",\"#42807e\",\"#4fe800\",\"#995428\",\"#5d0a00\",\"#a30057\",\"#0c8700\",\"#5982a7\",\"#ffebfb\",\"#4b6901\",\"#8775d4\",\"#e6c6ff\",\"#a5ffda\",\"#d86e77\",\"#df014b\",\"#69675b\",\"#776ba1\",\"#7e8067\",\"#594685\",\"#0000ca\",\"#7c002a\",\"#97ff72\",\"#b5e2e1\",\"#db52c8\",\"#777734\",\"#57bd8e\"]},\"id\":\"1032\",\"type\":\"CategoricalColorMapper\"},{\"attributes\":{\"active_drag\":\"auto\",\"active_inspect\":\"auto\",\"active_multi\":null,\"active_scroll\":\"auto\",\"active_tap\":\"auto\",\"tools\":[{\"id\":\"1018\"},{\"id\":\"1019\"},{\"id\":\"1020\"},{\"id\":\"1021\"},{\"id\":\"1022\"},{\"id\":\"1023\"}]},\"id\":\"1025\",\"type\":\"Toolbar\"},{\"attributes\":{\"bottom_units\":\"screen\",\"fill_alpha\":0.5,\"fill_color\":\"lightgrey\",\"left_units\":\"screen\",\"level\":\"overlay\",\"line_alpha\":1.0,\"line_color\":\"black\",\"line_dash\":[4,4],\"line_width\":2,\"right_units\":\"screen\",\"top_units\":\"screen\"},\"id\":\"1024\",\"type\":\"BoxAnnotation\"},{\"attributes\":{},\"id\":\"1004\",\"type\":\"DataRange1d\"},{\"attributes\":{},\"id\":\"1007\",\"type\":\"LinearScale\"},{\"attributes\":{},\"id\":\"1023\",\"type\":\"HelpTool\"},{\"attributes\":{},\"id\":\"1022\",\"type\":\"ResetTool\"},{\"attributes\":{\"axis\":{\"id\":\"1015\"},\"dimension\":1,\"grid_line_color\":null,\"ticker\":null},\"id\":\"1017\",\"type\":\"Grid\"},{\"attributes\":{\"data\":{\"ID\":[\"I\",\"I\",\"I\",\"I\",\"I\",\"I\",\"I\",\"I\",\"I\",\"I\",\"I\",\"I\",\"I\",\"I\",\"I\",\"I\",\"I\",\"I\",\"I\",\"I\",\"II\",\"II\",\"II\",\"II\",\"II\",\"II\",\"II\",\"II\",\"II\",\"II\",\"II\",\"II\",\"II\",\"II\",\"II\",\"II\",\"II\",\"II\",\"II\",\"II\",\"III\",\"III\",\"III\",\"III\",\"III\",\"III\",\"III\",\"III\",\"III\",\"III\",\"III\",\"III\",\"III\",\"III\",\"III\",\"III\",\"III\",\"III\",\"III\",\"III\",\"IV\",\"IV\",\"IV\",\"IV\",\"IV\",\"IV\",\"IV\",\"IV\",\"IV\",\"IV\",\"IV\",\"IV\",\"IV\",\"IV\",\"IV\",\"IV\",\"IV\",\"IV\",\"IV\",\"IV\"],\"__label\":[\"adult\",\"adult\",\"adult\",\"adult\",\"adult\",\"adult\",\"adult\",\"adult\",\"adult\",\"adult\",\"adult\",\"adult\",\"adult\",\"adult\",\"adult\",\"adult\",\"adult\",\"adult\",\"adult\",\"adult\",\"adult\",\"adult\",\"adult\",\"adult\",\"adult\",\"adult\",\"adult\",\"adult\",\"adult\",\"adult\",\"adult\",\"adult\",\"adult\",\"adult\",\"adult\",\"adult\",\"adult\",\"adult\",\"adult\",\"adult\",\"juvenile\",\"juvenile\",\"juvenile\",\"juvenile\",\"juvenile\",\"juvenile\",\"juvenile\",\"juvenile\",\"juvenile\",\"juvenile\",\"juvenile\",\"juvenile\",\"juvenile\",\"juvenile\",\"juvenile\",\"juvenile\",\"juvenile\",\"juvenile\",\"juvenile\",\"juvenile\",\"juvenile\",\"juvenile\",\"juvenile\",\"juvenile\",\"juvenile\",\"juvenile\",\"juvenile\",\"juvenile\",\"juvenile\",\"juvenile\",\"juvenile\",\"juvenile\",\"juvenile\",\"juvenile\",\"juvenile\",\"juvenile\",\"juvenile\",\"juvenile\",\"juvenile\",\"juvenile\"],\"age\":[\"adult\",\"adult\",\"adult\",\"adult\",\"adult\",\"adult\",\"adult\",\"adult\",\"adult\",\"adult\",\"adult\",\"adult\",\"adult\",\"adult\",\"adult\",\"adult\",\"adult\",\"adult\",\"adult\",\"adult\",\"adult\",\"adult\",\"adult\",\"adult\",\"adult\",\"adult\",\"adult\",\"adult\",\"adult\",\"adult\",\"adult\",\"adult\",\"adult\",\"adult\",\"adult\",\"adult\",\"adult\",\"adult\",\"adult\",\"adult\",\"juvenile\",\"juvenile\",\"juvenile\",\"juvenile\",\"juvenile\",\"juvenile\",\"juvenile\",\"juvenile\",\"juvenile\",\"juvenile\",\"juvenile\",\"juvenile\",\"juvenile\",\"juvenile\",\"juvenile\",\"juvenile\",\"juvenile\",\"juvenile\",\"juvenile\",\"juvenile\",\"juvenile\",\"juvenile\",\"juvenile\",\"juvenile\",\"juvenile\",\"juvenile\",\"juvenile\",\"juvenile\",\"juvenile\",\"juvenile\",\"juvenile\",\"juvenile\",\"juvenile\",\"juvenile\",\"juvenile\",\"juvenile\",\"juvenile\",\"juvenile\",\"juvenile\",\"juvenile\"],\"cat\":[\"I\",\"I\",\"I\",\"I\",\"I\",\"I\",\"I\",\"I\",\"I\",\"I\",\"I\",\"I\",\"I\",\"I\",\"I\",\"I\",\"I\",\"I\",\"I\",\"I\",\"II\",\"II\",\"II\",\"II\",\"II\",\"II\",\"II\",\"II\",\"II\",\"II\",\"II\",\"II\",\"II\",\"II\",\"II\",\"II\",\"II\",\"II\",\"II\",\"II\",\"III\",\"III\",\"III\",\"III\",\"III\",\"III\",\"III\",\"III\",\"III\",\"III\",\"III\",\"III\",\"III\",\"III\",\"III\",\"III\",\"III\",\"III\",\"III\",\"III\",\"IV\",\"IV\",\"IV\",\"IV\",\"IV\",\"IV\",\"IV\",\"IV\",\"IV\",\"IV\",\"IV\",\"IV\",\"IV\",\"IV\",\"IV\",\"IV\",\"IV\",\"IV\",\"IV\",\"IV\"],\"impact force / body weight\":[1.95,4.08,2.82,2.51,0.8,3.68,0.9,3.11,4.27,3.06,3.06,2.5,2.11,2.73,2.49,2.07,1.25,3.28,2.0,0.76,3.79,1.42,0.77,2.23,1.27,3.62,1.24,1.48,3.42,0.7,1.65,0.63,1.77,0.58,2.78,1.21,1.02,0.9,1.08,1.72,4.94,3.33,2.61,6.24,4.91,4.38,4.32,4.66,6.49,3.7,3.69,5.04,4.99,4.38,4.3,3.09,3.23,4.94,5.35,3.93,1.28,1.05,0.28,3.37,2.64,0.17,3.74,2.03,5.36,4.34,1.47,1.47,4.44,3.84,6.07,3.0,4.5,5.3,4.57,3.48]},\"selected\":{\"id\":\"1048\"},\"selection_policy\":{\"id\":\"1047\"}},\"id\":\"1033\",\"type\":\"ColumnDataSource\"},{\"attributes\":{},\"id\":\"1044\",\"type\":\"BasicTickFormatter\"},{\"attributes\":{},\"id\":\"1018\",\"type\":\"PanTool\"},{\"attributes\":{\"factors\":[\"IV\",\"III\",\"II\",\"I\"]},\"id\":\"1002\",\"type\":\"FactorRange\"},{\"attributes\":{\"overlay\":{\"id\":\"1024\"}},\"id\":\"1020\",\"type\":\"BoxZoomTool\"},{\"attributes\":{},\"id\":\"1019\",\"type\":\"WheelZoomTool\"},{\"attributes\":{\"axis\":{\"id\":\"1011\"},\"ticker\":null},\"id\":\"1014\",\"type\":\"Grid\"},{\"attributes\":{\"text\":\"\"},\"id\":\"1041\",\"type\":\"Title\"},{\"attributes\":{},\"id\":\"1009\",\"type\":\"CategoricalScale\"},{\"attributes\":{},\"id\":\"1047\",\"type\":\"UnionRenderers\"},{\"attributes\":{\"axis_label\":\"frog ID\",\"formatter\":{\"id\":\"1042\"},\"ticker\":{\"id\":\"1016\"}},\"id\":\"1015\",\"type\":\"CategoricalAxis\"},{\"attributes\":{},\"id\":\"1048\",\"type\":\"Selection\"},{\"attributes\":{\"fill_color\":{\"field\":\"age\",\"transform\":{\"id\":\"1032\"}},\"line_color\":{\"field\":\"age\",\"transform\":{\"id\":\"1032\"}},\"x\":{\"field\":\"impact force / body weight\"},\"y\":{\"field\":\"cat\",\"transform\":{\"id\":\"1034\"}}},\"id\":\"1036\",\"type\":\"Circle\"},{\"attributes\":{},\"id\":\"1012\",\"type\":\"BasicTicker\"},{\"attributes\":{},\"id\":\"1021\",\"type\":\"SaveTool\"},{\"attributes\":{\"distribution\":\"normal\",\"range\":{\"id\":\"1002\"},\"width\":0.1},\"id\":\"1034\",\"type\":\"Jitter\"},{\"attributes\":{\"fill_alpha\":{\"value\":0.1},\"fill_color\":{\"field\":\"age\",\"transform\":{\"id\":\"1032\"}},\"line_alpha\":{\"value\":0.1},\"line_color\":{\"field\":\"age\",\"transform\":{\"id\":\"1032\"}},\"x\":{\"field\":\"impact force / body weight\"},\"y\":{\"field\":\"cat\",\"transform\":{\"id\":\"1034\"}}},\"id\":\"1037\",\"type\":\"Circle\"}],\"root_ids\":[\"1003\"]},\"title\":\"Bokeh Application\",\"version\":\"2.2.1\"}};\n", " var render_items = [{\"docid\":\"a97c334c-0eed-479d-a81a-b001d35eea56\",\"root_ids\":[\"1003\"],\"roots\":{\"1003\":\"8e2a6d80-3f30-4625-a4b0-c858ad2b94d7\"}}];\n", " root.Bokeh.embed.embed_items_notebook(docs_json, render_items);\n", "\n", " }\n", " if (root.Bokeh !== undefined) {\n", " embed_document(root);\n", " } else {\n", " var attempts = 0;\n", " var timer = setInterval(function(root) {\n", " if (root.Bokeh !== undefined) {\n", " clearInterval(timer);\n", " embed_document(root);\n", " } else {\n", " attempts++;\n", " if (attempts > 100) {\n", " clearInterval(timer);\n", " console.log(\"Bokeh: ERROR: Unable to run BokehJS code because BokehJS library is missing\");\n", " }\n", " }\n", " }, 10, root)\n", " }\n", "})(window);" ], "application/vnd.bokehjs_exec.v0+json": "" }, "metadata": { "application/vnd.bokehjs_exec.v0+json": { "id": "1003" } }, "output_type": "display_data" } ], "source": [ "p = iqplot.strip(\n", " df,\n", " q=\"impact force / body weight\",\n", " cats=\"ID\",\n", " color_column=\"age\",\n", " jitter=True,\n", " x_axis_label=\"impact force / body weight (mN/g)\",\n", " y_axis_label=\"frog ID\"\n", ")\n", "\n", "bokeh.io.show(p)" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "Apparently Frog III consistently packs a powerful punch, er.... tongue." ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "## Computing environment" ] }, { "cell_type": "code", "execution_count": 22, "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "CPython 3.8.5\n", "IPython 7.18.1\n", "\n", "numpy 1.19.1\n", "pandas 1.1.3\n", "bokeh 2.2.1\n", "iqplot 0.1.6\n", "jupyterlab 2.2.6\n" ] } ], "source": [ "%load_ext watermark\n", "%watermark -v -p numpy,pandas,bokeh,iqplot,jupyterlab" ] } ], "metadata": { "kernelspec": { "display_name": "Python 3", "language": "python", "name": "python3" }, "language_info": { "codemirror_mode": { "name": "ipython", "version": 3 }, "file_extension": ".py", "mimetype": "text/x-python", "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", "version": "3.8.5" } }, "nbformat": 4, "nbformat_minor": 4 }