{ "cells": [ { "cell_type": "markdown", "metadata": {}, "source": [ "# Parallel bootstrap calculations\n", "\n", "[Dataset download](https://s3.amazonaws.com/bebi103.caltech.edu/data/singer_transcript_counts.csv)\n", "\n", "
" ] }, { "cell_type": "code", "execution_count": 1, "metadata": { "nbsphinx": "hidden", "tags": [] }, "outputs": [], "source": [ "#| code-fold: true\n", "\n", "# Colab setup ------------------\n", "import os, sys, subprocess\n", "if \"google.colab\" in sys.modules:\n", " cmd = \"pip install --upgrade polars iqplot bebi103 multiprocess watermark\"\n", " process = subprocess.Popen(cmd.split(), stdout=subprocess.PIPE, stderr=subprocess.PIPE)\n", " stdout, stderr = process.communicate()\n", " data_path = \"https://s3.amazonaws.com/bebi103.caltech.edu/data/\"\n", "else:\n", " data_path = \"../data/\"\n", "# ------------------------------" ] }, { "cell_type": "code", "execution_count": 2, "metadata": {}, "outputs": [ { "data": { "text/html": [ " \n", "
\n", " \n", " Loading BokehJS ...\n", "
\n" ] }, "metadata": {}, "output_type": "display_data" }, { "data": { "application/javascript": [ "'use strict';\n", "(function(root) {\n", " function now() {\n", " return new Date();\n", " }\n", "\n", " const force = true;\n", "\n", " if (typeof root._bokeh_onload_callbacks === \"undefined\" || force === true) {\n", " root._bokeh_onload_callbacks = [];\n", " root._bokeh_is_loading = undefined;\n", " }\n", "\n", "const JS_MIME_TYPE = 'application/javascript';\n", " const HTML_MIME_TYPE = 'text/html';\n", " const EXEC_MIME_TYPE = 'application/vnd.bokehjs_exec.v0+json';\n", " const CLASS_NAME = 'output_bokeh rendered_html';\n", "\n", " /**\n", " * Render data to the DOM node\n", " */\n", " function render(props, node) {\n", " const script = document.createElement(\"script\");\n", " node.appendChild(script);\n", " }\n", "\n", " /**\n", " * Handle when an output is cleared or removed\n", " */\n", " function handleClearOutput(event, handle) {\n", " function drop(id) {\n", " const view = Bokeh.index.get_by_id(id)\n", " if (view != null) {\n", " view.model.document.clear()\n", " Bokeh.index.delete(view)\n", " }\n", " }\n", "\n", " const cell = handle.cell;\n", "\n", " const id = cell.output_area._bokeh_element_id;\n", " const server_id = cell.output_area._bokeh_server_id;\n", "\n", " // Clean up Bokeh references\n", " if (id != null) {\n", " drop(id)\n", " }\n", "\n", " if (server_id !== undefined) {\n", " // Clean up Bokeh references\n", " const cmd_clean = \"from bokeh.io.state import curstate; print(curstate().uuid_to_server['\" + server_id + \"'].get_sessions()[0].document.roots[0]._id)\";\n", " cell.notebook.kernel.execute(cmd_clean, {\n", " iopub: {\n", " output: function(msg) {\n", " const id = msg.content.text.trim()\n", " drop(id)\n", " }\n", " }\n", " });\n", " // Destroy server and session\n", " const cmd_destroy = \"import bokeh.io.notebook as ion; ion.destroy_server('\" + server_id + \"')\";\n", " cell.notebook.kernel.execute(cmd_destroy);\n", " }\n", " }\n", "\n", " /**\n", " * Handle when a new output is added\n", " */\n", " function handleAddOutput(event, handle) {\n", " const output_area = handle.output_area;\n", " const output = handle.output;\n", "\n", " // limit handleAddOutput to display_data with EXEC_MIME_TYPE content only\n", " if ((output.output_type != \"display_data\") || (!Object.prototype.hasOwnProperty.call(output.data, EXEC_MIME_TYPE))) {\n", " return\n", " }\n", "\n", " const toinsert = output_area.element.find(\".\" + CLASS_NAME.split(' ')[0]);\n", "\n", " if (output.metadata[EXEC_MIME_TYPE][\"id\"] !== undefined) {\n", " toinsert[toinsert.length - 1].firstChild.textContent = output.data[JS_MIME_TYPE];\n", " // store reference to embed id on output_area\n", " output_area._bokeh_element_id = output.metadata[EXEC_MIME_TYPE][\"id\"];\n", " }\n", " if (output.metadata[EXEC_MIME_TYPE][\"server_id\"] !== undefined) {\n", " const bk_div = document.createElement(\"div\");\n", " bk_div.innerHTML = output.data[HTML_MIME_TYPE];\n", " const script_attrs = bk_div.children[0].attributes;\n", " for (let i = 0; i < script_attrs.length; i++) {\n", " toinsert[toinsert.length - 1].firstChild.setAttribute(script_attrs[i].name, script_attrs[i].value);\n", " toinsert[toinsert.length - 1].firstChild.textContent = bk_div.children[0].textContent\n", " }\n", " // store reference to server id on output_area\n", " output_area._bokeh_server_id = output.metadata[EXEC_MIME_TYPE][\"server_id\"];\n", " }\n", " }\n", "\n", " function register_renderer(events, OutputArea) {\n", "\n", " function append_mime(data, metadata, element) {\n", " // create a DOM node to render to\n", " const toinsert = this.create_output_subarea(\n", " metadata,\n", " CLASS_NAME,\n", " EXEC_MIME_TYPE\n", " );\n", " this.keyboard_manager.register_events(toinsert);\n", " // Render to node\n", " const props = {data: data, metadata: metadata[EXEC_MIME_TYPE]};\n", " render(props, toinsert[toinsert.length - 1]);\n", " element.append(toinsert);\n", " return toinsert\n", " }\n", "\n", " /* Handle when an output is cleared or removed */\n", " events.on('clear_output.CodeCell', handleClearOutput);\n", " events.on('delete.Cell', handleClearOutput);\n", "\n", " /* Handle when a new output is added */\n", " events.on('output_added.OutputArea', handleAddOutput);\n", "\n", " /**\n", " * Register the mime type and append_mime function with output_area\n", " */\n", " OutputArea.prototype.register_mime_type(EXEC_MIME_TYPE, append_mime, {\n", " /* Is output safe? */\n", " safe: true,\n", " /* Index of renderer in `output_area.display_order` */\n", " index: 0\n", " });\n", " }\n", "\n", " // register the mime type if in Jupyter Notebook environment and previously unregistered\n", " if (root.Jupyter !== undefined) {\n", " const events = require('base/js/events');\n", " const OutputArea = require('notebook/js/outputarea').OutputArea;\n", "\n", " if (OutputArea.prototype.mime_types().indexOf(EXEC_MIME_TYPE) == -1) {\n", " register_renderer(events, OutputArea);\n", " }\n", " }\n", " if (typeof (root._bokeh_timeout) === \"undefined\" || force === true) {\n", " root._bokeh_timeout = Date.now() + 5000;\n", " root._bokeh_failed_load = false;\n", " }\n", "\n", " const NB_LOAD_WARNING = {'data': {'text/html':\n", " \"
\\n\"+\n", " \"

\\n\"+\n", " \"BokehJS does not appear to have successfully loaded. If loading BokehJS from CDN, this \\n\"+\n", " \"may be due to a slow or bad network connection. Possible fixes:\\n\"+\n", " \"

\\n\"+\n", " \"\\n\"+\n", " \"\\n\"+\n", " \"from bokeh.resources import INLINE\\n\"+\n", " \"output_notebook(resources=INLINE)\\n\"+\n", " \"\\n\"+\n", " \"
\"}};\n", "\n", " function display_loaded(error = null) {\n", " const el = document.getElementById(\"d213cc83-820f-452a-b6b1-90245128e4e2\");\n", " if (el != null) {\n", " const html = (() => {\n", " if (typeof root.Bokeh === \"undefined\") {\n", " if (error == null) {\n", " return \"BokehJS is loading ...\";\n", " } else {\n", " return \"BokehJS failed to load.\";\n", " }\n", " } else {\n", " const prefix = `BokehJS ${root.Bokeh.version}`;\n", " if (error == null) {\n", " return `${prefix} successfully loaded.`;\n", " } else {\n", " return `${prefix} encountered errors while loading and may not function as expected.`;\n", " }\n", " }\n", " })();\n", " el.innerHTML = html;\n", "\n", " if (error != null) {\n", " const wrapper = document.createElement(\"div\");\n", " wrapper.style.overflow = \"auto\";\n", " wrapper.style.height = \"5em\";\n", " wrapper.style.resize = \"vertical\";\n", " const content = document.createElement(\"div\");\n", " content.style.fontFamily = \"monospace\";\n", " content.style.whiteSpace = \"pre-wrap\";\n", " content.style.backgroundColor = \"rgb(255, 221, 221)\";\n", " content.textContent = error.stack ?? error.toString();\n", " wrapper.append(content);\n", " el.append(wrapper);\n", " }\n", " } else if (Date.now() < root._bokeh_timeout) {\n", " setTimeout(() => display_loaded(error), 100);\n", " }\n", " }\n", "\n", " function run_callbacks() {\n", " try {\n", " root._bokeh_onload_callbacks.forEach(function(callback) {\n", " if (callback != null)\n", " callback();\n", " });\n", " } finally {\n", " delete root._bokeh_onload_callbacks\n", " }\n", " console.debug(\"Bokeh: all callbacks have finished\");\n", " }\n", "\n", " function load_libs(css_urls, js_urls, callback) {\n", " if (css_urls == null) css_urls = [];\n", " if (js_urls == null) js_urls = [];\n", "\n", " root._bokeh_onload_callbacks.push(callback);\n", " if (root._bokeh_is_loading > 0) {\n", " console.debug(\"Bokeh: BokehJS is being loaded, scheduling callback at\", now());\n", " return null;\n", " }\n", " if (js_urls == null || js_urls.length === 0) {\n", " run_callbacks();\n", " return null;\n", " }\n", " console.debug(\"Bokeh: BokehJS not loaded, scheduling load and callback at\", now());\n", " root._bokeh_is_loading = css_urls.length + js_urls.length;\n", "\n", " function on_load() {\n", " root._bokeh_is_loading--;\n", " if (root._bokeh_is_loading === 0) {\n", " console.debug(\"Bokeh: all BokehJS libraries/stylesheets loaded\");\n", " run_callbacks()\n", " }\n", " }\n", "\n", " function on_error(url) {\n", " console.error(\"failed to load \" + url);\n", " }\n", "\n", " for (let i = 0; i < css_urls.length; i++) {\n", " const url = css_urls[i];\n", " const element = document.createElement(\"link\");\n", " element.onload = on_load;\n", " element.onerror = on_error.bind(null, url);\n", " element.rel = \"stylesheet\";\n", " element.type = \"text/css\";\n", " element.href = url;\n", " console.debug(\"Bokeh: injecting link tag for BokehJS stylesheet: \", url);\n", " document.body.appendChild(element);\n", " }\n", "\n", " for (let i = 0; i < js_urls.length; i++) {\n", " const url = js_urls[i];\n", " const element = document.createElement('script');\n", " element.onload = on_load;\n", " element.onerror = on_error.bind(null, url);\n", " element.async = false;\n", " element.src = url;\n", " console.debug(\"Bokeh: injecting script tag for BokehJS library: \", url);\n", " document.head.appendChild(element);\n", " }\n", " };\n", "\n", " function inject_raw_css(css) {\n", " const element = document.createElement(\"style\");\n", " element.appendChild(document.createTextNode(css));\n", " document.body.appendChild(element);\n", " }\n", "\n", " const js_urls = [\"https://cdn.bokeh.org/bokeh/release/bokeh-3.7.3.min.js\", \"https://cdn.bokeh.org/bokeh/release/bokeh-gl-3.7.3.min.js\", \"https://cdn.bokeh.org/bokeh/release/bokeh-widgets-3.7.3.min.js\", \"https://cdn.bokeh.org/bokeh/release/bokeh-tables-3.7.3.min.js\", \"https://cdn.bokeh.org/bokeh/release/bokeh-mathjax-3.7.3.min.js\"];\n", " const css_urls = [];\n", "\n", " const inline_js = [ function(Bokeh) {\n", " Bokeh.set_log_level(\"info\");\n", " },\n", "function(Bokeh) {\n", " }\n", " ];\n", "\n", " function run_inline_js() {\n", " if (root.Bokeh !== undefined || force === true) {\n", " try {\n", " for (let i = 0; i < inline_js.length; i++) {\n", " inline_js[i].call(root, root.Bokeh);\n", " }\n", "\n", " } catch (error) {display_loaded(error);throw error;\n", " }if (force === true) {\n", " display_loaded();\n", " }} else if (Date.now() < root._bokeh_timeout) {\n", " setTimeout(run_inline_js, 100);\n", " } else if (!root._bokeh_failed_load) {\n", " console.log(\"Bokeh: BokehJS failed to load within specified timeout.\");\n", " root._bokeh_failed_load = true;\n", " } else if (force !== true) {\n", " const cell = $(document.getElementById(\"d213cc83-820f-452a-b6b1-90245128e4e2\")).parents('.cell').data().cell;\n", " cell.output_area.append_execute_result(NB_LOAD_WARNING)\n", " }\n", " }\n", "\n", " if (root._bokeh_is_loading === 0) {\n", " console.debug(\"Bokeh: BokehJS loaded, going straight to plotting\");\n", " run_inline_js();\n", " } else {\n", " load_libs(css_urls, js_urls, function() {\n", " console.debug(\"Bokeh: BokehJS plotting callback run at\", now());\n", " run_inline_js();\n", " });\n", " }\n", "}(window));" ], "application/vnd.bokehjs_load.v0+json": "'use strict';\n(function(root) {\n function now() {\n return new Date();\n }\n\n const force = true;\n\n if (typeof root._bokeh_onload_callbacks === \"undefined\" || force === true) {\n root._bokeh_onload_callbacks = [];\n root._bokeh_is_loading = undefined;\n }\n\n\n if (typeof (root._bokeh_timeout) === \"undefined\" || force === true) {\n root._bokeh_timeout = Date.now() + 5000;\n root._bokeh_failed_load = false;\n }\n\n const NB_LOAD_WARNING = {'data': {'text/html':\n \"
\\n\"+\n \"

\\n\"+\n \"BokehJS does not appear to have successfully loaded. If loading BokehJS from CDN, this \\n\"+\n \"may be due to a slow or bad network connection. Possible fixes:\\n\"+\n \"

\\n\"+\n \"\\n\"+\n \"\\n\"+\n \"from bokeh.resources import INLINE\\n\"+\n \"output_notebook(resources=INLINE)\\n\"+\n \"\\n\"+\n \"
\"}};\n\n function display_loaded(error = null) {\n const el = document.getElementById(\"d213cc83-820f-452a-b6b1-90245128e4e2\");\n if (el != null) {\n const html = (() => {\n if (typeof root.Bokeh === \"undefined\") {\n if (error == null) {\n return \"BokehJS is loading ...\";\n } else {\n return \"BokehJS failed to load.\";\n }\n } else {\n const prefix = `BokehJS ${root.Bokeh.version}`;\n if (error == null) {\n return `${prefix} successfully loaded.`;\n } else {\n return `${prefix} encountered errors while loading and may not function as expected.`;\n }\n }\n })();\n el.innerHTML = html;\n\n if (error != null) {\n const wrapper = document.createElement(\"div\");\n wrapper.style.overflow = \"auto\";\n wrapper.style.height = \"5em\";\n wrapper.style.resize = \"vertical\";\n const content = document.createElement(\"div\");\n content.style.fontFamily = \"monospace\";\n content.style.whiteSpace = \"pre-wrap\";\n content.style.backgroundColor = \"rgb(255, 221, 221)\";\n content.textContent = error.stack ?? error.toString();\n wrapper.append(content);\n el.append(wrapper);\n }\n } else if (Date.now() < root._bokeh_timeout) {\n setTimeout(() => display_loaded(error), 100);\n }\n }\n\n function run_callbacks() {\n try {\n root._bokeh_onload_callbacks.forEach(function(callback) {\n if (callback != null)\n callback();\n });\n } finally {\n delete root._bokeh_onload_callbacks\n }\n console.debug(\"Bokeh: all callbacks have finished\");\n }\n\n function load_libs(css_urls, js_urls, callback) {\n if (css_urls == null) css_urls = [];\n if (js_urls == null) js_urls = [];\n\n root._bokeh_onload_callbacks.push(callback);\n if (root._bokeh_is_loading > 0) {\n console.debug(\"Bokeh: BokehJS is being loaded, scheduling callback at\", now());\n return null;\n }\n if (js_urls == null || js_urls.length === 0) {\n run_callbacks();\n return null;\n }\n console.debug(\"Bokeh: BokehJS not loaded, scheduling load and callback at\", now());\n root._bokeh_is_loading = css_urls.length + js_urls.length;\n\n function on_load() {\n root._bokeh_is_loading--;\n if (root._bokeh_is_loading === 0) {\n console.debug(\"Bokeh: all BokehJS libraries/stylesheets loaded\");\n run_callbacks()\n }\n }\n\n function on_error(url) {\n console.error(\"failed to load \" + url);\n }\n\n for (let i = 0; i < css_urls.length; i++) {\n const url = css_urls[i];\n const element = document.createElement(\"link\");\n element.onload = on_load;\n element.onerror = on_error.bind(null, url);\n element.rel = \"stylesheet\";\n element.type = \"text/css\";\n element.href = url;\n console.debug(\"Bokeh: injecting link tag for BokehJS stylesheet: \", url);\n document.body.appendChild(element);\n }\n\n for (let i = 0; i < js_urls.length; i++) {\n const url = js_urls[i];\n const element = document.createElement('script');\n element.onload = on_load;\n element.onerror = on_error.bind(null, url);\n element.async = false;\n element.src = url;\n console.debug(\"Bokeh: injecting script tag for BokehJS library: \", url);\n document.head.appendChild(element);\n }\n };\n\n function inject_raw_css(css) {\n const element = document.createElement(\"style\");\n element.appendChild(document.createTextNode(css));\n document.body.appendChild(element);\n }\n\n const js_urls = [\"https://cdn.bokeh.org/bokeh/release/bokeh-3.7.3.min.js\", \"https://cdn.bokeh.org/bokeh/release/bokeh-gl-3.7.3.min.js\", \"https://cdn.bokeh.org/bokeh/release/bokeh-widgets-3.7.3.min.js\", \"https://cdn.bokeh.org/bokeh/release/bokeh-tables-3.7.3.min.js\", \"https://cdn.bokeh.org/bokeh/release/bokeh-mathjax-3.7.3.min.js\"];\n const css_urls = [];\n\n const inline_js = [ function(Bokeh) {\n Bokeh.set_log_level(\"info\");\n },\nfunction(Bokeh) {\n }\n ];\n\n function run_inline_js() {\n if (root.Bokeh !== undefined || force === true) {\n try {\n for (let i = 0; i < inline_js.length; i++) {\n inline_js[i].call(root, root.Bokeh);\n }\n\n } catch (error) {display_loaded(error);throw error;\n }if (force === true) {\n display_loaded();\n }} else if (Date.now() < root._bokeh_timeout) {\n setTimeout(run_inline_js, 100);\n } else if (!root._bokeh_failed_load) {\n console.log(\"Bokeh: BokehJS failed to load within specified timeout.\");\n root._bokeh_failed_load = true;\n } else if (force !== true) {\n const cell = $(document.getElementById(\"d213cc83-820f-452a-b6b1-90245128e4e2\")).parents('.cell').data().cell;\n cell.output_area.append_execute_result(NB_LOAD_WARNING)\n }\n }\n\n if (root._bokeh_is_loading === 0) {\n console.debug(\"Bokeh: BokehJS loaded, going straight to plotting\");\n run_inline_js();\n } else {\n load_libs(css_urls, js_urls, function() {\n console.debug(\"Bokeh: BokehJS plotting callback run at\", now());\n run_inline_js();\n });\n }\n}(window));" }, "metadata": {}, "output_type": "display_data" } ], "source": [ "try:\n", " import multiprocess\n", "except:\n", " import multiprocessing as multiprocess\n", "\n", "import warnings\n", "\n", "import numpy as np\n", "import polars as pl\n", "import scipy.optimize\n", "import scipy.stats as st\n", "\n", "import tqdm\n", "\n", "import bebi103\n", "\n", "import bokeh.io\n", "bokeh.io.output_notebook()" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "
\n", "\n", "In the last part of this lesson, we saw that computing the MLE confidence intervals can take some time (several minutes). To speed up calculations, we can compute the bootstrap replicates in parallel. That is, we split the sampling job up into parts, and different processing cores work on each part, and then combine the results together at the end. \n", "\n", "To begin, we load in the data." ] }, { "cell_type": "code", "execution_count": 3, "metadata": {}, "outputs": [], "source": [ "df = pl.read_csv(os.path.join(data_path, \"singer_transcript_counts.csv\"), comment_prefix=\"#\")" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "And we also need the functions we used for performing the MLE calculation from earlier in the lesson." ] }, { "cell_type": "code", "execution_count": 4, "metadata": {}, "outputs": [], "source": [ "def log_like_iid_nbinom(params, n):\n", " \"\"\"Log likelihood for i.i.d. NBinom measurements, parametrized\n", " by alpha, b=1/beta.\"\"\"\n", " alpha, b = params\n", " \n", " if alpha <= 0 or b <= 0:\n", " return -np.inf\n", "\n", " return np.sum(st.nbinom.logpmf(n, alpha, 1/(1+b)))\n", "\n", "\n", "def mle_iid_nbinom(n):\n", " \"\"\"Perform maximum likelihood estimates for parameters for i.i.d. \n", " NBinom measurements, parametrized by alpha, b=1/beta\"\"\"\n", "\n", " with warnings.catch_warnings():\n", " warnings.simplefilter(\"ignore\")\n", " res = scipy.optimize.minimize(\n", " fun=lambda params, n: -log_like_iid_nbinom(params, n),\n", " x0=np.array([3, 3]),\n", " args=(n,),\n", " method='Powell'\n", " )\n", "\n", " if res.success:\n", " return res.x\n", " else:\n", " raise RuntimeError('Convergence failed with message', res.message)" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "## Explicit RNGs\n", "\n", "When we parallelize the calculation, we have to be very careful about the random number generators we use. If we define a random number generator globally with `rng = np.random.default_rng()`, then each parallel process will use the same sequence of pseudorandom numbers. We therefore need to make a new RNG for each of the parallel processes. As a result, we need to explicitly supply a random number generator to the function we use to generate samples. This changes the call signature to include the RNG, but requires no other changes." ] }, { "cell_type": "code", "execution_count": 5, "metadata": {}, "outputs": [], "source": [ "def gen_nbinom(alpha, b, size, rng):\n", " return rng.negative_binomial(alpha, 1 / (1 + b), size=size)" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "Next, we need to adjust the `draw_bs_reps_mle()` function to also have explicit specification of a RNG. If we do not supply one, a new one should be created within the function.\n", "\n", "Finally, since the parallel version will supersede the `draw_bs_reps_mle()` function we have already written, we will prepend its name with an underscore, designating it as a private function (essentially a function an end user will not call), since we will not really call it directly again except for demonstration purposes." ] }, { "cell_type": "code", "execution_count": 6, "metadata": {}, "outputs": [], "source": [ "def _draw_parametric_bs_reps_mle(\n", " mle_fun, gen_fun, data, args=(), size=1, progress_bar=False, rng=None,\n", "):\n", " \"\"\"Draw parametric bootstrap replicates of maximum likelihood estimator.\n", " \n", " Parameters\n", " ----------\n", " mle_fun : function\n", " Function with call signature mle_fun(data, *args) that computes\n", " a MLE for the parameters\n", " gen_fun : function\n", " Function to randomly draw a new data set out of the model\n", " distribution parametrized by the MLE. Must have call\n", " signature `gen_fun(*params, size, *args, rng)`.\n", " data : one-dimemsional Numpy array\n", " Array of measurements\n", " args : tuple, default ()\n", " Arguments to be passed to `mle_fun()`.\n", " size : int, default 1\n", " Number of bootstrap replicates to draw.\n", " progress_bar : bool, default False\n", " Whether or not to display progress bar.\n", " rng : numpy.random.Generator instance, default None\n", " RNG to be used in bootstrapping. If None, the default\n", " Numpy RNG is used with a fresh seed based on the clock.\n", " \n", " Returns\n", " -------\n", " output : numpy array\n", " Bootstrap replicates of MLEs.\n", " \"\"\"\n", " if rng is None:\n", " rng = np.random.default_rng()\n", " \n", " params = mle_fun(data, *args)\n", "\n", " if progress_bar:\n", " iterator = tqdm.tqdm(range(size))\n", " else:\n", " iterator = range(size)\n", "\n", " return np.array(\n", " [mle_fun(gen_fun(*params, size=len(data), *args, rng=rng)) for _ in iterator]\n", " )" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "These functions still work as advertised. We can call `_draw_parametric_bs_reps_mle()` exactly as we have earlier in the lesson. To get 100 bootstrap replicates (only 100 so we don't have to wait too long), we can do the following for Nanog." ] }, { "cell_type": "code", "execution_count": 7, "metadata": {}, "outputs": [ { "name": "stderr", "output_type": "stream", "text": [ "100%|█████████████████████████████████████████████████████████| 100/100 [00:01<00:00, 99.94it/s]\n" ] } ], "source": [ "bs_reps = _draw_parametric_bs_reps_mle(\n", " mle_iid_nbinom, gen_nbinom, df[\"Nanog\"].to_numpy(), args=(), size=100, progress_bar=True\n", ")" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "## Enabling parallelization\n", "\n", "To enable multiple cores to draw bootstrap replicates, we could use Python's built-in [multiprocessing module](https://docs.python.org/3.11/library/multiprocessing.html). This module has [some problems running with Jupyter notebooks on some OSs](https://github.com/ipython/ipython/issues/12396), though. As a drop-in replacement, we could use Mike McKerns's [multiprocess package](https://github.com/uqfoundation/multiprocess), which offers improvements on the built-in multiprocessing module.\n", "\n", "To enable seamless switching between these two multiprocessing packages, I included the following import statement.\n", "\n", "```python\n", "try:\n", " import multiprocess\n", "except:\n", " import multiprocessing as multiprocess\n", "```\n", "\n", "We can therefore use them interchangably, calling whichever package we are using `multiprocess`. The syntax is as follows.\n", "\n", "```python\n", "with multiprocess.Pool(n_jobs) as pool:\n", " result = pool.starmap(fun, arg_iterable)\n", "```\n", "\n", "Here, `fun` is a function and `arg_iterable` is an iterable that produces arguments to the `fun` as tuples that will be splatted when passed to `fun`. This is best seen with a simple example before we get into the more complicated example of bootstrapping. We will choose a function that adds four arguments. Then, `arg_iterable` will be an iterable (in this case a list) with tuples, each of which has four elements." ] }, { "cell_type": "code", "execution_count": 8, "metadata": {}, "outputs": [ { "data": { "text/plain": [ "[(0, 1, 2, 3),\n", " (1, 2, 3, 4),\n", " (2, 3, 4, 5),\n", " (3, 4, 5, 6),\n", " (4, 5, 6, 7),\n", " (5, 6, 7, 8),\n", " (6, 7, 8, 9),\n", " (7, 8, 9, 10)]" ] }, "execution_count": 8, "metadata": {}, "output_type": "execute_result" } ], "source": [ "def fun(a, b, c, d):\n", " return a + b + c + d\n", "\n", "arg_iterable = [(i, i+1, i+2, i+3) for i in range(8)]\n", "\n", "# Display arg_iterable\n", "arg_iterable" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "If we call `fun` with one of the arguments from the `arg_iterable` using a splat operator, we get the appropriate sum." ] }, { "cell_type": "code", "execution_count": 9, "metadata": {}, "outputs": [ { "data": { "text/plain": [ "6" ] }, "execution_count": 9, "metadata": {}, "output_type": "execute_result" } ], "source": [ "fun(*arg_iterable[0])" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "To iterate through the arguments and evaluate them in `fun` each time, we can do it in parallel, for example using two cores." ] }, { "cell_type": "code", "execution_count": 10, "metadata": {}, "outputs": [ { "data": { "text/plain": [ "[6, 10, 14, 18, 22, 26, 30, 34]" ] }, "execution_count": 10, "metadata": {}, "output_type": "execute_result" } ], "source": [ "with multiprocess.Pool(2) as pool:\n", " result = pool.starmap(fun, arg_iterable)\n", " \n", "# Take a look\n", "result" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "This gives the same result as if we did the following." ] }, { "cell_type": "code", "execution_count": 11, "metadata": {}, "outputs": [ { "data": { "text/plain": [ "[6, 10, 14, 18, 22, 26, 30, 34]" ] }, "execution_count": 11, "metadata": {}, "output_type": "execute_result" } ], "source": [ "[fun(*args) for args in arg_iterable]" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "We can see the equivalence then.\n", "\n", "```python\n", "with multiprocess.Pool(2) as pool:\n", " result = pool.starmap(fun, arg_iterable)\n", "```\n", "\n", "is the same as\n", "\n", "```python\n", "result = [fun(*args) for args in arg_iterable]\n", "```\n", "\n", "We can use this same structure for our bootstrap replicates. Here, `fun` is `_draw_bs_reps_mle`, and we need to make a list of arguments to pass to that function. This means splitting up `size` into chunks of size `size // n_jobs`," ] }, { "cell_type": "code", "execution_count": 12, "metadata": {}, "outputs": [], "source": [ "def draw_parametric_bs_reps_mle(\n", " mle_fun, gen_fun, data, args=(), size=1, n_jobs=1, progress_bar=False\n", "):\n", " \"\"\"Draw nonparametric bootstrap replicates of maximum likelihood estimator.\n", " \n", " Parameters\n", " ----------\n", " mle_fun : function\n", " Function with call signature mle_fun(data, *args) that computes\n", " a MLE for the parameters\n", " gen_fun : function\n", " Function to randomly draw a new data set out of the model\n", " distribution parametrized by the MLE. Must have call\n", " signature `gen_fun(*params, size, *args, rg)`.\n", " data : one-dimemsional Numpy array\n", " Array of measurements\n", " args : tuple, default ()\n", " Arguments to be passed to `mle_fun()`.\n", " size : int, default 1\n", " Number of bootstrap replicates to draw.\n", " n_jobs : int, default 1\n", " Number of cores to use in drawing bootstrap replicates.\n", " progress_bar : bool, default False\n", " Whether or not to display progress bar.\n", " \n", " Returns\n", " -------\n", " output : numpy array\n", " Bootstrap replicates of MLEs.\n", " \"\"\"\n", " # Just call the original function if n_jobs is 1 (no parallelization)\n", " if n_jobs == 1:\n", " return _draw_parametric_bs_reps_mle(\n", " mle_fun, gen_fun, data, args=args, size=size, progress_bar=progress_bar\n", " )\n", "\n", " # Set up sizes of bootstrap replicates for each core, making sure we\n", " # get all of them, even if sizes % n_jobs != 0\n", " sizes = [size // n_jobs for _ in range(n_jobs)]\n", " sizes[-1] += size - sum(sizes)\n", "\n", " # Build arguments\n", " arg_iterable = [(mle_fun, gen_fun, data, args, s, progress_bar, None) for s in sizes]\n", "\n", " with multiprocess.Pool(n_jobs) as pool:\n", " result = pool.starmap(_draw_parametric_bs_reps_mle, arg_iterable)\n", "\n", " return np.concatenate(result)" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "Now, we can specify `n_jobs` to be greater than one to take advantage of multiple cores. It is important to know how many cores you have on your machine. I usually keep one or two cores open for other processes on my computer so it doesn't die on me. You can check how many cores you have using the `multiprocess.cpu_count()` function." ] }, { "cell_type": "code", "execution_count": 13, "metadata": {}, "outputs": [ { "data": { "text/plain": [ "8" ] }, "execution_count": 13, "metadata": {}, "output_type": "execute_result" } ], "source": [ "multiprocess.cpu_count()" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "My computer has eight cores; I will therefore use seven cores. And now that I am unleashing nine cores on the problem, I will take 10,000 replicates again, and this time it will only take about 1/7 as long as it did the first time I did the calculation." ] }, { "cell_type": "code", "execution_count": 14, "metadata": {}, "outputs": [ { "name": "stderr", "output_type": "stream", "text": [ "100%|███████████████████████████████████████████████████████| 1428/1428 [00:20<00:00, 69.00it/s]\n", "100%|███████████████████████████████████████████████████████| 1428/1428 [00:20<00:00, 68.91it/s]\n", "100%|███████████████████████████████████████████████████████| 1428/1428 [00:20<00:00, 68.75it/s]\n", "100%|███████████████████████████████████████████████████████| 1428/1428 [00:20<00:00, 68.63it/s]\n", "100%|███████████████████████████████████████████████████████| 1428/1428 [00:20<00:00, 68.50it/s]\n", "100%|███████████████████████████████████████████████████████| 1432/1432 [00:20<00:00, 68.46it/s]\n", "100%|███████████████████████████████████████████████████████| 1428/1428 [00:20<00:00, 68.09it/s]\n" ] } ], "source": [ "bs_reps = draw_parametric_bs_reps_mle(\n", " mle_iid_nbinom,\n", " gen_nbinom,\n", " df[\"Nanog\"].to_numpy(),\n", " args=(),\n", " size=10000,\n", " n_jobs=7,\n", " progress_bar=True,\n", ")" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "From these replicates, we can compute the confidence intervals for $\\alpha$ and $b$ for Nanog." ] }, { "cell_type": "code", "execution_count": 15, "metadata": {}, "outputs": [ { "data": { "text/plain": [ "array([[ 1.09689208, 56.59336337],\n", " [ 1.49451796, 82.48374033]])" ] }, "execution_count": 15, "metadata": {}, "output_type": "execute_result" } ], "source": [ "np.percentile(bs_reps, [2.5, 97.5], axis=0)" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "## Parameter estimation for all genes\n", "\n", "We are now empowered to compute the MLE and confidence intervals for all four genes. We will do so with parametric bootstrap. We can ignore the warnings, as they occur when the solver attempts to try illegal (negative) parameter values. Powell's method is immune to this issue." ] }, { "cell_type": "code", "execution_count": 16, "metadata": {}, "outputs": [ { "name": "stderr", "output_type": "stream", "text": [ "100%|█████████████████████████████████████████████████████████████| 4/4 [01:35<00:00, 23.88s/it]\n" ] } ], "source": [ "genes = [\"Nanog\", \"Prdm14\", \"Rest\", \"Rex1\"]\n", "\n", "# Data frame to store results\n", "df_mle = pl.DataFrame(\n", " schema=[\n", " (\"gene\", str),\n", " (\"parameter\", str),\n", " (\"mle\", float),\n", " (\"conf_int_low\", float),\n", " (\"conf_int_high\", float),\n", " ]\n", ")\n", "\n", "# Perform MLE for each gene\n", "for gene in tqdm.tqdm(genes):\n", " mle = mle_iid_nbinom(df[gene].to_numpy())\n", "\n", " bs_reps = draw_parametric_bs_reps_mle(\n", " mle_iid_nbinom,\n", " gen_nbinom,\n", " df[gene].to_numpy(),\n", " args=(),\n", " size=10000,\n", " n_jobs=7,\n", " progress_bar=False,\n", " )\n", " conf_int = np.percentile(bs_reps, [2.5, 97.5], axis=0)\n", "\n", " sub_df = pl.DataFrame(\n", " {\n", " \"gene\": [gene] * 2,\n", " \"parameter\": [\"alpha\", \"b\"],\n", " \"mle\": mle,\n", " \"conf_int_low\": conf_int[0, :],\n", " \"conf_int_high\": conf_int[1, :],\n", " }\n", " )\n", " df_mle = pl.concat([df_mle, sub_df])" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "We now have a data frame with the results from our statistical inference. Let's take a look." ] }, { "cell_type": "code", "execution_count": 17, "metadata": {}, "outputs": [ { "data": { "text/html": [ "
\n", "shape: (8, 5)
geneparametermleconf_int_lowconf_int_high
strstrf64f64f64
"Nanog""alpha"1.2630971.0911141.487293
"Nanog""b"69.34784257.07288782.716729
"Prdm14""alpha"0.5528860.4534440.685822
"Prdm14""b"8.2006366.1559910.544581
"Rest""alpha"4.5303353.866495.457839
"Rest""b"16.54305413.58482519.454817
"Rex1""alpha"1.6345621.4135691.933929
"Rex1""b"84.68091569.805029100.023021
" ], "text/plain": [ "shape: (8, 5)\n", "┌────────┬───────────┬───────────┬──────────────┬───────────────┐\n", "│ gene ┆ parameter ┆ mle ┆ conf_int_low ┆ conf_int_high │\n", "│ --- ┆ --- ┆ --- ┆ --- ┆ --- │\n", "│ str ┆ str ┆ f64 ┆ f64 ┆ f64 │\n", "╞════════╪═══════════╪═══════════╪══════════════╪═══════════════╡\n", "│ Nanog ┆ alpha ┆ 1.263097 ┆ 1.091114 ┆ 1.487293 │\n", "│ Nanog ┆ b ┆ 69.347842 ┆ 57.072887 ┆ 82.716729 │\n", "│ Prdm14 ┆ alpha ┆ 0.552886 ┆ 0.453444 ┆ 0.685822 │\n", "│ Prdm14 ┆ b ┆ 8.200636 ┆ 6.15599 ┆ 10.544581 │\n", "│ Rest ┆ alpha ┆ 4.530335 ┆ 3.86649 ┆ 5.457839 │\n", "│ Rest ┆ b ┆ 16.543054 ┆ 13.584825 ┆ 19.454817 │\n", "│ Rex1 ┆ alpha ┆ 1.634562 ┆ 1.413569 ┆ 1.933929 │\n", "│ Rex1 ┆ b ┆ 84.680915 ┆ 69.805029 ┆ 100.023021 │\n", "└────────┴───────────┴───────────┴──────────────┴───────────────┘" ] }, "execution_count": 17, "metadata": {}, "output_type": "execute_result" } ], "source": [ "df_mle" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "Finally, we can make a plot of the MLEs of the parameter values with confidence interbals." ] }, { "cell_type": "code", "execution_count": 18, "metadata": {}, "outputs": [ { "data": { "text/html": [ "\n", "
\n" ] }, "metadata": {}, "output_type": "display_data" }, { "data": { "application/javascript": [ "(function(root) {\n", " function embed_document(root) {\n", " const docs_json = {\"0135de28-9a55-4649-88bd-91bf3d26eacf\":{\"version\":\"3.7.3\",\"title\":\"Bokeh Application\",\"roots\":[{\"type\":\"object\",\"name\":\"Column\",\"id\":\"p1218\",\"attributes\":{\"children\":[{\"type\":\"object\",\"name\":\"Figure\",\"id\":\"p1003\",\"attributes\":{\"x_range\":{\"type\":\"object\",\"name\":\"DataRange1d\",\"id\":\"p1005\"},\"y_range\":{\"type\":\"object\",\"name\":\"FactorRange\",\"id\":\"p1012\",\"attributes\":{\"factors\":[\"Rex1\",\"Rest\",\"Prdm14\",\"Nanog\"]}},\"x_scale\":{\"type\":\"object\",\"name\":\"LinearScale\",\"id\":\"p1013\"},\"y_scale\":{\"type\":\"object\",\"name\":\"CategoricalScale\",\"id\":\"p1014\"},\"title\":{\"type\":\"object\",\"name\":\"Title\",\"id\":\"p1010\"},\"renderers\":[{\"type\":\"object\",\"name\":\"GlyphRenderer\",\"id\":\"p1044\",\"attributes\":{\"data_source\":{\"type\":\"object\",\"name\":\"ColumnDataSource\",\"id\":\"p1038\",\"attributes\":{\"selected\":{\"type\":\"object\",\"name\":\"Selection\",\"id\":\"p1039\",\"attributes\":{\"indices\":[],\"line_indices\":[]}},\"selection_policy\":{\"type\":\"object\",\"name\":\"UnionRenderers\",\"id\":\"p1040\"},\"data\":{\"type\":\"map\",\"entries\":[[\"x\",[1.2630973722808048]],[\"y\",[\"Nanog\"]]]}}},\"view\":{\"type\":\"object\",\"name\":\"CDSView\",\"id\":\"p1045\",\"attributes\":{\"filter\":{\"type\":\"object\",\"name\":\"AllIndices\",\"id\":\"p1046\"}}},\"glyph\":{\"type\":\"object\",\"name\":\"Scatter\",\"id\":\"p1041\",\"attributes\":{\"x\":{\"type\":\"field\",\"field\":\"x\"},\"y\":{\"type\":\"field\",\"field\":\"y\"},\"size\":{\"type\":\"value\",\"value\":5},\"line_color\":{\"type\":\"value\",\"value\":\"#1f77b4\"},\"fill_color\":{\"type\":\"value\",\"value\":\"#1f77b4\"}}},\"nonselection_glyph\":{\"type\":\"object\",\"name\":\"Scatter\",\"id\":\"p1042\",\"attributes\":{\"x\":{\"type\":\"field\",\"field\":\"x\"},\"y\":{\"type\":\"field\",\"field\":\"y\"},\"size\":{\"type\":\"value\",\"value\":5},\"line_color\":{\"type\":\"value\",\"value\":\"#1f77b4\"},\"line_alpha\":{\"type\":\"value\",\"value\":0.1},\"fill_color\":{\"type\":\"value\",\"value\":\"#1f77b4\"},\"fill_alpha\":{\"type\":\"value\",\"value\":0.1},\"hatch_alpha\":{\"type\":\"value\",\"value\":0.1}}},\"muted_glyph\":{\"type\":\"object\",\"name\":\"Scatter\",\"id\":\"p1043\",\"attributes\":{\"x\":{\"type\":\"field\",\"field\":\"x\"},\"y\":{\"type\":\"field\",\"field\":\"y\"},\"size\":{\"type\":\"value\",\"value\":5},\"line_color\":{\"type\":\"value\",\"value\":\"#1f77b4\"},\"line_alpha\":{\"type\":\"value\",\"value\":0.2},\"fill_color\":{\"type\":\"value\",\"value\":\"#1f77b4\"},\"fill_alpha\":{\"type\":\"value\",\"value\":0.2},\"hatch_alpha\":{\"type\":\"value\",\"value\":0.2}}}}},{\"type\":\"object\",\"name\":\"GlyphRenderer\",\"id\":\"p1053\",\"attributes\":{\"data_source\":{\"type\":\"object\",\"name\":\"ColumnDataSource\",\"id\":\"p1047\",\"attributes\":{\"selected\":{\"type\":\"object\",\"name\":\"Selection\",\"id\":\"p1048\",\"attributes\":{\"indices\":[],\"line_indices\":[]}},\"selection_policy\":{\"type\":\"object\",\"name\":\"UnionRenderers\",\"id\":\"p1049\"},\"data\":{\"type\":\"map\",\"entries\":[[\"x\",{\"type\":\"ndarray\",\"array\":{\"type\":\"bytes\",\"data\":\"cN+J9zN18T/Wc/Jb88v3Pw==\"},\"shape\":[2],\"dtype\":\"float64\",\"order\":\"little\"}],[\"y\",[\"Nanog\",\"Nanog\"]]]}}},\"view\":{\"type\":\"object\",\"name\":\"CDSView\",\"id\":\"p1054\",\"attributes\":{\"filter\":{\"type\":\"object\",\"name\":\"AllIndices\",\"id\":\"p1055\"}}},\"glyph\":{\"type\":\"object\",\"name\":\"Line\",\"id\":\"p1050\",\"attributes\":{\"x\":{\"type\":\"field\",\"field\":\"x\"},\"y\":{\"type\":\"field\",\"field\":\"y\"},\"line_color\":\"#1f77b4\",\"line_width\":3}},\"nonselection_glyph\":{\"type\":\"object\",\"name\":\"Line\",\"id\":\"p1051\",\"attributes\":{\"x\":{\"type\":\"field\",\"field\":\"x\"},\"y\":{\"type\":\"field\",\"field\":\"y\"},\"line_color\":\"#1f77b4\",\"line_alpha\":0.1,\"line_width\":3}},\"muted_glyph\":{\"type\":\"object\",\"name\":\"Line\",\"id\":\"p1052\",\"attributes\":{\"x\":{\"type\":\"field\",\"field\":\"x\"},\"y\":{\"type\":\"field\",\"field\":\"y\"},\"line_color\":\"#1f77b4\",\"line_alpha\":0.2,\"line_width\":3}}}},{\"type\":\"object\",\"name\":\"GlyphRenderer\",\"id\":\"p1062\",\"attributes\":{\"data_source\":{\"type\":\"object\",\"name\":\"ColumnDataSource\",\"id\":\"p1056\",\"attributes\":{\"selected\":{\"type\":\"object\",\"name\":\"Selection\",\"id\":\"p1057\",\"attributes\":{\"indices\":[],\"line_indices\":[]}},\"selection_policy\":{\"type\":\"object\",\"name\":\"UnionRenderers\",\"id\":\"p1058\"},\"data\":{\"type\":\"map\",\"entries\":[[\"x\",[0.5528859648100356]],[\"y\",[\"Prdm14\"]]]}}},\"view\":{\"type\":\"object\",\"name\":\"CDSView\",\"id\":\"p1063\",\"attributes\":{\"filter\":{\"type\":\"object\",\"name\":\"AllIndices\",\"id\":\"p1064\"}}},\"glyph\":{\"type\":\"object\",\"name\":\"Scatter\",\"id\":\"p1059\",\"attributes\":{\"x\":{\"type\":\"field\",\"field\":\"x\"},\"y\":{\"type\":\"field\",\"field\":\"y\"},\"size\":{\"type\":\"value\",\"value\":5},\"line_color\":{\"type\":\"value\",\"value\":\"#1f77b4\"},\"fill_color\":{\"type\":\"value\",\"value\":\"#1f77b4\"}}},\"nonselection_glyph\":{\"type\":\"object\",\"name\":\"Scatter\",\"id\":\"p1060\",\"attributes\":{\"x\":{\"type\":\"field\",\"field\":\"x\"},\"y\":{\"type\":\"field\",\"field\":\"y\"},\"size\":{\"type\":\"value\",\"value\":5},\"line_color\":{\"type\":\"value\",\"value\":\"#1f77b4\"},\"line_alpha\":{\"type\":\"value\",\"value\":0.1},\"fill_color\":{\"type\":\"value\",\"value\":\"#1f77b4\"},\"fill_alpha\":{\"type\":\"value\",\"value\":0.1},\"hatch_alpha\":{\"type\":\"value\",\"value\":0.1}}},\"muted_glyph\":{\"type\":\"object\",\"name\":\"Scatter\",\"id\":\"p1061\",\"attributes\":{\"x\":{\"type\":\"field\",\"field\":\"x\"},\"y\":{\"type\":\"field\",\"field\":\"y\"},\"size\":{\"type\":\"value\",\"value\":5},\"line_color\":{\"type\":\"value\",\"value\":\"#1f77b4\"},\"line_alpha\":{\"type\":\"value\",\"value\":0.2},\"fill_color\":{\"type\":\"value\",\"value\":\"#1f77b4\"},\"fill_alpha\":{\"type\":\"value\",\"value\":0.2},\"hatch_alpha\":{\"type\":\"value\",\"value\":0.2}}}}},{\"type\":\"object\",\"name\":\"GlyphRenderer\",\"id\":\"p1071\",\"attributes\":{\"data_source\":{\"type\":\"object\",\"name\":\"ColumnDataSource\",\"id\":\"p1065\",\"attributes\":{\"selected\":{\"type\":\"object\",\"name\":\"Selection\",\"id\":\"p1066\",\"attributes\":{\"indices\":[],\"line_indices\":[]}},\"selection_policy\":{\"type\":\"object\",\"name\":\"UnionRenderers\",\"id\":\"p1067\"},\"data\":{\"type\":\"map\",\"entries\":[[\"x\",{\"type\":\"ndarray\",\"array\":{\"type\":\"bytes\",\"data\":\"F5hDCjkF3T8EKclRQPLlPw==\"},\"shape\":[2],\"dtype\":\"float64\",\"order\":\"little\"}],[\"y\",[\"Prdm14\",\"Prdm14\"]]]}}},\"view\":{\"type\":\"object\",\"name\":\"CDSView\",\"id\":\"p1072\",\"attributes\":{\"filter\":{\"type\":\"object\",\"name\":\"AllIndices\",\"id\":\"p1073\"}}},\"glyph\":{\"type\":\"object\",\"name\":\"Line\",\"id\":\"p1068\",\"attributes\":{\"x\":{\"type\":\"field\",\"field\":\"x\"},\"y\":{\"type\":\"field\",\"field\":\"y\"},\"line_color\":\"#1f77b4\",\"line_width\":3}},\"nonselection_glyph\":{\"type\":\"object\",\"name\":\"Line\",\"id\":\"p1069\",\"attributes\":{\"x\":{\"type\":\"field\",\"field\":\"x\"},\"y\":{\"type\":\"field\",\"field\":\"y\"},\"line_color\":\"#1f77b4\",\"line_alpha\":0.1,\"line_width\":3}},\"muted_glyph\":{\"type\":\"object\",\"name\":\"Line\",\"id\":\"p1070\",\"attributes\":{\"x\":{\"type\":\"field\",\"field\":\"x\"},\"y\":{\"type\":\"field\",\"field\":\"y\"},\"line_color\":\"#1f77b4\",\"line_alpha\":0.2,\"line_width\":3}}}},{\"type\":\"object\",\"name\":\"GlyphRenderer\",\"id\":\"p1080\",\"attributes\":{\"data_source\":{\"type\":\"object\",\"name\":\"ColumnDataSource\",\"id\":\"p1074\",\"attributes\":{\"selected\":{\"type\":\"object\",\"name\":\"Selection\",\"id\":\"p1075\",\"attributes\":{\"indices\":[],\"line_indices\":[]}},\"selection_policy\":{\"type\":\"object\",\"name\":\"UnionRenderers\",\"id\":\"p1076\"},\"data\":{\"type\":\"map\",\"entries\":[[\"x\",[4.530334885171684]],[\"y\",[\"Rest\"]]]}}},\"view\":{\"type\":\"object\",\"name\":\"CDSView\",\"id\":\"p1081\",\"attributes\":{\"filter\":{\"type\":\"object\",\"name\":\"AllIndices\",\"id\":\"p1082\"}}},\"glyph\":{\"type\":\"object\",\"name\":\"Scatter\",\"id\":\"p1077\",\"attributes\":{\"x\":{\"type\":\"field\",\"field\":\"x\"},\"y\":{\"type\":\"field\",\"field\":\"y\"},\"size\":{\"type\":\"value\",\"value\":5},\"line_color\":{\"type\":\"value\",\"value\":\"#1f77b4\"},\"fill_color\":{\"type\":\"value\",\"value\":\"#1f77b4\"}}},\"nonselection_glyph\":{\"type\":\"object\",\"name\":\"Scatter\",\"id\":\"p1078\",\"attributes\":{\"x\":{\"type\":\"field\",\"field\":\"x\"},\"y\":{\"type\":\"field\",\"field\":\"y\"},\"size\":{\"type\":\"value\",\"value\":5},\"line_color\":{\"type\":\"value\",\"value\":\"#1f77b4\"},\"line_alpha\":{\"type\":\"value\",\"value\":0.1},\"fill_color\":{\"type\":\"value\",\"value\":\"#1f77b4\"},\"fill_alpha\":{\"type\":\"value\",\"value\":0.1},\"hatch_alpha\":{\"type\":\"value\",\"value\":0.1}}},\"muted_glyph\":{\"type\":\"object\",\"name\":\"Scatter\",\"id\":\"p1079\",\"attributes\":{\"x\":{\"type\":\"field\",\"field\":\"x\"},\"y\":{\"type\":\"field\",\"field\":\"y\"},\"size\":{\"type\":\"value\",\"value\":5},\"line_color\":{\"type\":\"value\",\"value\":\"#1f77b4\"},\"line_alpha\":{\"type\":\"value\",\"value\":0.2},\"fill_color\":{\"type\":\"value\",\"value\":\"#1f77b4\"},\"fill_alpha\":{\"type\":\"value\",\"value\":0.2},\"hatch_alpha\":{\"type\":\"value\",\"value\":0.2}}}}},{\"type\":\"object\",\"name\":\"GlyphRenderer\",\"id\":\"p1089\",\"attributes\":{\"data_source\":{\"type\":\"object\",\"name\":\"ColumnDataSource\",\"id\":\"p1083\",\"attributes\":{\"selected\":{\"type\":\"object\",\"name\":\"Selection\",\"id\":\"p1084\",\"attributes\":{\"indices\":[],\"line_indices\":[]}},\"selection_policy\":{\"type\":\"object\",\"name\":\"UnionRenderers\",\"id\":\"p1085\"},\"data\":{\"type\":\"map\",\"entries\":[[\"x\",{\"type\":\"ndarray\",\"array\":{\"type\":\"bytes\",\"data\":\"0IRuSpLuDkBLi4er09QVQA==\"},\"shape\":[2],\"dtype\":\"float64\",\"order\":\"little\"}],[\"y\",[\"Rest\",\"Rest\"]]]}}},\"view\":{\"type\":\"object\",\"name\":\"CDSView\",\"id\":\"p1090\",\"attributes\":{\"filter\":{\"type\":\"object\",\"name\":\"AllIndices\",\"id\":\"p1091\"}}},\"glyph\":{\"type\":\"object\",\"name\":\"Line\",\"id\":\"p1086\",\"attributes\":{\"x\":{\"type\":\"field\",\"field\":\"x\"},\"y\":{\"type\":\"field\",\"field\":\"y\"},\"line_color\":\"#1f77b4\",\"line_width\":3}},\"nonselection_glyph\":{\"type\":\"object\",\"name\":\"Line\",\"id\":\"p1087\",\"attributes\":{\"x\":{\"type\":\"field\",\"field\":\"x\"},\"y\":{\"type\":\"field\",\"field\":\"y\"},\"line_color\":\"#1f77b4\",\"line_alpha\":0.1,\"line_width\":3}},\"muted_glyph\":{\"type\":\"object\",\"name\":\"Line\",\"id\":\"p1088\",\"attributes\":{\"x\":{\"type\":\"field\",\"field\":\"x\"},\"y\":{\"type\":\"field\",\"field\":\"y\"},\"line_color\":\"#1f77b4\",\"line_alpha\":0.2,\"line_width\":3}}}},{\"type\":\"object\",\"name\":\"GlyphRenderer\",\"id\":\"p1098\",\"attributes\":{\"data_source\":{\"type\":\"object\",\"name\":\"ColumnDataSource\",\"id\":\"p1092\",\"attributes\":{\"selected\":{\"type\":\"object\",\"name\":\"Selection\",\"id\":\"p1093\",\"attributes\":{\"indices\":[],\"line_indices\":[]}},\"selection_policy\":{\"type\":\"object\",\"name\":\"UnionRenderers\",\"id\":\"p1094\"},\"data\":{\"type\":\"map\",\"entries\":[[\"x\",[1.6345624303464514]],[\"y\",[\"Rex1\"]]]}}},\"view\":{\"type\":\"object\",\"name\":\"CDSView\",\"id\":\"p1099\",\"attributes\":{\"filter\":{\"type\":\"object\",\"name\":\"AllIndices\",\"id\":\"p1100\"}}},\"glyph\":{\"type\":\"object\",\"name\":\"Scatter\",\"id\":\"p1095\",\"attributes\":{\"x\":{\"type\":\"field\",\"field\":\"x\"},\"y\":{\"type\":\"field\",\"field\":\"y\"},\"size\":{\"type\":\"value\",\"value\":5},\"line_color\":{\"type\":\"value\",\"value\":\"#1f77b4\"},\"fill_color\":{\"type\":\"value\",\"value\":\"#1f77b4\"}}},\"nonselection_glyph\":{\"type\":\"object\",\"name\":\"Scatter\",\"id\":\"p1096\",\"attributes\":{\"x\":{\"type\":\"field\",\"field\":\"x\"},\"y\":{\"type\":\"field\",\"field\":\"y\"},\"size\":{\"type\":\"value\",\"value\":5},\"line_color\":{\"type\":\"value\",\"value\":\"#1f77b4\"},\"line_alpha\":{\"type\":\"value\",\"value\":0.1},\"fill_color\":{\"type\":\"value\",\"value\":\"#1f77b4\"},\"fill_alpha\":{\"type\":\"value\",\"value\":0.1},\"hatch_alpha\":{\"type\":\"value\",\"value\":0.1}}},\"muted_glyph\":{\"type\":\"object\",\"name\":\"Scatter\",\"id\":\"p1097\",\"attributes\":{\"x\":{\"type\":\"field\",\"field\":\"x\"},\"y\":{\"type\":\"field\",\"field\":\"y\"},\"size\":{\"type\":\"value\",\"value\":5},\"line_color\":{\"type\":\"value\",\"value\":\"#1f77b4\"},\"line_alpha\":{\"type\":\"value\",\"value\":0.2},\"fill_color\":{\"type\":\"value\",\"value\":\"#1f77b4\"},\"fill_alpha\":{\"type\":\"value\",\"value\":0.2},\"hatch_alpha\":{\"type\":\"value\",\"value\":0.2}}}}},{\"type\":\"object\",\"name\":\"GlyphRenderer\",\"id\":\"p1107\",\"attributes\":{\"data_source\":{\"type\":\"object\",\"name\":\"ColumnDataSource\",\"id\":\"p1101\",\"attributes\":{\"selected\":{\"type\":\"object\",\"name\":\"Selection\",\"id\":\"p1102\",\"attributes\":{\"indices\":[],\"line_indices\":[]}},\"selection_policy\":{\"type\":\"object\",\"name\":\"UnionRenderers\",\"id\":\"p1103\"},\"data\":{\"type\":\"map\",\"entries\":[[\"x\",{\"type\":\"ndarray\",\"array\":{\"type\":\"bytes\",\"data\":\"gog6pPqd9j/5LgtcX/H+Pw==\"},\"shape\":[2],\"dtype\":\"float64\",\"order\":\"little\"}],[\"y\",[\"Rex1\",\"Rex1\"]]]}}},\"view\":{\"type\":\"object\",\"name\":\"CDSView\",\"id\":\"p1108\",\"attributes\":{\"filter\":{\"type\":\"object\",\"name\":\"AllIndices\",\"id\":\"p1109\"}}},\"glyph\":{\"type\":\"object\",\"name\":\"Line\",\"id\":\"p1104\",\"attributes\":{\"x\":{\"type\":\"field\",\"field\":\"x\"},\"y\":{\"type\":\"field\",\"field\":\"y\"},\"line_color\":\"#1f77b4\",\"line_width\":3}},\"nonselection_glyph\":{\"type\":\"object\",\"name\":\"Line\",\"id\":\"p1105\",\"attributes\":{\"x\":{\"type\":\"field\",\"field\":\"x\"},\"y\":{\"type\":\"field\",\"field\":\"y\"},\"line_color\":\"#1f77b4\",\"line_alpha\":0.1,\"line_width\":3}},\"muted_glyph\":{\"type\":\"object\",\"name\":\"Line\",\"id\":\"p1106\",\"attributes\":{\"x\":{\"type\":\"field\",\"field\":\"x\"},\"y\":{\"type\":\"field\",\"field\":\"y\"},\"line_color\":\"#1f77b4\",\"line_alpha\":0.2,\"line_width\":3}}}}],\"toolbar\":{\"type\":\"object\",\"name\":\"Toolbar\",\"id\":\"p1011\",\"attributes\":{\"tools\":[{\"type\":\"object\",\"name\":\"PanTool\",\"id\":\"p1025\"},{\"type\":\"object\",\"name\":\"WheelZoomTool\",\"id\":\"p1026\",\"attributes\":{\"renderers\":\"auto\"}},{\"type\":\"object\",\"name\":\"BoxZoomTool\",\"id\":\"p1027\",\"attributes\":{\"dimensions\":\"both\",\"overlay\":{\"type\":\"object\",\"name\":\"BoxAnnotation\",\"id\":\"p1028\",\"attributes\":{\"syncable\":false,\"line_color\":\"black\",\"line_alpha\":1.0,\"line_width\":2,\"line_dash\":[4,4],\"fill_color\":\"lightgrey\",\"fill_alpha\":0.5,\"level\":\"overlay\",\"visible\":false,\"left\":{\"type\":\"number\",\"value\":\"nan\"},\"right\":{\"type\":\"number\",\"value\":\"nan\"},\"top\":{\"type\":\"number\",\"value\":\"nan\"},\"bottom\":{\"type\":\"number\",\"value\":\"nan\"},\"left_units\":\"canvas\",\"right_units\":\"canvas\",\"top_units\":\"canvas\",\"bottom_units\":\"canvas\",\"handles\":{\"type\":\"object\",\"name\":\"BoxInteractionHandles\",\"id\":\"p1034\",\"attributes\":{\"all\":{\"type\":\"object\",\"name\":\"AreaVisuals\",\"id\":\"p1033\",\"attributes\":{\"fill_color\":\"white\",\"hover_fill_color\":\"lightgray\"}}}}}}}},{\"type\":\"object\",\"name\":\"SaveTool\",\"id\":\"p1035\"},{\"type\":\"object\",\"name\":\"ResetTool\",\"id\":\"p1036\"},{\"type\":\"object\",\"name\":\"HelpTool\",\"id\":\"p1037\"}]}},\"toolbar_location\":\"above\",\"left\":[{\"type\":\"object\",\"name\":\"CategoricalAxis\",\"id\":\"p1020\",\"attributes\":{\"ticker\":{\"type\":\"object\",\"name\":\"CategoricalTicker\",\"id\":\"p1021\"},\"formatter\":{\"type\":\"object\",\"name\":\"CategoricalTickFormatter\",\"id\":\"p1022\"},\"major_label_policy\":{\"type\":\"object\",\"name\":\"AllLabels\",\"id\":\"p1023\"}}}],\"below\":[{\"type\":\"object\",\"name\":\"LinearAxis\",\"id\":\"p1015\",\"attributes\":{\"ticker\":{\"type\":\"object\",\"name\":\"BasicTicker\",\"id\":\"p1016\",\"attributes\":{\"mantissas\":[1,2,5]}},\"formatter\":{\"type\":\"object\",\"name\":\"BasicTickFormatter\",\"id\":\"p1017\"},\"axis_label\":\"\\u03b1*\",\"major_label_policy\":{\"type\":\"object\",\"name\":\"AllLabels\",\"id\":\"p1018\"}}}],\"center\":[{\"type\":\"object\",\"name\":\"Grid\",\"id\":\"p1019\",\"attributes\":{\"axis\":{\"id\":\"p1015\"}}},{\"type\":\"object\",\"name\":\"Grid\",\"id\":\"p1024\",\"attributes\":{\"dimension\":1,\"axis\":{\"id\":\"p1020\"}}}],\"frame_width\":450,\"frame_height\":75}},{\"type\":\"object\",\"name\":\"Spacer\",\"id\":\"p1217\",\"attributes\":{\"height\":20}},{\"type\":\"object\",\"name\":\"Figure\",\"id\":\"p1110\",\"attributes\":{\"x_range\":{\"type\":\"object\",\"name\":\"DataRange1d\",\"id\":\"p1112\"},\"y_range\":{\"type\":\"object\",\"name\":\"FactorRange\",\"id\":\"p1119\",\"attributes\":{\"factors\":[\"Rex1\",\"Rest\",\"Prdm14\",\"Nanog\"]}},\"x_scale\":{\"type\":\"object\",\"name\":\"LinearScale\",\"id\":\"p1120\"},\"y_scale\":{\"type\":\"object\",\"name\":\"CategoricalScale\",\"id\":\"p1121\"},\"title\":{\"type\":\"object\",\"name\":\"Title\",\"id\":\"p1117\"},\"renderers\":[{\"type\":\"object\",\"name\":\"GlyphRenderer\",\"id\":\"p1151\",\"attributes\":{\"data_source\":{\"type\":\"object\",\"name\":\"ColumnDataSource\",\"id\":\"p1145\",\"attributes\":{\"selected\":{\"type\":\"object\",\"name\":\"Selection\",\"id\":\"p1146\",\"attributes\":{\"indices\":[],\"line_indices\":[]}},\"selection_policy\":{\"type\":\"object\",\"name\":\"UnionRenderers\",\"id\":\"p1147\"},\"data\":{\"type\":\"map\",\"entries\":[[\"x\",[69.34784233619546]],[\"y\",[\"Nanog\"]]]}}},\"view\":{\"type\":\"object\",\"name\":\"CDSView\",\"id\":\"p1152\",\"attributes\":{\"filter\":{\"type\":\"object\",\"name\":\"AllIndices\",\"id\":\"p1153\"}}},\"glyph\":{\"type\":\"object\",\"name\":\"Scatter\",\"id\":\"p1148\",\"attributes\":{\"x\":{\"type\":\"field\",\"field\":\"x\"},\"y\":{\"type\":\"field\",\"field\":\"y\"},\"size\":{\"type\":\"value\",\"value\":5},\"line_color\":{\"type\":\"value\",\"value\":\"#1f77b4\"},\"fill_color\":{\"type\":\"value\",\"value\":\"#1f77b4\"}}},\"nonselection_glyph\":{\"type\":\"object\",\"name\":\"Scatter\",\"id\":\"p1149\",\"attributes\":{\"x\":{\"type\":\"field\",\"field\":\"x\"},\"y\":{\"type\":\"field\",\"field\":\"y\"},\"size\":{\"type\":\"value\",\"value\":5},\"line_color\":{\"type\":\"value\",\"value\":\"#1f77b4\"},\"line_alpha\":{\"type\":\"value\",\"value\":0.1},\"fill_color\":{\"type\":\"value\",\"value\":\"#1f77b4\"},\"fill_alpha\":{\"type\":\"value\",\"value\":0.1},\"hatch_alpha\":{\"type\":\"value\",\"value\":0.1}}},\"muted_glyph\":{\"type\":\"object\",\"name\":\"Scatter\",\"id\":\"p1150\",\"attributes\":{\"x\":{\"type\":\"field\",\"field\":\"x\"},\"y\":{\"type\":\"field\",\"field\":\"y\"},\"size\":{\"type\":\"value\",\"value\":5},\"line_color\":{\"type\":\"value\",\"value\":\"#1f77b4\"},\"line_alpha\":{\"type\":\"value\",\"value\":0.2},\"fill_color\":{\"type\":\"value\",\"value\":\"#1f77b4\"},\"fill_alpha\":{\"type\":\"value\",\"value\":0.2},\"hatch_alpha\":{\"type\":\"value\",\"value\":0.2}}}}},{\"type\":\"object\",\"name\":\"GlyphRenderer\",\"id\":\"p1160\",\"attributes\":{\"data_source\":{\"type\":\"object\",\"name\":\"ColumnDataSource\",\"id\":\"p1154\",\"attributes\":{\"selected\":{\"type\":\"object\",\"name\":\"Selection\",\"id\":\"p1155\",\"attributes\":{\"indices\":[],\"line_indices\":[]}},\"selection_policy\":{\"type\":\"object\",\"name\":\"UnionRenderers\",\"id\":\"p1156\"},\"data\":{\"type\":\"map\",\"entries\":[[\"x\",{\"type\":\"ndarray\",\"array\":{\"type\":\"bytes\",\"data\":\"nbsRX1SJTEAIyXjj3q1UQA==\"},\"shape\":[2],\"dtype\":\"float64\",\"order\":\"little\"}],[\"y\",[\"Nanog\",\"Nanog\"]]]}}},\"view\":{\"type\":\"object\",\"name\":\"CDSView\",\"id\":\"p1161\",\"attributes\":{\"filter\":{\"type\":\"object\",\"name\":\"AllIndices\",\"id\":\"p1162\"}}},\"glyph\":{\"type\":\"object\",\"name\":\"Line\",\"id\":\"p1157\",\"attributes\":{\"x\":{\"type\":\"field\",\"field\":\"x\"},\"y\":{\"type\":\"field\",\"field\":\"y\"},\"line_color\":\"#1f77b4\",\"line_width\":3}},\"nonselection_glyph\":{\"type\":\"object\",\"name\":\"Line\",\"id\":\"p1158\",\"attributes\":{\"x\":{\"type\":\"field\",\"field\":\"x\"},\"y\":{\"type\":\"field\",\"field\":\"y\"},\"line_color\":\"#1f77b4\",\"line_alpha\":0.1,\"line_width\":3}},\"muted_glyph\":{\"type\":\"object\",\"name\":\"Line\",\"id\":\"p1159\",\"attributes\":{\"x\":{\"type\":\"field\",\"field\":\"x\"},\"y\":{\"type\":\"field\",\"field\":\"y\"},\"line_color\":\"#1f77b4\",\"line_alpha\":0.2,\"line_width\":3}}}},{\"type\":\"object\",\"name\":\"GlyphRenderer\",\"id\":\"p1169\",\"attributes\":{\"data_source\":{\"type\":\"object\",\"name\":\"ColumnDataSource\",\"id\":\"p1163\",\"attributes\":{\"selected\":{\"type\":\"object\",\"name\":\"Selection\",\"id\":\"p1164\",\"attributes\":{\"indices\":[],\"line_indices\":[]}},\"selection_policy\":{\"type\":\"object\",\"name\":\"UnionRenderers\",\"id\":\"p1165\"},\"data\":{\"type\":\"map\",\"entries\":[[\"x\",[8.200635706742071]],[\"y\",[\"Prdm14\"]]]}}},\"view\":{\"type\":\"object\",\"name\":\"CDSView\",\"id\":\"p1170\",\"attributes\":{\"filter\":{\"type\":\"object\",\"name\":\"AllIndices\",\"id\":\"p1171\"}}},\"glyph\":{\"type\":\"object\",\"name\":\"Scatter\",\"id\":\"p1166\",\"attributes\":{\"x\":{\"type\":\"field\",\"field\":\"x\"},\"y\":{\"type\":\"field\",\"field\":\"y\"},\"size\":{\"type\":\"value\",\"value\":5},\"line_color\":{\"type\":\"value\",\"value\":\"#1f77b4\"},\"fill_color\":{\"type\":\"value\",\"value\":\"#1f77b4\"}}},\"nonselection_glyph\":{\"type\":\"object\",\"name\":\"Scatter\",\"id\":\"p1167\",\"attributes\":{\"x\":{\"type\":\"field\",\"field\":\"x\"},\"y\":{\"type\":\"field\",\"field\":\"y\"},\"size\":{\"type\":\"value\",\"value\":5},\"line_color\":{\"type\":\"value\",\"value\":\"#1f77b4\"},\"line_alpha\":{\"type\":\"value\",\"value\":0.1},\"fill_color\":{\"type\":\"value\",\"value\":\"#1f77b4\"},\"fill_alpha\":{\"type\":\"value\",\"value\":0.1},\"hatch_alpha\":{\"type\":\"value\",\"value\":0.1}}},\"muted_glyph\":{\"type\":\"object\",\"name\":\"Scatter\",\"id\":\"p1168\",\"attributes\":{\"x\":{\"type\":\"field\",\"field\":\"x\"},\"y\":{\"type\":\"field\",\"field\":\"y\"},\"size\":{\"type\":\"value\",\"value\":5},\"line_color\":{\"type\":\"value\",\"value\":\"#1f77b4\"},\"line_alpha\":{\"type\":\"value\",\"value\":0.2},\"fill_color\":{\"type\":\"value\",\"value\":\"#1f77b4\"},\"fill_alpha\":{\"type\":\"value\",\"value\":0.2},\"hatch_alpha\":{\"type\":\"value\",\"value\":0.2}}}}},{\"type\":\"object\",\"name\":\"GlyphRenderer\",\"id\":\"p1178\",\"attributes\":{\"data_source\":{\"type\":\"object\",\"name\":\"ColumnDataSource\",\"id\":\"p1172\",\"attributes\":{\"selected\":{\"type\":\"object\",\"name\":\"Selection\",\"id\":\"p1173\",\"attributes\":{\"indices\":[],\"line_indices\":[]}},\"selection_policy\":{\"type\":\"object\",\"name\":\"UnionRenderers\",\"id\":\"p1174\"},\"data\":{\"type\":\"map\",\"entries\":[[\"x\",{\"type\":\"ndarray\",\"array\":{\"type\":\"bytes\",\"data\":\"e//jy7ufGEDZ7OVO0xYlQA==\"},\"shape\":[2],\"dtype\":\"float64\",\"order\":\"little\"}],[\"y\",[\"Prdm14\",\"Prdm14\"]]]}}},\"view\":{\"type\":\"object\",\"name\":\"CDSView\",\"id\":\"p1179\",\"attributes\":{\"filter\":{\"type\":\"object\",\"name\":\"AllIndices\",\"id\":\"p1180\"}}},\"glyph\":{\"type\":\"object\",\"name\":\"Line\",\"id\":\"p1175\",\"attributes\":{\"x\":{\"type\":\"field\",\"field\":\"x\"},\"y\":{\"type\":\"field\",\"field\":\"y\"},\"line_color\":\"#1f77b4\",\"line_width\":3}},\"nonselection_glyph\":{\"type\":\"object\",\"name\":\"Line\",\"id\":\"p1176\",\"attributes\":{\"x\":{\"type\":\"field\",\"field\":\"x\"},\"y\":{\"type\":\"field\",\"field\":\"y\"},\"line_color\":\"#1f77b4\",\"line_alpha\":0.1,\"line_width\":3}},\"muted_glyph\":{\"type\":\"object\",\"name\":\"Line\",\"id\":\"p1177\",\"attributes\":{\"x\":{\"type\":\"field\",\"field\":\"x\"},\"y\":{\"type\":\"field\",\"field\":\"y\"},\"line_color\":\"#1f77b4\",\"line_alpha\":0.2,\"line_width\":3}}}},{\"type\":\"object\",\"name\":\"GlyphRenderer\",\"id\":\"p1187\",\"attributes\":{\"data_source\":{\"type\":\"object\",\"name\":\"ColumnDataSource\",\"id\":\"p1181\",\"attributes\":{\"selected\":{\"type\":\"object\",\"name\":\"Selection\",\"id\":\"p1182\",\"attributes\":{\"indices\":[],\"line_indices\":[]}},\"selection_policy\":{\"type\":\"object\",\"name\":\"UnionRenderers\",\"id\":\"p1183\"},\"data\":{\"type\":\"map\",\"entries\":[[\"x\",[16.54305361977416]],[\"y\",[\"Rest\"]]]}}},\"view\":{\"type\":\"object\",\"name\":\"CDSView\",\"id\":\"p1188\",\"attributes\":{\"filter\":{\"type\":\"object\",\"name\":\"AllIndices\",\"id\":\"p1189\"}}},\"glyph\":{\"type\":\"object\",\"name\":\"Scatter\",\"id\":\"p1184\",\"attributes\":{\"x\":{\"type\":\"field\",\"field\":\"x\"},\"y\":{\"type\":\"field\",\"field\":\"y\"},\"size\":{\"type\":\"value\",\"value\":5},\"line_color\":{\"type\":\"value\",\"value\":\"#1f77b4\"},\"fill_color\":{\"type\":\"value\",\"value\":\"#1f77b4\"}}},\"nonselection_glyph\":{\"type\":\"object\",\"name\":\"Scatter\",\"id\":\"p1185\",\"attributes\":{\"x\":{\"type\":\"field\",\"field\":\"x\"},\"y\":{\"type\":\"field\",\"field\":\"y\"},\"size\":{\"type\":\"value\",\"value\":5},\"line_color\":{\"type\":\"value\",\"value\":\"#1f77b4\"},\"line_alpha\":{\"type\":\"value\",\"value\":0.1},\"fill_color\":{\"type\":\"value\",\"value\":\"#1f77b4\"},\"fill_alpha\":{\"type\":\"value\",\"value\":0.1},\"hatch_alpha\":{\"type\":\"value\",\"value\":0.1}}},\"muted_glyph\":{\"type\":\"object\",\"name\":\"Scatter\",\"id\":\"p1186\",\"attributes\":{\"x\":{\"type\":\"field\",\"field\":\"x\"},\"y\":{\"type\":\"field\",\"field\":\"y\"},\"size\":{\"type\":\"value\",\"value\":5},\"line_color\":{\"type\":\"value\",\"value\":\"#1f77b4\"},\"line_alpha\":{\"type\":\"value\",\"value\":0.2},\"fill_color\":{\"type\":\"value\",\"value\":\"#1f77b4\"},\"fill_alpha\":{\"type\":\"value\",\"value\":0.2},\"hatch_alpha\":{\"type\":\"value\",\"value\":0.2}}}}},{\"type\":\"object\",\"name\":\"GlyphRenderer\",\"id\":\"p1196\",\"attributes\":{\"data_source\":{\"type\":\"object\",\"name\":\"ColumnDataSource\",\"id\":\"p1190\",\"attributes\":{\"selected\":{\"type\":\"object\",\"name\":\"Selection\",\"id\":\"p1191\",\"attributes\":{\"indices\":[],\"line_indices\":[]}},\"selection_policy\":{\"type\":\"object\",\"name\":\"UnionRenderers\",\"id\":\"p1192\"},\"data\":{\"type\":\"map\",\"entries\":[[\"x\",{\"type\":\"ndarray\",\"array\":{\"type\":\"bytes\",\"data\":\"1/sHM24rK0D9ytvkbnQzQA==\"},\"shape\":[2],\"dtype\":\"float64\",\"order\":\"little\"}],[\"y\",[\"Rest\",\"Rest\"]]]}}},\"view\":{\"type\":\"object\",\"name\":\"CDSView\",\"id\":\"p1197\",\"attributes\":{\"filter\":{\"type\":\"object\",\"name\":\"AllIndices\",\"id\":\"p1198\"}}},\"glyph\":{\"type\":\"object\",\"name\":\"Line\",\"id\":\"p1193\",\"attributes\":{\"x\":{\"type\":\"field\",\"field\":\"x\"},\"y\":{\"type\":\"field\",\"field\":\"y\"},\"line_color\":\"#1f77b4\",\"line_width\":3}},\"nonselection_glyph\":{\"type\":\"object\",\"name\":\"Line\",\"id\":\"p1194\",\"attributes\":{\"x\":{\"type\":\"field\",\"field\":\"x\"},\"y\":{\"type\":\"field\",\"field\":\"y\"},\"line_color\":\"#1f77b4\",\"line_alpha\":0.1,\"line_width\":3}},\"muted_glyph\":{\"type\":\"object\",\"name\":\"Line\",\"id\":\"p1195\",\"attributes\":{\"x\":{\"type\":\"field\",\"field\":\"x\"},\"y\":{\"type\":\"field\",\"field\":\"y\"},\"line_color\":\"#1f77b4\",\"line_alpha\":0.2,\"line_width\":3}}}},{\"type\":\"object\",\"name\":\"GlyphRenderer\",\"id\":\"p1205\",\"attributes\":{\"data_source\":{\"type\":\"object\",\"name\":\"ColumnDataSource\",\"id\":\"p1199\",\"attributes\":{\"selected\":{\"type\":\"object\",\"name\":\"Selection\",\"id\":\"p1200\",\"attributes\":{\"indices\":[],\"line_indices\":[]}},\"selection_policy\":{\"type\":\"object\",\"name\":\"UnionRenderers\",\"id\":\"p1201\"},\"data\":{\"type\":\"map\",\"entries\":[[\"x\",[84.68091531208012]],[\"y\",[\"Rex1\"]]]}}},\"view\":{\"type\":\"object\",\"name\":\"CDSView\",\"id\":\"p1206\",\"attributes\":{\"filter\":{\"type\":\"object\",\"name\":\"AllIndices\",\"id\":\"p1207\"}}},\"glyph\":{\"type\":\"object\",\"name\":\"Scatter\",\"id\":\"p1202\",\"attributes\":{\"x\":{\"type\":\"field\",\"field\":\"x\"},\"y\":{\"type\":\"field\",\"field\":\"y\"},\"size\":{\"type\":\"value\",\"value\":5},\"line_color\":{\"type\":\"value\",\"value\":\"#1f77b4\"},\"fill_color\":{\"type\":\"value\",\"value\":\"#1f77b4\"}}},\"nonselection_glyph\":{\"type\":\"object\",\"name\":\"Scatter\",\"id\":\"p1203\",\"attributes\":{\"x\":{\"type\":\"field\",\"field\":\"x\"},\"y\":{\"type\":\"field\",\"field\":\"y\"},\"size\":{\"type\":\"value\",\"value\":5},\"line_color\":{\"type\":\"value\",\"value\":\"#1f77b4\"},\"line_alpha\":{\"type\":\"value\",\"value\":0.1},\"fill_color\":{\"type\":\"value\",\"value\":\"#1f77b4\"},\"fill_alpha\":{\"type\":\"value\",\"value\":0.1},\"hatch_alpha\":{\"type\":\"value\",\"value\":0.1}}},\"muted_glyph\":{\"type\":\"object\",\"name\":\"Scatter\",\"id\":\"p1204\",\"attributes\":{\"x\":{\"type\":\"field\",\"field\":\"x\"},\"y\":{\"type\":\"field\",\"field\":\"y\"},\"size\":{\"type\":\"value\",\"value\":5},\"line_color\":{\"type\":\"value\",\"value\":\"#1f77b4\"},\"line_alpha\":{\"type\":\"value\",\"value\":0.2},\"fill_color\":{\"type\":\"value\",\"value\":\"#1f77b4\"},\"fill_alpha\":{\"type\":\"value\",\"value\":0.2},\"hatch_alpha\":{\"type\":\"value\",\"value\":0.2}}}}},{\"type\":\"object\",\"name\":\"GlyphRenderer\",\"id\":\"p1214\",\"attributes\":{\"data_source\":{\"type\":\"object\",\"name\":\"ColumnDataSource\",\"id\":\"p1208\",\"attributes\":{\"selected\":{\"type\":\"object\",\"name\":\"Selection\",\"id\":\"p1209\",\"attributes\":{\"indices\":[],\"line_indices\":[]}},\"selection_policy\":{\"type\":\"object\",\"name\":\"UnionRenderers\",\"id\":\"p1210\"},\"data\":{\"type\":\"map\",\"entries\":[[\"x\",{\"type\":\"ndarray\",\"array\":{\"type\":\"bytes\",\"data\":\"GzChmIVzUUAh6BcseQFZQA==\"},\"shape\":[2],\"dtype\":\"float64\",\"order\":\"little\"}],[\"y\",[\"Rex1\",\"Rex1\"]]]}}},\"view\":{\"type\":\"object\",\"name\":\"CDSView\",\"id\":\"p1215\",\"attributes\":{\"filter\":{\"type\":\"object\",\"name\":\"AllIndices\",\"id\":\"p1216\"}}},\"glyph\":{\"type\":\"object\",\"name\":\"Line\",\"id\":\"p1211\",\"attributes\":{\"x\":{\"type\":\"field\",\"field\":\"x\"},\"y\":{\"type\":\"field\",\"field\":\"y\"},\"line_color\":\"#1f77b4\",\"line_width\":3}},\"nonselection_glyph\":{\"type\":\"object\",\"name\":\"Line\",\"id\":\"p1212\",\"attributes\":{\"x\":{\"type\":\"field\",\"field\":\"x\"},\"y\":{\"type\":\"field\",\"field\":\"y\"},\"line_color\":\"#1f77b4\",\"line_alpha\":0.1,\"line_width\":3}},\"muted_glyph\":{\"type\":\"object\",\"name\":\"Line\",\"id\":\"p1213\",\"attributes\":{\"x\":{\"type\":\"field\",\"field\":\"x\"},\"y\":{\"type\":\"field\",\"field\":\"y\"},\"line_color\":\"#1f77b4\",\"line_alpha\":0.2,\"line_width\":3}}}}],\"toolbar\":{\"type\":\"object\",\"name\":\"Toolbar\",\"id\":\"p1118\",\"attributes\":{\"tools\":[{\"type\":\"object\",\"name\":\"PanTool\",\"id\":\"p1132\"},{\"type\":\"object\",\"name\":\"WheelZoomTool\",\"id\":\"p1133\",\"attributes\":{\"renderers\":\"auto\"}},{\"type\":\"object\",\"name\":\"BoxZoomTool\",\"id\":\"p1134\",\"attributes\":{\"dimensions\":\"both\",\"overlay\":{\"type\":\"object\",\"name\":\"BoxAnnotation\",\"id\":\"p1135\",\"attributes\":{\"syncable\":false,\"line_color\":\"black\",\"line_alpha\":1.0,\"line_width\":2,\"line_dash\":[4,4],\"fill_color\":\"lightgrey\",\"fill_alpha\":0.5,\"level\":\"overlay\",\"visible\":false,\"left\":{\"type\":\"number\",\"value\":\"nan\"},\"right\":{\"type\":\"number\",\"value\":\"nan\"},\"top\":{\"type\":\"number\",\"value\":\"nan\"},\"bottom\":{\"type\":\"number\",\"value\":\"nan\"},\"left_units\":\"canvas\",\"right_units\":\"canvas\",\"top_units\":\"canvas\",\"bottom_units\":\"canvas\",\"handles\":{\"type\":\"object\",\"name\":\"BoxInteractionHandles\",\"id\":\"p1141\",\"attributes\":{\"all\":{\"type\":\"object\",\"name\":\"AreaVisuals\",\"id\":\"p1140\",\"attributes\":{\"fill_color\":\"white\",\"hover_fill_color\":\"lightgray\"}}}}}}}},{\"type\":\"object\",\"name\":\"SaveTool\",\"id\":\"p1142\"},{\"type\":\"object\",\"name\":\"ResetTool\",\"id\":\"p1143\"},{\"type\":\"object\",\"name\":\"HelpTool\",\"id\":\"p1144\"}]}},\"toolbar_location\":\"above\",\"left\":[{\"type\":\"object\",\"name\":\"CategoricalAxis\",\"id\":\"p1127\",\"attributes\":{\"ticker\":{\"type\":\"object\",\"name\":\"CategoricalTicker\",\"id\":\"p1128\"},\"formatter\":{\"type\":\"object\",\"name\":\"CategoricalTickFormatter\",\"id\":\"p1129\"},\"major_label_policy\":{\"type\":\"object\",\"name\":\"AllLabels\",\"id\":\"p1130\"}}}],\"below\":[{\"type\":\"object\",\"name\":\"LinearAxis\",\"id\":\"p1122\",\"attributes\":{\"ticker\":{\"type\":\"object\",\"name\":\"BasicTicker\",\"id\":\"p1123\",\"attributes\":{\"mantissas\":[1,2,5]}},\"formatter\":{\"type\":\"object\",\"name\":\"BasicTickFormatter\",\"id\":\"p1124\"},\"axis_label\":\"b*\",\"major_label_policy\":{\"type\":\"object\",\"name\":\"AllLabels\",\"id\":\"p1125\"}}}],\"center\":[{\"type\":\"object\",\"name\":\"Grid\",\"id\":\"p1126\",\"attributes\":{\"axis\":{\"id\":\"p1122\"}}},{\"type\":\"object\",\"name\":\"Grid\",\"id\":\"p1131\",\"attributes\":{\"dimension\":1,\"axis\":{\"id\":\"p1127\"}}}],\"frame_width\":450,\"frame_height\":75}}]}}]}};\n", " const render_items = [{\"docid\":\"0135de28-9a55-4649-88bd-91bf3d26eacf\",\"roots\":{\"p1218\":\"df482999-8554-445f-a067-2504fdd477c9\"},\"root_ids\":[\"p1218\"]}];\n", " void root.Bokeh.embed.embed_items_notebook(docs_json, render_items);\n", " }\n", " if (root.Bokeh !== undefined) {\n", " embed_document(root);\n", " } else {\n", " let attempts = 0;\n", " const timer = setInterval(function(root) {\n", " if (root.Bokeh !== undefined) {\n", " clearInterval(timer);\n", " embed_document(root);\n", " } else {\n", " attempts++;\n", " if (attempts > 100) {\n", " clearInterval(timer);\n", " console.log(\"Bokeh: ERROR: Unable to run BokehJS code because BokehJS library is missing\");\n", " }\n", " }\n", " }, 10, root)\n", " }\n", "})(window);" ], "application/vnd.bokehjs_exec.v0+json": "" }, "metadata": { "application/vnd.bokehjs_exec.v0+json": { "id": "p1218" } }, "output_type": "display_data" } ], "source": [ "sub_df = df_mle.filter(pl.col(\"parameter\") == \"alpha\")\n", "summaries = [\n", " {\"estimate\": est, \"conf_int\": conf, \"label\": label}\n", " for est, conf, label in zip(\n", " sub_df[\"mle\"].to_numpy(),\n", " sub_df[[\"conf_int_low\", \"conf_int_high\"]].to_numpy(),\n", " sub_df[\"gene\"],\n", " )\n", "]\n", "\n", "p1 = bebi103.viz.confints(\n", " summaries,\n", " x_axis_label=\"α*\",\n", " frame_height=75,\n", " )\n", "\n", "sub_df = df_mle.filter(pl.col(\"parameter\") == \"b\")\n", "summaries = [\n", " {\"estimate\": est, \"conf_int\": conf, \"label\": label}\n", " for est, conf, label in zip(\n", " sub_df[\"mle\"].to_numpy(),\n", " sub_df[[\"conf_int_low\", \"conf_int_high\"]].to_numpy(),\n", " sub_df[\"gene\"],\n", " )\n", "]\n", "p2 = bebi103.viz.confints(\n", " summaries,\n", " x_axis_label=\"b*\",\n", " frame_height=75,\n", " )\n", "\n", "bokeh.io.show(bokeh.layouts.column(p1, bokeh.models.Spacer(height=20), p2))" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "## Implementation of MLE bootstrapping in the bebi103 package\n", "\n", "The functionality we have developed in this lesson for bootstrapping MLE estimates is available in the function `bebi103.bootstrap.draw_bs_reps_mle()`. It has call signature\n", "\n", "```python\n", "bebi103.draw_bs_reps_mle(\n", " mle_fun,\n", " gen_fun,\n", " data,\n", " mle_args=(),\n", " gen_args=(),\n", " size=1,\n", " n_jobs=1,\n", " progress_bar=False,\n", " rg=None,\n", "):\n", "```\n", "\n", "Here, `mle_fun` is a function with call signature `mle_fun(data, *mle_fun_args)` that computes the maximum likelihood estimate from the data. `gen_fun` is a function with call signature `gen_fun(params, *gen_args, size, rg)` that generates bootstrap samples to be used to make bootstrap replicates of the maximum likelihood estimates. Here, `params` is a tuple of the parameters of the model. Note that not all of the inputs for `gen_fun` are necessarily used, but having them allows for flexibility. The `bebi103.bootstrap.draw_bs_reps_mle()` function allows for both parametric and nonparametric bootstrapping.\n", "\n", "As an example, we will repeat the bootstrapped MLE confidence region calculation for Nanog, except we will do it nonparametrically this time." ] }, { "cell_type": "code", "execution_count": 19, "metadata": {}, "outputs": [ { "name": "stderr", "output_type": "stream", "text": [ "100%|███████████████████████████████████████████████████████| 1428/1428 [00:23<00:00, 60.17it/s]\n", "100%|███████████████████████████████████████████████████████| 1432/1432 [00:23<00:00, 60.34it/s]\n", "100%|███████████████████████████████████████████████████████| 1428/1428 [00:23<00:00, 59.97it/s]\n", "100%|███████████████████████████████████████████████████████| 1428/1428 [00:23<00:00, 59.73it/s]\n", "100%|███████████████████████████████████████████████████████| 1428/1428 [00:23<00:00, 59.64it/s]\n", "100%|███████████████████████████████████████████████████████| 1428/1428 [00:23<00:00, 59.57it/s]\n", "100%|███████████████████████████████████████████████████████| 1428/1428 [00:23<00:00, 59.54it/s]\n" ] }, { "data": { "text/plain": [ "array([[ 1.05605495, 57.21452956],\n", " [ 1.54194682, 82.706094 ]])" ] }, "execution_count": 19, "metadata": {}, "output_type": "execute_result" } ], "source": [ "# This is gen_fun. For nonparametric, just a resampling\n", "# params = (alpha, beta), but is ignored because we're just resampling\n", "# gen_params = (n,); this is used for nonparametric sample\n", "def resample_fun(params, n, size, rng):\n", " return rng.choice(n, size=size)\n", "\n", "\n", "bs_reps = bebi103.bootstrap.draw_bs_reps_mle(\n", " mle_iid_nbinom,\n", " resample_fun,\n", " df['Nanog'].to_numpy(),\n", " gen_args=(df['Nanog'].to_numpy(), ),\n", " size=10000,\n", " n_jobs=7,\n", " progress_bar=True,\n", ")\n", "\n", "# Show the confidence interval\n", "np.percentile(bs_reps, [2.5, 97.5], axis=0)" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "The confidence intervals are slightly different than in the parametric case, but are quite similar." ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "## Computing environment" ] }, { "cell_type": "code", "execution_count": 20, "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "Python implementation: CPython\n", "Python version : 3.13.5\n", "IPython version : 9.4.0\n", "\n", "multiprocess: 0.70.18\n", "numpy : 2.2.6\n", "scipy : 1.16.0\n", "polars : 1.31.0\n", "tqdm : 4.67.1\n", "bokeh : 3.7.3\n", "bebi103 : 0.1.28\n", "jupyterlab : 4.4.5\n", "\n" ] } ], "source": [ "%load_ext watermark\n", "%watermark -v -p multiprocess,numpy,scipy,polars,tqdm,bokeh,bebi103,jupyterlab" ] } ], "metadata": { "kernelspec": { "display_name": "Python 3 (ipykernel)", "language": "python", "name": "python3" }, "language_info": { "codemirror_mode": { "name": "ipython", "version": 3 }, "file_extension": ".py", "mimetype": "text/x-python", "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", "version": "3.13.5" } }, "nbformat": 4, "nbformat_minor": 4 }