{ "metadata": { "name": "", "signature": "sha256:ff62d4e21ced5ad7d5023c5fbde1f6dfd895bff978dbbc984b27d6dba9595273" }, "nbformat": 3, "nbformat_minor": 0, "worksheets": [ { "cells": [ { "cell_type": "code", "collapsed": false, "input": [ "# grab the some data sets from course repository using the command line program wget\n", "! wget https://github.com/pmagwene/Bio723/raw/master/datasets/iris.csv" ], "language": "python", "metadata": {}, "outputs": [] }, { "cell_type": "code", "collapsed": false, "input": [ "! wget https://github.com/pmagwene/Bio723/raw/master/datasets/anscombe.csv" ], "language": "python", "metadata": {}, "outputs": [] }, { "cell_type": "heading", "level": 1, "metadata": {}, "source": [ "Vector Mathematics" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "NumPy arrays support basic arithmetic operations that correspond to the same operations on geometric vectors. The following examples illustrate this:" ] }, { "cell_type": "code", "collapsed": false, "input": [ "# enable inline plots\n", "%matplotlib inline " ], "language": "python", "metadata": {}, "outputs": [] }, { "cell_type": "code", "collapsed": false, "input": [ "import math\n", "import numpy as np\n", "from pylab import *\n", "from matplotlib import pyplot as plt" ], "language": "python", "metadata": {}, "outputs": [] }, { "cell_type": "code", "collapsed": false, "input": [ "x = np.array([1,2,3,4,5])\n", "y = np.array([6,7,8,9,10])" ], "language": "python", "metadata": {}, "outputs": [] }, { "cell_type": "code", "collapsed": false, "input": [ "# vector addition" ], "language": "python", "metadata": {}, "outputs": [] }, { "cell_type": "code", "collapsed": false, "input": [ "# vector subtraction" ], "language": "python", "metadata": {}, "outputs": [] }, { "cell_type": "code", "collapsed": false, "input": [ "# scalar multiplication" ], "language": "python", "metadata": {}, "outputs": [] }, { "cell_type": "code", "collapsed": false, "input": [ "# element-wise scalar multiplication, NOT dot product" ], "language": "python", "metadata": {}, "outputs": [] }, { "cell_type": "code", "collapsed": false, "input": [ "# scalar division" ], "language": "python", "metadata": {}, "outputs": [] }, { "cell_type": "code", "collapsed": false, "input": [ "# dot product" ], "language": "python", "metadata": {}, "outputs": [] }, { "cell_type": "markdown", "metadata": {}, "source": [ "# Vector Geometry" ] }, { "cell_type": "code", "collapsed": false, "input": [ "# create two simple 2D vectors\n", "a = np.array((1,0))\n", "b = np.array((0,1))" ], "language": "python", "metadata": {}, "outputs": [] }, { "cell_type": "code", "collapsed": false, "input": [ "# represent those vectors in the Cartesian plan" ], "language": "python", "metadata": {}, "outputs": [] }, { "cell_type": "code", "collapsed": false, "input": [ "# calculate lengths of vectors using dot products" ], "language": "python", "metadata": {}, "outputs": [] }, { "cell_type": "code", "collapsed": false, "input": [ "# cosine of angle between a and b" ], "language": "python", "metadata": {}, "outputs": [] }, { "cell_type": "code", "collapsed": false, "input": [ "# convert from cos theta to theta (radians)" ], "language": "python", "metadata": {}, "outputs": [] }, { "cell_type": "code", "collapsed": false, "input": [ "# convert radians to degrees" ], "language": "python", "metadata": {}, "outputs": [] }, { "cell_type": "markdown", "metadata": {}, "source": [ "# Writing functions in Python" ] }, { "cell_type": "code", "collapsed": false, "input": [ "# General form of a Python function\n", "def funcname(arg1,arg2):\n", " # body of function\n", " pass # substitute this line with a return statement" ], "language": "python", "metadata": {}, "outputs": [] }, { "cell_type": "code", "collapsed": false, "input": [ "# vector functions\n", "\n", "def veclength(x):\n", " pass # replace with function definition\n" ], "language": "python", "metadata": {}, "outputs": [] }, { "cell_type": "code", "collapsed": false, "input": [ "x = np.array([1,2,3,4,5])\n", "print \"x: \", x\n", "print \"length of x: \", veclength(x)" ], "language": "python", "metadata": {}, "outputs": [] }, { "cell_type": "code", "collapsed": false, "input": [ "# In an external editor create a module called vecgeom.py\n", "# write two functions -- veclength, unitvector" ], "language": "python", "metadata": {}, "outputs": [] }, { "cell_type": "code", "collapsed": false, "input": [ "import vecgeom as vg" ], "language": "python", "metadata": {}, "outputs": [] }, { "cell_type": "code", "collapsed": false, "input": [ "vg.veclength(x)" ], "language": "python", "metadata": {}, "outputs": [] }, { "cell_type": "code", "collapsed": false, "input": [ "vg.unitvector(x)" ], "language": "python", "metadata": {}, "outputs": [] }, { "cell_type": "code", "collapsed": false, "input": [ "vg.veclength(vg.unitvector(x))" ], "language": "python", "metadata": {}, "outputs": [] }, { "cell_type": "code", "collapsed": false, "input": [ "# Add the following function to your vecgeom module\n", "\n", "def draw_vectors(a, b, colors=('red', 'blue'), clear = True):\n", " \"\"\" Given vectors a and b, draw their geometric representation. \"\"\"\n", " # figure out the limits such that the origin and the vector\n", " # end points are all included in the plot\n", " xhi = max(0, a[0], b[0])\n", " xlo = min(0, a[0], b[0])\n", " yhi = max(0, a[1], b[1])\n", " ylo = min(0, a[1], b[1])\n", "\n", " xlims = np.array((xlo, xhi))*1.10 # give a little breathing space around vectors\n", " ylims = np.array((ylo, yhi))*1.10\n", "\n", " if clear:\n", " ax = plt.axes()\n", " plt.xlim(*xlims)\n", " plt.ylim(*ylims)\n", " plt.xlabel(\"x-coord\")\n", " plt.ylabel(\"y-coord\")\n", "\n", " plt.arrow(0, 0, a[0], a[1], color=colors[0])\n", " plt.arrow(0, 0, b[0], b[1], color=colors[1])" ], "language": "python", "metadata": {}, "outputs": [] }, { "cell_type": "code", "collapsed": false, "input": [ "reload(vg) # if you change your module, you must reload it" ], "language": "python", "metadata": {}, "outputs": [] }, { "cell_type": "code", "collapsed": false, "input": [ "a = np.array((1,0.4))\n", "b = np.array((0.2,0.8))\n", "vg.draw_vectors(a,b)" ], "language": "python", "metadata": {}, "outputs": [] }, { "cell_type": "markdown", "metadata": {}, "source": [ "## Question?\n", "As it's written, the `draw_vectors` function only works with 2D vectors. How would you generalize to work with pairs of vectors of arbitrary dimension?\n" ] }, { "cell_type": "code", "collapsed": false, "input": [ "## write a vec_cos function" ], "language": "python", "metadata": {}, "outputs": [] }, { "cell_type": "code", "collapsed": false, "input": [ "# test your vec_cos function\n", "vec_cos(a,b)" ], "language": "python", "metadata": {}, "outputs": [] }, { "cell_type": "markdown", "metadata": {}, "source": [ "# Vector stats" ] }, { "cell_type": "code", "collapsed": false, "input": [ "x = np.random.normal(loc=1, size=20)\n", "y = np.random.normal(loc=2, size=20)" ], "language": "python", "metadata": {}, "outputs": [] }, { "cell_type": "code", "collapsed": false, "input": [ "# np.mean function" ], "language": "python", "metadata": {}, "outputs": [] }, { "cell_type": "code", "collapsed": false, "input": [ "# centered variables x and y" ], "language": "python", "metadata": {}, "outputs": [] }, { "cell_type": "code", "collapsed": false, "input": [ "# Numpy has a built in variance function\n", "# need to specificy ddof=1 to get sample variance (rather than popn variance)\n", "np.var(x, ddof=1)" ], "language": "python", "metadata": {}, "outputs": [] }, { "cell_type": "code", "collapsed": false, "input": [ "# compare length of the residual vector squared divided by d.f. \n", "# to the variance" ], "language": "python", "metadata": {}, "outputs": [] }, { "cell_type": "code", "collapsed": false, "input": [ "# Numpy has a built in correlation function\n", "np.corrcoef(x,y)" ], "language": "python", "metadata": {}, "outputs": [] }, { "cell_type": "code", "collapsed": false, "input": [ "# compare np.corrcoeff to our vec_cos function applied to the centered data" ], "language": "python", "metadata": {}, "outputs": [] }, { "cell_type": "code", "collapsed": false, "input": [ "# write a function, vcorr()that calculates the correlation between two variables in terms\n", "# of the cosine of the angle between their vector representations\n", "\n", "def vcorr(x,y):\n", " pass # replace by function definition" ], "language": "python", "metadata": {}, "outputs": [] }, { "cell_type": "markdown", "metadata": {}, "source": [ "# Bivariate regression in vector terms" ] }, { "cell_type": "code", "collapsed": false, "input": [ "def bivariate_regression(x,y):\n", " \"\"\"Calculate the regression of x on y.\n", " \"\"\"\n", " ctrx = x - np.mean(x)\n", " ctry = y - np.mean(y)\n", " slope = np.dot(ctrx, ctry)/np.dot(ctrx, ctrx)\n", " intercept = np.mean(y) - slope*np.mean(x)\n", " return intercept, slope\n", " " ], "language": "python", "metadata": {}, "outputs": [] }, { "cell_type": "code", "collapsed": false, "input": [ "# test your bivariate regression function\n", "# use the Panda's library\n", "import pandas as pd\n", "anscombe = pd.read_csv('anscombe.csv')\n", "\n", "bivariate_regression(anscombe.x1, anscombe.y1)" ], "language": "python", "metadata": {}, "outputs": [] }, { "cell_type": "markdown", "metadata": {}, "source": [ "# Anscombe's data" ] }, { "cell_type": "code", "collapsed": false, "input": [ "# use the Panda's library\n", "import pandas as pd\n", "anscombe = pd.read_csv('anscombe.csv')" ], "language": "python", "metadata": {}, "outputs": [] }, { "cell_type": "code", "collapsed": false, "input": [ "anscombe.columns" ], "language": "python", "metadata": {}, "outputs": [] }, { "cell_type": "code", "collapsed": false, "input": [ "anscombe.mean()" ], "language": "python", "metadata": {}, "outputs": [] }, { "cell_type": "code", "collapsed": false, "input": [ "anscombe.var()" ], "language": "python", "metadata": {}, "outputs": [] }, { "cell_type": "code", "collapsed": false, "input": [ "np.corrcoeff(anscombe.x1, anscombe.y1)" ], "language": "python", "metadata": {}, "outputs": [] }, { "cell_type": "code", "collapsed": false, "input": [ "np.corrcoeff(anscombe.x2, anscombe.y2)" ], "language": "python", "metadata": {}, "outputs": [] }, { "cell_type": "code", "collapsed": false, "input": [ "np.corrcoeff(anscombe.x3, anscombe.y3)" ], "language": "python", "metadata": {}, "outputs": [] }, { "cell_type": "code", "collapsed": false, "input": [ "np.corrcoeff(anscombe.x4, anscombe.y4)" ], "language": "python", "metadata": {}, "outputs": [] }, { "cell_type": "code", "collapsed": false, "input": [ "bivariate_regression(anscombe.x2, anscombe.y2)" ], "language": "python", "metadata": {}, "outputs": [] }, { "cell_type": "markdown", "metadata": {}, "source": [ "# Using Pandas to work with table formatted data" ] }, { "cell_type": "code", "collapsed": false, "input": [ "import pandas as pd\n", "iris = pd.read_csv('iris.csv')" ], "language": "python", "metadata": {}, "outputs": [] }, { "cell_type": "code", "collapsed": false, "input": [ "type(iris)" ], "language": "python", "metadata": {}, "outputs": [] }, { "cell_type": "code", "collapsed": false, "input": [ "iris.columns" ], "language": "python", "metadata": {}, "outputs": [] }, { "cell_type": "code", "collapsed": false, "input": [ "# let's grab a picture from the web to remind ourselves\n", "# about flower morphology\n", "from IPython.display import display, Image\n", "i = Image('http://sebastianraschka.com/Images/2014_python_lda/iris_petal_sepal.png')\n", "display(i)" ], "language": "python", "metadata": {}, "outputs": [] }, { "cell_type": "code", "collapsed": false, "input": [ "# 'head' of the data set\n", "iris.head()" ], "language": "python", "metadata": {}, "outputs": [] }, { "cell_type": "code", "collapsed": false, "input": [ "# 'tail' of the data set\n", "iris.tail()" ], "language": "python", "metadata": {}, "outputs": [] }, { "cell_type": "code", "collapsed": false, "input": [ "# explicitly get first three rows\n", "iris[:3]" ], "language": "python", "metadata": {}, "outputs": [] }, { "cell_type": "code", "collapsed": false, "input": [ "# get last three rows\n", "iris[-3:]" ], "language": "python", "metadata": {}, "outputs": [] }, { "cell_type": "code", "collapsed": false, "input": [ "# get one column, by name\n", "iris['Species']" ], "language": "python", "metadata": {}, "outputs": [] }, { "cell_type": "code", "collapsed": false, "input": [ "# get multiple columns, by name\n", "iris[['Sepal Length', 'Petal Length']] " ], "language": "python", "metadata": {}, "outputs": [] }, { "cell_type": "code", "collapsed": false, "input": [ "# To index a data frame like a numpy array, use the iloc method\n", "iris.iloc[:5,:4] # first five rows, first 4 columns" ], "language": "python", "metadata": {}, "outputs": [] }, { "cell_type": "markdown", "metadata": {}, "source": [ "# Univariate plots" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "## Histograms and Density Plots" ] }, { "cell_type": "code", "collapsed": false, "input": [ "hist(iris['Sepal Length'])\n", "xlabel('Sepal Length')" ], "language": "python", "metadata": {}, "outputs": [] }, { "cell_type": "code", "collapsed": false, "input": [ "from scipy import stats\n", "# use a Gaussian kernel density estimator to estimate density distribution\n", "sl = iris['Sepal Length']\n", "sl_density = stats.kde.gaussian_kde(sl)\n", "\n", "# 100 evenly space points between min and max\n", "sl_min, sl_max = sl.min(), sl.max()\n", "sl_points = np.linspace(sl_min - 0.10*sl_min, sl_max + 0.10*sl_max, 100) \n", "\n", "# evaluate density estimator over range of values\n", "plot(sl_points, sl_density(sl_points), color='royalblue', linewidth=2)\n", "\n", "# compare to normed histogram\n", "hist(sl,normed=True, color='royalblue', alpha=0.5)\n", "ylabel('Density')" ], "language": "python", "metadata": {}, "outputs": [] }, { "cell_type": "code", "collapsed": false, "input": [ "# let's wrap up those steps into a function\n", "def estimate_density(x, bw=None, npts=100):\n", " densityfxn = stats.kde.gaussian_kde(x, bw_method=bw)\n", " xmin, xmax = min(x), max(x)\n", " evalpts = np.linspace(xmin - 0.15*xmin, xmax + 0.15*xmax, npts) \n", " estimate = densityfxn(evalpts)\n", " return evalpts, estimate\n", "\n", "slpts, sldensity = estimate_density(iris['Sepal Length'])\n", "# let's compare to the kde with a smaller bandwidth\n", "slpts_alt, sldensity_alt = estimate_density(iris['Sepal Length'], bw=0.25)\n", "\n", "plot(slpts, sldensity, color='crimson', lw=2)\n", "plot(slpts_alt, sldensity_alt, color='purple', lw=2)\n" ], "language": "python", "metadata": {}, "outputs": [] }, { "cell_type": "code", "collapsed": false, "input": [ "hist(iris['Sepal Length'], color='royalblue',alpha=0.5, normed=True)\n", "hist(iris['Sepal Width'], color='crimson',alpha=0.5, normed=True)\n", "None" ], "language": "python", "metadata": {}, "outputs": [] }, { "cell_type": "code", "collapsed": false, "input": [ "# too many histograms and the plot starts to get unreadable\n", "hist(iris['Sepal Length'], color='royalblue',alpha=0.5)\n", "hist(iris['Sepal Width'], color='crimson',alpha=0.5)\n", "hist(iris['Petal Length'], color='forestgreen',alpha=0.5)\n", "hist(iris['Petal Width'], color='darkmagenta',alpha=0.5)\n", "None" ], "language": "python", "metadata": {}, "outputs": [] }, { "cell_type": "code", "collapsed": false, "input": [ "# multiple density plots work a little better\n", "\n", "slpts, sldensity = estimate_density(iris['Sepal Length'])\n", "swpts, swdensity = estimate_density(iris['Sepal Width'])\n", "plpts, pldensity = estimate_density(iris['Petal Length'])\n", "pwpts, pwdensity = estimate_density(iris['Petal Width'])\n", "\n", "plot(slpts, sldensity, color='royalblue')\n", "plot(swpts, swdensity, color='crimson')\n", "plot(plpts, pldensity, color='forestgreen')\n", "plot(pwpts, pwdensity, color='darkmagenta')\n", "None\n", "\n" ], "language": "python", "metadata": {}, "outputs": [] }, { "cell_type": "markdown", "metadata": {}, "source": [ "## Boxplots, Violin Plots, Beanplots" ] }, { "cell_type": "code", "collapsed": false, "input": [ "# a boxplot is an alternative that works better for more than a few variables\n", "toshow = ['Sepal Length', 'Sepal Width', 'Petal Length', 'Petal Width']\n", "plt.boxplot(iris[toshow].values, labels=toshow)\n", "ylabel(\"Linear measurements (cm)\")\n", "None" ], "language": "python", "metadata": {}, "outputs": [] }, { "cell_type": "code", "collapsed": false, "input": [ "# a violin plot is an combines the information in a boxplot with a kernel desnity estimate\n", "toshow = ['Sepal Length', 'Sepal Width', 'Petal Length', 'Petal Width']\n", "plt.violinplot(iris[toshow].values)\n", "\n", "# violin plots are a recent addition to matplotlib, so they don't currently support\n", "# the labels argument found in boxplots. But we can add our own tick labels\n", "ax = gca()\n", "ax.set_xticks([1,2,3,4])\n", "ax.set_xticklabels(toshow)\n", "\n", "ylabel(\"Linear measurements (cm)\")" ], "language": "python", "metadata": {}, "outputs": [] }, { "cell_type": "markdown", "metadata": {}, "source": [ "A beanplot is similar to a violin plot, except it combines the density estimate with a line scatter plot of individual observations. Matplotlib doesn't have a built in beanplot function, but the package StatsModels (included when we installed the Anaconda Python Distribution) does have one." ] }, { "cell_type": "code", "collapsed": false, "input": [ "from statsmodels.graphics.boxplots import beanplot\n", "\n", "# the beanplot function expects each distn to be plotted in a sequence\n", "data = [iris[i] for i in toshow]\n", "ax = axes() \n", "# we explicitly pass the plot axes to beanplot\n", "# otherwise it will create to copies of the plot\n", "bp = beanplot(data, ax = ax, labels=toshow)" ], "language": "python", "metadata": {}, "outputs": [] }, { "cell_type": "markdown", "metadata": {}, "source": [ "# Bivariate Plots" ] }, { "cell_type": "code", "collapsed": false, "input": [ "axes(aspect='equal')\n", "scatter(iris['Sepal Length'], iris['Sepal Width'])\n", "xlabel(\"Sepal Length\")\n", "ylabel(\"Sepal Width\")" ], "language": "python", "metadata": {}, "outputs": [] }, { "cell_type": "code", "collapsed": false, "input": [ "# pandas includes a useful function, scatter_matrix to draw many pairwise scatter plots\n", "pd.tools.plotting.scatter_matrix(iris, figsize=(8,8), \n", " alpha=0.5, color='black', \n", " hist_kwds={'color':'grey', 'alpha':0.5})\n", "None" ], "language": "python", "metadata": {}, "outputs": [] }, { "cell_type": "markdown", "metadata": {}, "source": [ "## Labeling Subsets of the data" ] }, { "cell_type": "code", "collapsed": false, "input": [ "species = list(pd.unique(iris['Species']))" ], "language": "python", "metadata": {}, "outputs": [] }, { "cell_type": "code", "collapsed": false, "input": [ "species" ], "language": "python", "metadata": {}, "outputs": [] }, { "cell_type": "code", "collapsed": false, "input": [ "species_names = [\"I. \" + s for s in species]" ], "language": "python", "metadata": {}, "outputs": [] }, { "cell_type": "code", "collapsed": false, "input": [ "species_names" ], "language": "python", "metadata": {}, "outputs": [] }, { "cell_type": "code", "collapsed": false, "input": [ "colors = ['crimson','royalblue','forestgreen']" ], "language": "python", "metadata": {}, "outputs": [] }, { "cell_type": "code", "collapsed": false, "input": [ "ax = axes(aspect='equal')\n", "for i in range(3):\n", " subset = iris[iris['Species'] == species[i]]\n", " scatter(subset['Sepal Length'], subset['Sepal Width'], color=colors[i], alpha=0.4)\n", "\n", "xlabel(\"Sepal Length\")\n", "ylabel(\"Sepal Width\")\n", "legend(species_names, loc=\"upper right\", scatterpoints=1, fontsize=9)\n" ], "language": "python", "metadata": {}, "outputs": [] } ], "metadata": {} } ] }