{
 "metadata": {
  "name": "",
  "signature": "sha256:e5ba3c856cfd5375cef1d2d53af86aea926411dd442ff963016667bd1dff4353"
 },
 "nbformat": 3,
 "nbformat_minor": 0,
 "worksheets": [
  {
   "cells": [
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "import pandas\n",
      "import scipy.stats\n",
      "import json"
     ],
     "language": "python",
     "metadata": {},
     "outputs": [],
     "prompt_number": 16
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "import pywikibot\n",
      "#Tranforming QIDs into English labels.\n",
      "enwp = pywikibot.Site('en','wikipedia')\n",
      "wikidata = enwp.data_repository()\n",
      "\n",
      "retrieved = dict()\n",
      "\n",
      "def english_label(qid):\n",
      "    if type(qid) is float:\n",
      "        if math.isnan(qid):\n",
      "            return None\n",
      "    #first see if we've done it\n",
      "    try:\n",
      "        return retrieved[qid]\n",
      "    except KeyError:\n",
      "        try:\n",
      "            page = pywikibot.ItemPage(wikidata, qid)\n",
      "            data = page.get()\n",
      "            lab = data['labels']['en']\n",
      "            retrieved[qid] = lab\n",
      "            return lab\n",
      "        except KeyError:\n",
      "            retrieved[qid] = qid\n",
      "            return qid"
     ],
     "language": "python",
     "metadata": {},
     "outputs": [
      {
       "output_type": "stream",
       "stream": "stderr",
       "text": [
        "VERBOSE:pywiki:Starting 1 threads...\n"
       ]
      }
     ],
     "prompt_number": 22
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "# gen_cult = pandas.read_csv('helpers/Chi_Squared_Test_Data.csv') was doing it this way but no longer."
     ],
     "language": "python",
     "metadata": {},
     "outputs": [],
     "prompt_number": 25
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "allrecs = pandas.DataFrame.from_dict(json.load(open('helpers/world_cultures_shortcut.json','r')))"
     ],
     "language": "python",
     "metadata": {},
     "outputs": [],
     "prompt_number": 17
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "obs = pandas.crosstab(allrecs['culture'], allrecs['gender'])"
     ],
     "language": "python",
     "metadata": {},
     "outputs": [],
     "prompt_number": 19
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "obs.columns = map(english_label, obs.columns)"
     ],
     "language": "python",
     "metadata": {},
     "outputs": [
      {
       "output_type": "stream",
       "stream": "stderr",
       "text": [
        "VERBOSE:pywiki:Found 1 wikidata:wikidata processes running, including this one.\n"
       ]
      }
     ],
     "prompt_number": 23
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "obs"
     ],
     "language": "python",
     "metadata": {},
     "outputs": [
      {
       "html": [
        "<div style=\"max-height:1000px;max-width:1500px;overflow:auto;\">\n",
        "<table border=\"1\" class=\"dataframe\">\n",
        "  <thead>\n",
        "    <tr style=\"text-align: right;\">\n",
        "      <th></th>\n",
        "      <th>transgender female</th>\n",
        "      <th>intersex</th>\n",
        "      <th>fa'afafine</th>\n",
        "      <th>transgender male</th>\n",
        "      <th>male animal</th>\n",
        "      <th>woman</th>\n",
        "      <th>genderqueer</th>\n",
        "      <th>female</th>\n",
        "      <th>male</th>\n",
        "      <th>kathoey</th>\n",
        "    </tr>\n",
        "    <tr>\n",
        "      <th>culture</th>\n",
        "      <th></th>\n",
        "      <th></th>\n",
        "      <th></th>\n",
        "      <th></th>\n",
        "      <th></th>\n",
        "      <th></th>\n",
        "      <th></th>\n",
        "      <th></th>\n",
        "      <th></th>\n",
        "      <th></th>\n",
        "    </tr>\n",
        "  </thead>\n",
        "  <tbody>\n",
        "    <tr>\n",
        "      <th>africa</th>\n",
        "      <td>  0</td>\n",
        "      <td> 0</td>\n",
        "      <td> 0</td>\n",
        "      <td> 0</td>\n",
        "      <td> 0</td>\n",
        "      <td> 0</td>\n",
        "      <td> 0</td>\n",
        "      <td>  2256</td>\n",
        "      <td>  13915</td>\n",
        "      <td> 0</td>\n",
        "    </tr>\n",
        "    <tr>\n",
        "      <th>catholic european</th>\n",
        "      <td>  7</td>\n",
        "      <td> 2</td>\n",
        "      <td> 0</td>\n",
        "      <td> 1</td>\n",
        "      <td> 4</td>\n",
        "      <td> 0</td>\n",
        "      <td> 0</td>\n",
        "      <td> 38267</td>\n",
        "      <td> 262253</td>\n",
        "      <td> 0</td>\n",
        "    </tr>\n",
        "    <tr>\n",
        "      <th>confucian</th>\n",
        "      <td>  8</td>\n",
        "      <td> 0</td>\n",
        "      <td> 0</td>\n",
        "      <td> 0</td>\n",
        "      <td> 0</td>\n",
        "      <td> 0</td>\n",
        "      <td> 0</td>\n",
        "      <td>  5618</td>\n",
        "      <td>  14534</td>\n",
        "      <td> 0</td>\n",
        "    </tr>\n",
        "    <tr>\n",
        "      <th>english-speaking</th>\n",
        "      <td> 38</td>\n",
        "      <td> 7</td>\n",
        "      <td> 1</td>\n",
        "      <td> 9</td>\n",
        "      <td> 0</td>\n",
        "      <td> 0</td>\n",
        "      <td> 4</td>\n",
        "      <td> 60753</td>\n",
        "      <td> 223374</td>\n",
        "      <td> 0</td>\n",
        "    </tr>\n",
        "    <tr>\n",
        "      <th>islamic</th>\n",
        "      <td>  3</td>\n",
        "      <td> 0</td>\n",
        "      <td> 0</td>\n",
        "      <td> 0</td>\n",
        "      <td> 0</td>\n",
        "      <td> 0</td>\n",
        "      <td> 0</td>\n",
        "      <td>  4119</td>\n",
        "      <td>  22693</td>\n",
        "      <td> 0</td>\n",
        "    </tr>\n",
        "    <tr>\n",
        "      <th>latin america</th>\n",
        "      <td>  7</td>\n",
        "      <td> 0</td>\n",
        "      <td> 0</td>\n",
        "      <td> 0</td>\n",
        "      <td> 0</td>\n",
        "      <td> 0</td>\n",
        "      <td> 0</td>\n",
        "      <td> 11555</td>\n",
        "      <td>  64539</td>\n",
        "      <td> 0</td>\n",
        "    </tr>\n",
        "    <tr>\n",
        "      <th>orthodox</th>\n",
        "      <td>  1</td>\n",
        "      <td> 0</td>\n",
        "      <td> 0</td>\n",
        "      <td> 0</td>\n",
        "      <td> 1</td>\n",
        "      <td> 1</td>\n",
        "      <td> 0</td>\n",
        "      <td> 14178</td>\n",
        "      <td>  81513</td>\n",
        "      <td> 0</td>\n",
        "    </tr>\n",
        "    <tr>\n",
        "      <th>protestant european</th>\n",
        "      <td>  3</td>\n",
        "      <td> 1</td>\n",
        "      <td> 0</td>\n",
        "      <td> 2</td>\n",
        "      <td> 0</td>\n",
        "      <td> 0</td>\n",
        "      <td> 0</td>\n",
        "      <td> 49801</td>\n",
        "      <td> 244301</td>\n",
        "      <td> 0</td>\n",
        "    </tr>\n",
        "    <tr>\n",
        "      <th>south asia</th>\n",
        "      <td>  5</td>\n",
        "      <td> 0</td>\n",
        "      <td> 0</td>\n",
        "      <td> 0</td>\n",
        "      <td> 0</td>\n",
        "      <td> 0</td>\n",
        "      <td> 0</td>\n",
        "      <td>  5531</td>\n",
        "      <td>  23594</td>\n",
        "      <td> 1</td>\n",
        "    </tr>\n",
        "  </tbody>\n",
        "</table>\n",
        "</div>"
       ],
       "metadata": {},
       "output_type": "pyout",
       "prompt_number": 24,
       "text": [
        "                     transgender female  intersex  fa'afafine  \\\n",
        "culture                                                         \n",
        "africa                                0         0           0   \n",
        "catholic european                     7         2           0   \n",
        "confucian                             8         0           0   \n",
        "english-speaking                     38         7           1   \n",
        "islamic                               3         0           0   \n",
        "latin america                         7         0           0   \n",
        "orthodox                              1         0           0   \n",
        "protestant european                   3         1           0   \n",
        "south asia                            5         0           0   \n",
        "\n",
        "                     transgender male  male animal  woman  genderqueer  \\\n",
        "culture                                                                  \n",
        "africa                              0            0      0            0   \n",
        "catholic european                   1            4      0            0   \n",
        "confucian                           0            0      0            0   \n",
        "english-speaking                    9            0      0            4   \n",
        "islamic                             0            0      0            0   \n",
        "latin america                       0            0      0            0   \n",
        "orthodox                            0            1      1            0   \n",
        "protestant european                 2            0      0            0   \n",
        "south asia                          0            0      0            0   \n",
        "\n",
        "                     female    male  kathoey  \n",
        "culture                                       \n",
        "africa                 2256   13915        0  \n",
        "catholic european     38267  262253        0  \n",
        "confucian              5618   14534        0  \n",
        "english-speaking      60753  223374        0  \n",
        "islamic                4119   22693        0  \n",
        "latin america         11555   64539        0  \n",
        "orthodox              14178   81513        0  \n",
        "protestant european   49801  244301        0  \n",
        "south asia             5531   23594        1  "
       ]
      }
     ],
     "prompt_number": 24
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "chi2, p, dof, expected = scipy.stats.chi2_contingency(obs)"
     ],
     "language": "python",
     "metadata": {},
     "outputs": [],
     "prompt_number": 29
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "chi2, p, dof"
     ],
     "language": "python",
     "metadata": {},
     "outputs": [
      {
       "metadata": {},
       "output_type": "pyout",
       "prompt_number": 30,
       "text": [
        "(10430.455963736977, 0.0, 72)"
       ]
      }
     ],
     "prompt_number": 30
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "pandas.DataFrame(expected)"
     ],
     "language": "python",
     "metadata": {},
     "outputs": [
      {
       "html": [
        "<div style=\"max-height:1000px;max-width:1500px;overflow:auto;\">\n",
        "<table border=\"1\" class=\"dataframe\">\n",
        "  <thead>\n",
        "    <tr style=\"text-align: right;\">\n",
        "      <th></th>\n",
        "      <th>0</th>\n",
        "      <th>1</th>\n",
        "      <th>2</th>\n",
        "      <th>3</th>\n",
        "      <th>4</th>\n",
        "      <th>5</th>\n",
        "      <th>6</th>\n",
        "      <th>7</th>\n",
        "      <th>8</th>\n",
        "      <th>9</th>\n",
        "    </tr>\n",
        "  </thead>\n",
        "  <tbody>\n",
        "    <tr>\n",
        "      <th>0</th>\n",
        "      <td>  1.018735</td>\n",
        "      <td> 0.141491</td>\n",
        "      <td> 0.014149</td>\n",
        "      <td> 0.169789</td>\n",
        "      <td> 0.070745</td>\n",
        "      <td> 0.014149</td>\n",
        "      <td> 0.056596</td>\n",
        "      <td>  2717.729756</td>\n",
        "      <td>  13451.770440</td>\n",
        "      <td> 0.014149</td>\n",
        "    </tr>\n",
        "    <tr>\n",
        "      <th>1</th>\n",
        "      <td> 18.932932</td>\n",
        "      <td> 2.629574</td>\n",
        "      <td> 0.262957</td>\n",
        "      <td> 3.155489</td>\n",
        "      <td> 1.314787</td>\n",
        "      <td> 0.262957</td>\n",
        "      <td> 1.051830</td>\n",
        "      <td> 50508.329383</td>\n",
        "      <td> 249997.797134</td>\n",
        "      <td> 0.262957</td>\n",
        "    </tr>\n",
        "    <tr>\n",
        "      <th>2</th>\n",
        "      <td>  1.270032</td>\n",
        "      <td> 0.176393</td>\n",
        "      <td> 0.017639</td>\n",
        "      <td> 0.211672</td>\n",
        "      <td> 0.088197</td>\n",
        "      <td> 0.017639</td>\n",
        "      <td> 0.070557</td>\n",
        "      <td>  3388.128865</td>\n",
        "      <td>  16770.001365</td>\n",
        "      <td> 0.017639</td>\n",
        "    </tr>\n",
        "    <tr>\n",
        "      <th>3</th>\n",
        "      <td> 17.903047</td>\n",
        "      <td> 2.486534</td>\n",
        "      <td> 0.248653</td>\n",
        "      <td> 2.983841</td>\n",
        "      <td> 1.243267</td>\n",
        "      <td> 0.248653</td>\n",
        "      <td> 0.994614</td>\n",
        "      <td> 47760.852663</td>\n",
        "      <td> 236398.790074</td>\n",
        "      <td> 0.248653</td>\n",
        "    </tr>\n",
        "    <tr>\n",
        "      <th>4</th>\n",
        "      <td>  1.689282</td>\n",
        "      <td> 0.234622</td>\n",
        "      <td> 0.023462</td>\n",
        "      <td> 0.281547</td>\n",
        "      <td> 0.117311</td>\n",
        "      <td> 0.023462</td>\n",
        "      <td> 0.093849</td>\n",
        "      <td>  4506.581127</td>\n",
        "      <td>  22305.931875</td>\n",
        "      <td> 0.023462</td>\n",
        "    </tr>\n",
        "    <tr>\n",
        "      <th>5</th>\n",
        "      <td>  4.794183</td>\n",
        "      <td> 0.665859</td>\n",
        "      <td> 0.066586</td>\n",
        "      <td> 0.799031</td>\n",
        "      <td> 0.332929</td>\n",
        "      <td> 0.066586</td>\n",
        "      <td> 0.266344</td>\n",
        "      <td> 12789.682280</td>\n",
        "      <td>  63304.259617</td>\n",
        "      <td> 0.066586</td>\n",
        "    </tr>\n",
        "    <tr>\n",
        "      <th>6</th>\n",
        "      <td>  6.028496</td>\n",
        "      <td> 0.837291</td>\n",
        "      <td> 0.083729</td>\n",
        "      <td> 1.004749</td>\n",
        "      <td> 0.418646</td>\n",
        "      <td> 0.083729</td>\n",
        "      <td> 0.334916</td>\n",
        "      <td> 16082.520021</td>\n",
        "      <td>  79602.604693</td>\n",
        "      <td> 0.083729</td>\n",
        "    </tr>\n",
        "    <tr>\n",
        "      <th>7</th>\n",
        "      <td> 18.528109</td>\n",
        "      <td> 2.573348</td>\n",
        "      <td> 0.257335</td>\n",
        "      <td> 3.088018</td>\n",
        "      <td> 1.286674</td>\n",
        "      <td> 0.257335</td>\n",
        "      <td> 1.029339</td>\n",
        "      <td> 49428.363307</td>\n",
        "      <td> 244652.359199</td>\n",
        "      <td> 0.257335</td>\n",
        "    </tr>\n",
        "    <tr>\n",
        "      <th>8</th>\n",
        "      <td>  1.835184</td>\n",
        "      <td> 0.254887</td>\n",
        "      <td> 0.025489</td>\n",
        "      <td> 0.305864</td>\n",
        "      <td> 0.127443</td>\n",
        "      <td> 0.025489</td>\n",
        "      <td> 0.101955</td>\n",
        "      <td>  4895.812598</td>\n",
        "      <td>  24232.485603</td>\n",
        "      <td> 0.025489</td>\n",
        "    </tr>\n",
        "  </tbody>\n",
        "</table>\n",
        "</div>"
       ],
       "metadata": {},
       "output_type": "pyout",
       "prompt_number": 28,
       "text": [
        "           0         1         2         3         4         5         6  \\\n",
        "0   1.018735  0.141491  0.014149  0.169789  0.070745  0.014149  0.056596   \n",
        "1  18.932932  2.629574  0.262957  3.155489  1.314787  0.262957  1.051830   \n",
        "2   1.270032  0.176393  0.017639  0.211672  0.088197  0.017639  0.070557   \n",
        "3  17.903047  2.486534  0.248653  2.983841  1.243267  0.248653  0.994614   \n",
        "4   1.689282  0.234622  0.023462  0.281547  0.117311  0.023462  0.093849   \n",
        "5   4.794183  0.665859  0.066586  0.799031  0.332929  0.066586  0.266344   \n",
        "6   6.028496  0.837291  0.083729  1.004749  0.418646  0.083729  0.334916   \n",
        "7  18.528109  2.573348  0.257335  3.088018  1.286674  0.257335  1.029339   \n",
        "8   1.835184  0.254887  0.025489  0.305864  0.127443  0.025489  0.101955   \n",
        "\n",
        "              7              8         9  \n",
        "0   2717.729756   13451.770440  0.014149  \n",
        "1  50508.329383  249997.797134  0.262957  \n",
        "2   3388.128865   16770.001365  0.017639  \n",
        "3  47760.852663  236398.790074  0.248653  \n",
        "4   4506.581127   22305.931875  0.023462  \n",
        "5  12789.682280   63304.259617  0.066586  \n",
        "6  16082.520021   79602.604693  0.083729  \n",
        "7  49428.363307  244652.359199  0.257335  \n",
        "8   4895.812598   24232.485603  0.025489  "
       ]
      }
     ],
     "prompt_number": 28
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "np.ndarr"
     ],
     "language": "python",
     "metadata": {},
     "outputs": []
    }
   ],
   "metadata": {}
  }
 ]
}