{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Load Packages"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 440,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "4857"
      ]
     },
     "execution_count": 440,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "3522 + 35 + 1032 + 62 + 206"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "import sys\n",
    "import time\n",
    "import json"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "import numpy as np\n",
    "import pandas as pd\n",
    "import matplotlib.pyplot as plt\n",
    "import sqlite3 as lite\n",
    "\n",
    "from pandas import DataFrame\n",
    "from pandas import Series"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "#http://pandas.pydata.org/pandas-docs/stable/options.html\n",
    "pd.set_option('display.max_columns', None)\n",
    "pd.set_option('max_colwidth', 250)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 92,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "from __future__ import division"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "collapsed": true
   },
   "source": [
    "#### Set working directory"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "/Users/gregorysaxton/Google Drive/SOX\n"
     ]
    }
   ],
   "source": [
    "cd '/Users/gregorysaxton/Google Drive/SOX'"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Create List of EINs for '2016' charities -- I'll use this to create a list of EINs for matching with e-file database\n",
    "- Previous (Read in Saved List of EINs with 'current' (2016) donor advisories)\n",
    "- What I'm doing here is reading in the Stata 'Table 5' dataset so that I can create a list of the 8,304 relevant EINs."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "#f = open('2016 donor advisory EINs.json', 'r')\n",
    "#advisories_2016 = json.load(f)\n",
    "#print len(advisories_2016)\n",
    "#print advisories_2016[:10]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "#df = pd.read_pickle('2016 - Test 4 data.pkl')\n",
    "#print \"Number of columns:\", len(df.columns)\n",
    "#print \"Number of observations:\", len(df)\n",
    "#df.head(1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "#print len(set(df['EIN'].tolist()))\n",
    "#eins_2016 = df['EIN'].tolist()\n",
    "#print len(eins_2016), len(set(eins_2016))\n",
    "#print eins_2016[:5]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Read in Test 4 data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 393,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Number of columns: 35\n",
      "Number of observations: 8304\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>donor_advisory</th>\n",
       "      <th>donor_advisory_2016</th>\n",
       "      <th>donor_advisory_2011_to_2016</th>\n",
       "      <th>org_id</th>\n",
       "      <th>EIN</th>\n",
       "      <th>FYE</th>\n",
       "      <th>Form 990 FYE</th>\n",
       "      <th>ratings_system</th>\n",
       "      <th>2011_data</th>\n",
       "      <th>2016_data</th>\n",
       "      <th>conflict_of_interest_policy_v2</th>\n",
       "      <th>records_retention_policy_v2</th>\n",
       "      <th>whistleblower_policy_v2</th>\n",
       "      <th>SOX_policies</th>\n",
       "      <th>SOX_policies_binary</th>\n",
       "      <th>SOX_policies_all_binary</th>\n",
       "      <th>program_efficiency</th>\n",
       "      <th>complexity</th>\n",
       "      <th>complexity_2011</th>\n",
       "      <th>age</th>\n",
       "      <th>total_revenue_logged</th>\n",
       "      <th>category</th>\n",
       "      <th>state</th>\n",
       "      <th>tot_rev</th>\n",
       "      <th>category_Animals</th>\n",
       "      <th>category_Arts, Culture, Humanities</th>\n",
       "      <th>category_Community Development</th>\n",
       "      <th>category_Education</th>\n",
       "      <th>category_Environment</th>\n",
       "      <th>category_Health</th>\n",
       "      <th>category_Human Services</th>\n",
       "      <th>category_Human and Civil Rights</th>\n",
       "      <th>category_International</th>\n",
       "      <th>category_Religion</th>\n",
       "      <th>category_Research and Public Policy</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>50709</th>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>5954</td>\n",
       "      <td>010202467</td>\n",
       "      <td>FY2014</td>\n",
       "      <td>2014-12</td>\n",
       "      <td>CN 2.1</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>3.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.794457</td>\n",
       "      <td>6.0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>62.0</td>\n",
       "      <td>16.377993</td>\n",
       "      <td>Research and Public Policy</td>\n",
       "      <td>ME</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "       donor_advisory  donor_advisory_2016  donor_advisory_2011_to_2016  \\\n",
       "50709             0.0                  0.0                          0.0   \n",
       "\n",
       "      org_id        EIN     FYE Form 990 FYE ratings_system  2011_data  \\\n",
       "50709   5954  010202467  FY2014      2014-12         CN 2.1        0.0   \n",
       "\n",
       "       2016_data  conflict_of_interest_policy_v2  records_retention_policy_v2  \\\n",
       "50709        1.0                             1.0                          1.0   \n",
       "\n",
       "       whistleblower_policy_v2  SOX_policies  SOX_policies_binary  \\\n",
       "50709                      1.0           3.0                  1.0   \n",
       "\n",
       "       SOX_policies_all_binary  program_efficiency  complexity  \\\n",
       "50709                      1.0            0.794457         6.0   \n",
       "\n",
       "       complexity_2011   age  total_revenue_logged  \\\n",
       "50709              NaN  62.0             16.377993   \n",
       "\n",
       "                         category state  tot_rev  category_Animals  \\\n",
       "50709  Research and Public Policy    ME      NaN               0.0   \n",
       "\n",
       "       category_Arts, Culture, Humanities  category_Community Development  \\\n",
       "50709                                 0.0                             0.0   \n",
       "\n",
       "       category_Education  category_Environment  category_Health  \\\n",
       "50709                 0.0                   0.0              0.0   \n",
       "\n",
       "       category_Human Services  category_Human and Civil Rights  \\\n",
       "50709                      0.0                              0.0   \n",
       "\n",
       "       category_International  category_Religion  \\\n",
       "50709                     0.0                0.0   \n",
       "\n",
       "       category_Research and Public Policy  \n",
       "50709                                  1.0  "
      ]
     },
     "execution_count": 393,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df = pd.read_pickle('2016 - Test 4 data.pkl')\n",
    "print \"Number of columns:\", len(df.columns)\n",
    "print \"Number of observations:\", len(df)\n",
    "df.head(1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "8304\n",
      "8304 8304\n",
      "['010202467', '010211478', '010211513', '010211530', '010211543']\n"
     ]
    }
   ],
   "source": [
    "print len(set(df['EIN'].tolist()))\n",
    "eins_2016 = df['EIN'].tolist()\n",
    "print len(eins_2016), len(set(eins_2016))\n",
    "print eins_2016[:5]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "print len(eins_2016)\n",
    "with open('eins_2016.json', 'w') as outfile:\n",
    "    json.dump(eins_2016, outfile)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "#import json\n",
    "#with open('valid_next_v8.json', 'r') as fp:\n",
    "#    valid_next = json.load(fp)\n",
    "#print len(valid_next)\n",
    "#print valid_next[:5]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Read DB into PANDAS DF\n",
    "- I'm taking all 29,440 990 e-filings for our 8,304 EINs and importing them from MongoDB into a Python PANDAS dataframe for manipulation.\n",
    "- As you can see if you scroll horizontally through this sample row, there are 691 columns!"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "# of columns: 695\n",
      "# of observations: 29440\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>@documentId</th>\n",
       "      <th>@referenceDocumentId</th>\n",
       "      <th>@referenceDocumentName</th>\n",
       "      <th>@softwareId</th>\n",
       "      <th>@softwareVersion</th>\n",
       "      <th>@softwareVersionNum</th>\n",
       "      <th>AccountantCompileOrReview</th>\n",
       "      <th>AccountantCompileOrReviewBasis</th>\n",
       "      <th>AccountantCompileOrReviewInd</th>\n",
       "      <th>AccountsPayableAccrExpnssGrp</th>\n",
       "      <th>AccountsPayableAccruedExpenses</th>\n",
       "      <th>AccountsReceivable</th>\n",
       "      <th>AccountsReceivableGrp</th>\n",
       "      <th>AcctCompileOrReviewBasisGrp</th>\n",
       "      <th>ActivitiesConductedPartnership</th>\n",
       "      <th>ActivitiesConductedPrtshpInd</th>\n",
       "      <th>Activity2</th>\n",
       "      <th>Activity3</th>\n",
       "      <th>ActivityCd</th>\n",
       "      <th>ActivityCode</th>\n",
       "      <th>ActivityOrMissionDesc</th>\n",
       "      <th>ActivityOrMissionDescription</th>\n",
       "      <th>ActivityOther</th>\n",
       "      <th>AddressChange</th>\n",
       "      <th>AddressChangeInd</th>\n",
       "      <th>AddressPrincipalOfficerForeign</th>\n",
       "      <th>AddressPrincipalOfficerUS</th>\n",
       "      <th>Advertising</th>\n",
       "      <th>AdvertisingGrp</th>\n",
       "      <th>AllAffiliatesIncluded</th>\n",
       "      <th>AllAffiliatesIncludedInd</th>\n",
       "      <th>AllOtherContributions</th>\n",
       "      <th>AllOtherContributionsAmt</th>\n",
       "      <th>AllOtherExpenses</th>\n",
       "      <th>AllOtherExpensesGrp</th>\n",
       "      <th>AmendedReturn</th>\n",
       "      <th>AmendedReturnInd</th>\n",
       "      <th>AnnualDisclosureCoveredPersons</th>\n",
       "      <th>AnnualDisclosureCoveredPrsnInd</th>\n",
       "      <th>AuditCommittee</th>\n",
       "      <th>AuditCommitteeInd</th>\n",
       "      <th>AuditedFinancialStmtAttInd</th>\n",
       "      <th>BackupWthldComplianceInd</th>\n",
       "      <th>BalanceSheetAmountsReported</th>\n",
       "      <th>BenefitsPaidToMembersCY</th>\n",
       "      <th>BenefitsPaidToMembersPriorYear</th>\n",
       "      <th>BenefitsToMembers</th>\n",
       "      <th>BenefitsToMembersGrp</th>\n",
       "      <th>BooksInCareOfDetail</th>\n",
       "      <th>BsnssRltnshpThruFamilyMember</th>\n",
       "      <th>BsnssRltnshpWithOrganization</th>\n",
       "      <th>BusinessRlnWithFamMemInd</th>\n",
       "      <th>BusinessRlnWithOfficerEntInd</th>\n",
       "      <th>BusinessRlnWithOrgMemInd</th>\n",
       "      <th>CYBenefitsPaidToMembersAmt</th>\n",
       "      <th>CYContributionsGrantsAmt</th>\n",
       "      <th>CYGrantsAndSimilarPaidAmt</th>\n",
       "      <th>CYInvestmentIncomeAmt</th>\n",
       "      <th>CYOtherExpensesAmt</th>\n",
       "      <th>CYOtherRevenueAmt</th>\n",
       "      <th>CYProgramServiceRevenueAmt</th>\n",
       "      <th>CYRevenuesLessExpensesAmt</th>\n",
       "      <th>CYSalariesCompEmpBnftPaidAmt</th>\n",
       "      <th>CYTotalExpensesAmt</th>\n",
       "      <th>CYTotalFundraisingExpenseAmt</th>\n",
       "      <th>CYTotalProfFndrsngExpnsAmt</th>\n",
       "      <th>CYTotalRevenueAmt</th>\n",
       "      <th>CapStckTrstPrinCurrentFunds</th>\n",
       "      <th>CapStkTrPrinCurrentFundsGrp</th>\n",
       "      <th>CashNonInterestBearing</th>\n",
       "      <th>CashNonInterestBearingGrp</th>\n",
       "      <th>ChangeToOrgDocumentsInd</th>\n",
       "      <th>ChangesToOrganizingDocs</th>\n",
       "      <th>CntrbtnsRprtdFundraisingEvents</th>\n",
       "      <th>CntrctRcvdGreaterThan100KCnt</th>\n",
       "      <th>CollectionsOfArt</th>\n",
       "      <th>CollectionsOfArtInd</th>\n",
       "      <th>CompCurrentOfcrDirectorsGrp</th>\n",
       "      <th>CompCurrentOfficersDirectors</th>\n",
       "      <th>CompDisqualPersons</th>\n",
       "      <th>CompDisqualPersonsGrp</th>\n",
       "      <th>CompensationFromOtherSources</th>\n",
       "      <th>CompensationFromOtherSrcsInd</th>\n",
       "      <th>CompensationProcessCEO</th>\n",
       "      <th>CompensationProcessCEOInd</th>\n",
       "      <th>CompensationProcessOther</th>\n",
       "      <th>CompensationProcessOtherInd</th>\n",
       "      <th>ComplianceWithBackupWitholding</th>\n",
       "      <th>ConferencesMeetings</th>\n",
       "      <th>ConferencesMeetingsGrp</th>\n",
       "      <th>ConflictOfInterestPolicy</th>\n",
       "      <th>ConflictOfInterestPolicyInd</th>\n",
       "      <th>ConservationEasements</th>\n",
       "      <th>ConservationEasementsInd</th>\n",
       "      <th>ConsolidatedAuditFinancialStmt</th>\n",
       "      <th>ConsolidatedAuditFinclStmtInd</th>\n",
       "      <th>ContractTerminationInd</th>\n",
       "      <th>ContractorCompensation</th>\n",
       "      <th>ContractorCompensationGrp</th>\n",
       "      <th>ContriRptFundraisingEventAmt</th>\n",
       "      <th>ContributionsGrantsCurrentYear</th>\n",
       "      <th>ContributionsGrantsPriorYear</th>\n",
       "      <th>CostOfGoodsSold</th>\n",
       "      <th>CostOfGoodsSoldAmt</th>\n",
       "      <th>CountryLegalDomicile</th>\n",
       "      <th>CreditCounseling</th>\n",
       "      <th>CreditCounselingInd</th>\n",
       "      <th>DAFExcessBusinessHoldingsInd</th>\n",
       "      <th>DLN</th>\n",
       "      <th>DecisionsSubjectToApprovaInd</th>\n",
       "      <th>DecisionsSubjectToApproval</th>\n",
       "      <th>DeductibleArtContributionInd</th>\n",
       "      <th>DeductibleContributionsOfArt</th>\n",
       "      <th>DeductibleNonCashContriInd</th>\n",
       "      <th>DeductibleNonCashContributions</th>\n",
       "      <th>DeferredRevenue</th>\n",
       "      <th>DeferredRevenueGrp</th>\n",
       "      <th>DelegationOfManagementDuties</th>\n",
       "      <th>DelegationOfMgmtDutiesInd</th>\n",
       "      <th>DepreciationDepletion</th>\n",
       "      <th>DepreciationDepletionGrp</th>\n",
       "      <th>Desc</th>\n",
       "      <th>DescribedIn501C3</th>\n",
       "      <th>DescribedInSection501c3Ind</th>\n",
       "      <th>Description</th>\n",
       "      <th>DisregardedEntity</th>\n",
       "      <th>DisregardedEntityInd</th>\n",
       "      <th>DistributionToDonor</th>\n",
       "      <th>DistributionToDonorInd</th>\n",
       "      <th>DoNotFollowSFAS117</th>\n",
       "      <th>DocumentRetentionPolicy</th>\n",
       "      <th>DocumentRetentionPolicyInd</th>\n",
       "      <th>DoingBusinessAs</th>\n",
       "      <th>DoingBusinessAsName</th>\n",
       "      <th>DonatedServicesAndUseFcltsAmt</th>\n",
       "      <th>DonorAdvisedFundInd</th>\n",
       "      <th>DonorAdvisedFunds</th>\n",
       "      <th>EIN</th>\n",
       "      <th>ElectionOfBoardMembers</th>\n",
       "      <th>ElectionOfBoardMembersInd</th>\n",
       "      <th>EmployeeCnt</th>\n",
       "      <th>EmploymentTaxReturnsFiled</th>\n",
       "      <th>EmploymentTaxReturnsFiledInd</th>\n",
       "      <th>EngagedInExcessBenefitTransInd</th>\n",
       "      <th>EscrowAccount</th>\n",
       "      <th>EscrowAccountInd</th>\n",
       "      <th>EscrowAccountLiability</th>\n",
       "      <th>EscrowAccountLiabilityGrp</th>\n",
       "      <th>ExcessBenefitTransaction</th>\n",
       "      <th>ExcessBusinessHoldings</th>\n",
       "      <th>ExcessBusinessHoldingsInd</th>\n",
       "      <th>Expense</th>\n",
       "      <th>ExpenseAmt</th>\n",
       "      <th>FSAudited</th>\n",
       "      <th>FSAuditedBasis</th>\n",
       "      <th>FSAuditedBasisGrp</th>\n",
       "      <th>FSAuditedInd</th>\n",
       "      <th>FamilyOrBusinessRelationship</th>\n",
       "      <th>FamilyOrBusinessRlnInd</th>\n",
       "      <th>FederalGrantAuditPerformed</th>\n",
       "      <th>FederalGrantAuditPerformedInd</th>\n",
       "      <th>FederalGrantAuditRequired</th>\n",
       "      <th>FederalGrantAuditRequiredInd</th>\n",
       "      <th>FederatedCampaigns</th>\n",
       "      <th>FederatedCampaignsAmt</th>\n",
       "      <th>FeesForServicesAccounting</th>\n",
       "      <th>FeesForServicesAccountingGrp</th>\n",
       "      <th>FeesForServicesInvstMgmntFees</th>\n",
       "      <th>FeesForServicesLegal</th>\n",
       "      <th>FeesForServicesLegalGrp</th>\n",
       "      <th>FeesForServicesLobbying</th>\n",
       "      <th>FeesForServicesLobbyingGrp</th>\n",
       "      <th>FeesForServicesManagement</th>\n",
       "      <th>FeesForServicesManagementGrp</th>\n",
       "      <th>FeesForServicesOther</th>\n",
       "      <th>FeesForServicesOtherGrp</th>\n",
       "      <th>FeesForServicesProfFundraising</th>\n",
       "      <th>FeesForSrvcInvstMgmntFeesGrp</th>\n",
       "      <th>FiledLieu1041</th>\n",
       "      <th>FinalReturnInd</th>\n",
       "      <th>FinancialStatementBoth</th>\n",
       "      <th>FinancialStatementConsolidated</th>\n",
       "      <th>FinancialStatementSeparate</th>\n",
       "      <th>FinancialStmtAttached</th>\n",
       "      <th>FollowSFAS117</th>\n",
       "      <th>ForeignActivities</th>\n",
       "      <th>ForeignActivitiesInd</th>\n",
       "      <th>ForeignAddress</th>\n",
       "      <th>ForeignCountryCd</th>\n",
       "      <th>ForeignFinancialAccount</th>\n",
       "      <th>ForeignFinancialAccountInd</th>\n",
       "      <th>ForeignGrants</th>\n",
       "      <th>ForeignGrantsGrp</th>\n",
       "      <th>ForeignOffice</th>\n",
       "      <th>ForeignOfficeInd</th>\n",
       "      <th>Form1098CFiled</th>\n",
       "      <th>Form1098CFiledInd</th>\n",
       "      <th>Form720Filed</th>\n",
       "      <th>Form8282FiledCnt</th>\n",
       "      <th>Form8282PropertyDisposedOf</th>\n",
       "      <th>Form8282PropertyDisposedOfInd</th>\n",
       "      <th>Form8886TFiled</th>\n",
       "      <th>Form8886TFiledInd</th>\n",
       "      <th>Form8899Filed</th>\n",
       "      <th>Form8899Filedind</th>\n",
       "      <th>Form990-TFiled</th>\n",
       "      <th>Form990PartVIISectionA</th>\n",
       "      <th>Form990PartVIISectionAGrp</th>\n",
       "      <th>Form990ProvidedToGoverningBody</th>\n",
       "      <th>Form990ProvidedToGvrnBodyInd</th>\n",
       "      <th>Form990TFiledInd</th>\n",
       "      <th>FormType</th>\n",
       "      <th>FormationYr</th>\n",
       "      <th>FormerOfcrEmployeesListedInd</th>\n",
       "      <th>FormersListed</th>\n",
       "      <th>FundraisingActivities</th>\n",
       "      <th>FundraisingActivitiesInd</th>\n",
       "      <th>FundraisingAmt</th>\n",
       "      <th>FundraisingDirectExpenses</th>\n",
       "      <th>FundraisingDirectExpensesAmt</th>\n",
       "      <th>FundraisingEvents</th>\n",
       "      <th>FundraisingGrossIncomeAmt</th>\n",
       "      <th>FundsToPayPremiums</th>\n",
       "      <th>GainOrLoss</th>\n",
       "      <th>GainOrLossGrp</th>\n",
       "      <th>Gaming</th>\n",
       "      <th>GamingActivitiesInd</th>\n",
       "      <th>GamingDirectExpenses</th>\n",
       "      <th>GamingDirectExpensesAmt</th>\n",
       "      <th>GamingGrossIncomeAmt</th>\n",
       "      <th>GoverningBodyVotingMembersCnt</th>\n",
       "      <th>GovernmentGrants</th>\n",
       "      <th>GovernmentGrantsAmt</th>\n",
       "      <th>GrantAmt</th>\n",
       "      <th>GrantToRelatedPerson</th>\n",
       "      <th>GrantToRelatedPersonInd</th>\n",
       "      <th>Grants</th>\n",
       "      <th>GrantsAndSimilarAmntsCY</th>\n",
       "      <th>GrantsAndSimilarAmntsPriorYear</th>\n",
       "      <th>GrantsPayable</th>\n",
       "      <th>GrantsPayableGrp</th>\n",
       "      <th>GrantsToDomesticIndividuals</th>\n",
       "      <th>GrantsToDomesticIndividualsGrp</th>\n",
       "      <th>GrantsToDomesticOrgs</th>\n",
       "      <th>GrantsToDomesticOrgsGrp</th>\n",
       "      <th>GrantsToIndividuals</th>\n",
       "      <th>GrantsToIndividualsInd</th>\n",
       "      <th>GrantsToOrganizations</th>\n",
       "      <th>GrantsToOrganizationsInd</th>\n",
       "      <th>GrossAmountSalesAssets</th>\n",
       "      <th>GrossAmountSalesAssetsGrp</th>\n",
       "      <th>GrossIncomeFundraisingEvents</th>\n",
       "      <th>GrossIncomeGaming</th>\n",
       "      <th>GrossReceipts</th>\n",
       "      <th>GrossReceiptsAmt</th>\n",
       "      <th>GrossRents</th>\n",
       "      <th>GrossRentsGrp</th>\n",
       "      <th>GrossSalesOfInventory</th>\n",
       "      <th>GrossSalesOfInventoryAmt</th>\n",
       "      <th>GroupExemptionNum</th>\n",
       "      <th>GroupExemptionNumber</th>\n",
       "      <th>GroupReturnForAffiliates</th>\n",
       "      <th>GroupReturnForAffiliatesInd</th>\n",
       "      <th>Hospital</th>\n",
       "      <th>IRPDocumentCnt</th>\n",
       "      <th>IRPDocumentW2GCnt</th>\n",
       "      <th>IncludeFIN48FootnoteInd</th>\n",
       "      <th>IncmFromInvestBondProceedsGrp</th>\n",
       "      <th>IncomeFromInvestBondProceeds</th>\n",
       "      <th>IndependentAuditFinancialStmt</th>\n",
       "      <th>IndependentAuditFinclStmtInd</th>\n",
       "      <th>IndependentVotingMemberCnt</th>\n",
       "      <th>IndivRcvdGreaterThan100KCnt</th>\n",
       "      <th>IndoorTanningServices</th>\n",
       "      <th>IndoorTanningServicesInd</th>\n",
       "      <th>InfoInScheduleOPartIII</th>\n",
       "      <th>InfoInScheduleOPartIIIInd</th>\n",
       "      <th>InfoInScheduleOPartIX</th>\n",
       "      <th>InfoInScheduleOPartIXInd</th>\n",
       "      <th>InfoInScheduleOPartV</th>\n",
       "      <th>InfoInScheduleOPartVI</th>\n",
       "      <th>InfoInScheduleOPartVII</th>\n",
       "      <th>InfoInScheduleOPartVIII</th>\n",
       "      <th>InfoInScheduleOPartVIIIInd</th>\n",
       "      <th>InfoInScheduleOPartVIIInd</th>\n",
       "      <th>InfoInScheduleOPartVIInd</th>\n",
       "      <th>InfoInScheduleOPartVInd</th>\n",
       "      <th>InfoInScheduleOPartX</th>\n",
       "      <th>InfoInScheduleOPartXI</th>\n",
       "      <th>InfoInScheduleOPartXII</th>\n",
       "      <th>InfoInScheduleOPartXIIInd</th>\n",
       "      <th>InfoInScheduleOPartXIInd</th>\n",
       "      <th>InfoInScheduleOPartXInd</th>\n",
       "      <th>InformationTechnology</th>\n",
       "      <th>InformationTechnologyGrp</th>\n",
       "      <th>InitialReturn</th>\n",
       "      <th>InitialReturnInd</th>\n",
       "      <th>Insurance</th>\n",
       "      <th>InsuranceGrp</th>\n",
       "      <th>IntangibleAssets</th>\n",
       "      <th>IntangibleAssetsGrp</th>\n",
       "      <th>Interest</th>\n",
       "      <th>InterestGrp</th>\n",
       "      <th>InventoriesForSaleOrUse</th>\n",
       "      <th>InventoriesForSaleOrUseGrp</th>\n",
       "      <th>InvestTaxExemptBonds</th>\n",
       "      <th>InvestTaxExemptBondsInd</th>\n",
       "      <th>InvestmentExpenseAmt</th>\n",
       "      <th>InvestmentInJointVenture</th>\n",
       "      <th>InvestmentInJointVentureInd</th>\n",
       "      <th>InvestmentIncome</th>\n",
       "      <th>InvestmentIncomeCurrentYear</th>\n",
       "      <th>InvestmentIncomeGrp</th>\n",
       "      <th>InvestmentIncomePriorYear</th>\n",
       "      <th>InvestmentsOtherSecurities</th>\n",
       "      <th>InvestmentsOtherSecuritiesGrp</th>\n",
       "      <th>InvestmentsProgramRelated</th>\n",
       "      <th>InvestmentsProgramRelatedGrp</th>\n",
       "      <th>InvestmentsPubTradedSecGrp</th>\n",
       "      <th>InvestmentsPubTradedSecurities</th>\n",
       "      <th>IsAvailable</th>\n",
       "      <th>IsElectronic</th>\n",
       "      <th>JointCosts</th>\n",
       "      <th>JointCostsInd</th>\n",
       "      <th>LandBldgEquipAccumDeprecAmt</th>\n",
       "      <th>LandBldgEquipBasisNetGrp</th>\n",
       "      <th>LandBldgEquipCostOrOtherBssAmt</th>\n",
       "      <th>LandBldgEquipmentAccumDeprec</th>\n",
       "      <th>LandBuildingsEquipmentBasis</th>\n",
       "      <th>LandBuildingsEquipmentBasisNet</th>\n",
       "      <th>LastUpdated</th>\n",
       "      <th>LegalDomicileStateCd</th>\n",
       "      <th>LessCostOthBasisSalesExpenses</th>\n",
       "      <th>LessCostOthBasisSalesExpnssGrp</th>\n",
       "      <th>LessRentalExpenses</th>\n",
       "      <th>LessRentalExpensesGrp</th>\n",
       "      <th>LicensedMoreThanOneState</th>\n",
       "      <th>LicensedMoreThanOneStateInd</th>\n",
       "      <th>LoanOutstandingInd</th>\n",
       "      <th>LoanToOfficerOrDQP</th>\n",
       "      <th>LoansFromOfficersDirectors</th>\n",
       "      <th>LoansFromOfficersDirectorsGrp</th>\n",
       "      <th>LobbyingActivities</th>\n",
       "      <th>LobbyingActivitiesInd</th>\n",
       "      <th>LocalChapters</th>\n",
       "      <th>LocalChaptersInd</th>\n",
       "      <th>MaterialDiversionOrMisuse</th>\n",
       "      <th>MaterialDiversionOrMisuseInd</th>\n",
       "      <th>MembersOrStockholders</th>\n",
       "      <th>MembersOrStockholdersInd</th>\n",
       "      <th>MembershipDues</th>\n",
       "      <th>MembershipDuesAmt</th>\n",
       "      <th>MethodOfAccountingAccrual</th>\n",
       "      <th>MethodOfAccountingAccrualInd</th>\n",
       "      <th>MethodOfAccountingCash</th>\n",
       "      <th>MethodOfAccountingCashInd</th>\n",
       "      <th>MethodOfAccountingOther</th>\n",
       "      <th>MethodOfAccountingOtherInd</th>\n",
       "      <th>MinutesOfCommittees</th>\n",
       "      <th>MinutesOfCommitteesInd</th>\n",
       "      <th>MinutesOfGoverningBody</th>\n",
       "      <th>MinutesOfGoverningBodyInd</th>\n",
       "      <th>MiscellaneousRevenue</th>\n",
       "      <th>MiscellaneousRevenueGrp</th>\n",
       "      <th>MissionDesc</th>\n",
       "      <th>MissionDescription</th>\n",
       "      <th>MoreThan5000KToIndividuals</th>\n",
       "      <th>MoreThan5000KToIndividualsInd</th>\n",
       "      <th>MoreThan5000KToOrgInd</th>\n",
       "      <th>MoreThan5000KToOrganizations</th>\n",
       "      <th>MortNotesPyblSecuredInvestProp</th>\n",
       "      <th>MortgNotesPyblScrdInvstPropGrp</th>\n",
       "      <th>NameOfForeignCountry</th>\n",
       "      <th>NameOfPrincipalOfficerBusiness</th>\n",
       "      <th>NameOfPrincipalOfficerPerson</th>\n",
       "      <th>NbrIndependentVotingMembers</th>\n",
       "      <th>NbrVotingGoverningBodyMembers</th>\n",
       "      <th>NbrVotingMembersGoverningBody</th>\n",
       "      <th>NetAssetsOrFundBalancesBOY</th>\n",
       "      <th>NetAssetsOrFundBalancesBOYAmt</th>\n",
       "      <th>NetAssetsOrFundBalancesEOY</th>\n",
       "      <th>NetAssetsOrFundBalancesEOYAmt</th>\n",
       "      <th>NetGainOrLossInvestments</th>\n",
       "      <th>NetGainOrLossInvestmentsGrp</th>\n",
       "      <th>NetIncmFromFundraisingEvtGrp</th>\n",
       "      <th>NetIncomeFromFundraisingEvents</th>\n",
       "      <th>NetIncomeFromGaming</th>\n",
       "      <th>NetIncomeFromGamingGrp</th>\n",
       "      <th>NetIncomeOrLoss</th>\n",
       "      <th>NetIncomeOrLossGrp</th>\n",
       "      <th>NetRentalIncomeOrLoss</th>\n",
       "      <th>NetRentalIncomeOrLossGrp</th>\n",
       "      <th>NetUnrelatedBusTxblIncmAmt</th>\n",
       "      <th>NetUnrelatedBusinessTxblIncome</th>\n",
       "      <th>NetUnrlzdGainsLossesInvstAmt</th>\n",
       "      <th>NoListedPersonsCompensated</th>\n",
       "      <th>NoListedPersonsCompensatedInd</th>\n",
       "      <th>NonDeductibleContributions</th>\n",
       "      <th>NonDeductibleDisclosure</th>\n",
       "      <th>NoncashContributions</th>\n",
       "      <th>NoncashContributionsAmt</th>\n",
       "      <th>NondeductibleContriDisclInd</th>\n",
       "      <th>NondeductibleContributionsInd</th>\n",
       "      <th>NumberFormsTransmittedWith1096</th>\n",
       "      <th>NumberIndependentVotingMembers</th>\n",
       "      <th>NumberIndividualsGT100K</th>\n",
       "      <th>NumberOf8282Filed</th>\n",
       "      <th>NumberOfContractorsGT100K</th>\n",
       "      <th>NumberOfEmployees</th>\n",
       "      <th>NumberW2GIncluded</th>\n",
       "      <th>ObjectId</th>\n",
       "      <th>Occupancy</th>\n",
       "      <th>OccupancyGrp</th>\n",
       "      <th>OfficeExpenses</th>\n",
       "      <th>OfficeExpensesGrp</th>\n",
       "      <th>OfficerEntityWithBsnssRltnshp</th>\n",
       "      <th>OfficerMailingAddress</th>\n",
       "      <th>OfficerMailingAddressInd</th>\n",
       "      <th>OnBehalfOfIssuer</th>\n",
       "      <th>OnBehalfOfIssuerInd</th>\n",
       "      <th>OperateHospitalInd</th>\n",
       "      <th>OrgDoesNotFollowSFAS117Ind</th>\n",
       "      <th>OrgFiledInLieuOfForm1041Ind</th>\n",
       "      <th>Organization501c</th>\n",
       "      <th>Organization501c3</th>\n",
       "      <th>Organization501c3Ind</th>\n",
       "      <th>Organization501cInd</th>\n",
       "      <th>OrganizationFollowsSFAS117Ind</th>\n",
       "      <th>OrganizationName</th>\n",
       "      <th>OthNotesLoansReceivableNetGrp</th>\n",
       "      <th>OtherAssetsTotal</th>\n",
       "      <th>OtherAssetsTotalGrp</th>\n",
       "      <th>OtherChangesInNetAssetsAmt</th>\n",
       "      <th>OtherEmployeeBenefits</th>\n",
       "      <th>OtherEmployeeBenefitsGrp</th>\n",
       "      <th>OtherExpensePriorYear</th>\n",
       "      <th>OtherExpenses</th>\n",
       "      <th>OtherExpensesCurrentYear</th>\n",
       "      <th>OtherExpensesGrp</th>\n",
       "      <th>OtherExplainInSchO</th>\n",
       "      <th>OtherInd</th>\n",
       "      <th>OtherLiabilities</th>\n",
       "      <th>OtherLiabilitiesGrp</th>\n",
       "      <th>OtherNotesLoansReceivableNet</th>\n",
       "      <th>OtherOrganizationDsc</th>\n",
       "      <th>OtherRevenueCurrentYear</th>\n",
       "      <th>OtherRevenueMisc</th>\n",
       "      <th>OtherRevenueMiscGrp</th>\n",
       "      <th>OtherRevenuePriorYear</th>\n",
       "      <th>OtherRevenueTotalAmt</th>\n",
       "      <th>OtherSalariesAndWages</th>\n",
       "      <th>OtherSalariesAndWagesGrp</th>\n",
       "      <th>OtherWebsite</th>\n",
       "      <th>OtherWebsiteInd</th>\n",
       "      <th>OwnWebsite</th>\n",
       "      <th>OwnWebsiteInd</th>\n",
       "      <th>PYBenefitsPaidToMembersAmt</th>\n",
       "      <th>PYContributionsGrantsAmt</th>\n",
       "      <th>PYExcessBenefitTransInd</th>\n",
       "      <th>PYGrantsAndSimilarPaidAmt</th>\n",
       "      <th>PYInvestmentIncomeAmt</th>\n",
       "      <th>PYOtherExpensesAmt</th>\n",
       "      <th>PYOtherRevenueAmt</th>\n",
       "      <th>PYProgramServiceRevenueAmt</th>\n",
       "      <th>PYRevenuesLessExpensesAmt</th>\n",
       "      <th>PYSalariesCompEmpBnftPaidAmt</th>\n",
       "      <th>PYTotalExpensesAmt</th>\n",
       "      <th>PYTotalProfFndrsngExpnsAmt</th>\n",
       "      <th>PYTotalRevenueAmt</th>\n",
       "      <th>PaidInCapSrplsLandBldgEqpFund</th>\n",
       "      <th>PartialLiquidation</th>\n",
       "      <th>PartialLiquidationInd</th>\n",
       "      <th>PayPremiumsPrsnlBnftCntrctInd</th>\n",
       "      <th>PaymentsToAffiliates</th>\n",
       "      <th>PaymentsToAffiliatesGrp</th>\n",
       "      <th>PayrollTaxes</th>\n",
       "      <th>PayrollTaxesGrp</th>\n",
       "      <th>PdInCapSrplsLandBldgEqpFundGrp</th>\n",
       "      <th>PensionPlanContributions</th>\n",
       "      <th>PensionPlanContributionsGrp</th>\n",
       "      <th>PermanentlyRestrictedNetAssets</th>\n",
       "      <th>PermanentlyRstrNetAssetsGrp</th>\n",
       "      <th>PledgesAndGrantsReceivable</th>\n",
       "      <th>PledgesAndGrantsReceivableGrp</th>\n",
       "      <th>PoliciesReferenceChapters</th>\n",
       "      <th>PoliciesReferenceChaptersInd</th>\n",
       "      <th>PoliticalActivities</th>\n",
       "      <th>PoliticalCampaignActyInd</th>\n",
       "      <th>PremiumsPaid</th>\n",
       "      <th>PrepaidExpensesDeferredCharges</th>\n",
       "      <th>PrepaidExpensesDefrdChargesGrp</th>\n",
       "      <th>PrincipalOfcrBusinessAddress</th>\n",
       "      <th>PrincipalOfcrBusinessName</th>\n",
       "      <th>PrincipalOfficerNm</th>\n",
       "      <th>PriorExcessBenefitTransaction</th>\n",
       "      <th>PriorPeriodAdjustmentsAmt</th>\n",
       "      <th>ProfessionalFundraising</th>\n",
       "      <th>ProfessionalFundraisingInd</th>\n",
       "      <th>ProgSrvcAccomActy2Grp</th>\n",
       "      <th>ProgSrvcAccomActy3Grp</th>\n",
       "      <th>ProgSrvcAccomActyOtherGrp</th>\n",
       "      <th>ProgramServiceRevenue</th>\n",
       "      <th>ProgramServiceRevenueCY</th>\n",
       "      <th>ProgramServiceRevenueGrp</th>\n",
       "      <th>ProgramServiceRevenuePriorYear</th>\n",
       "      <th>ProhibitedTaxShelterTrans</th>\n",
       "      <th>ProhibitedTaxShelterTransInd</th>\n",
       "      <th>PymtTravelEntrtnmntPubOfclGrp</th>\n",
       "      <th>QuidProQuoContriDisclInd</th>\n",
       "      <th>QuidProQuoContributions</th>\n",
       "      <th>QuidProQuoContributionsInd</th>\n",
       "      <th>QuidProQuoDisclosure</th>\n",
       "      <th>RcvFndsToPayPrsnlBnftCntrctInd</th>\n",
       "      <th>RcvblFromDisqualifiedPrsnGrp</th>\n",
       "      <th>ReceivablesFromDisqualPersons</th>\n",
       "      <th>ReceivablesFromOfficersEtc</th>\n",
       "      <th>ReceivablesFromOfficersEtcGrp</th>\n",
       "      <th>ReconcilationDonatedServices</th>\n",
       "      <th>ReconcilationInvestExpenses</th>\n",
       "      <th>ReconcilationOtherChanges</th>\n",
       "      <th>ReconcilationPriorAdjustment</th>\n",
       "      <th>ReconcilationRevenueExpenses</th>\n",
       "      <th>ReconcilationRevenueExpnssAmt</th>\n",
       "      <th>ReconciliationUnrealizedInvest</th>\n",
       "      <th>RegularMonitoringEnforcement</th>\n",
       "      <th>RegularMonitoringEnfrcInd</th>\n",
       "      <th>RelatedEntity</th>\n",
       "      <th>RelatedEntityInd</th>\n",
       "      <th>RelatedOrgControlledEntity</th>\n",
       "      <th>RelatedOrganizationCtrlEntInd</th>\n",
       "      <th>RelatedOrganizations</th>\n",
       "      <th>RelatedOrganizationsAmt</th>\n",
       "      <th>RentalIncomeOrLoss</th>\n",
       "      <th>RentalIncomeOrLossGrp</th>\n",
       "      <th>ReportFin48Footnote</th>\n",
       "      <th>ReportInvestOthSecurities</th>\n",
       "      <th>ReportInvestmentsOtherSecInd</th>\n",
       "      <th>ReportLandBldgEquip</th>\n",
       "      <th>ReportLandBuildingEquipmentInd</th>\n",
       "      <th>ReportOtherAssets</th>\n",
       "      <th>ReportOtherAssetsInd</th>\n",
       "      <th>ReportOtherLiabilities</th>\n",
       "      <th>ReportOtherLiabilitiesInd</th>\n",
       "      <th>ReportProgRelInvest</th>\n",
       "      <th>ReportProgramRelatedInvstInd</th>\n",
       "      <th>ReservesMaintainedAmt</th>\n",
       "      <th>RetainedEarningsEndowmentEtc</th>\n",
       "      <th>Revenue</th>\n",
       "      <th>RevenueAmt</th>\n",
       "      <th>RevenuesLessExpensesCY</th>\n",
       "      <th>RevenuesLessExpensesPriorYear</th>\n",
       "      <th>Royalties</th>\n",
       "      <th>RoyaltiesGrp</th>\n",
       "      <th>RoyaltiesRevenue</th>\n",
       "      <th>RoyaltiesRevenueGrp</th>\n",
       "      <th>RtnEarnEndowmentIncmOthFndsGrp</th>\n",
       "      <th>SalariesEtcCurrentYear</th>\n",
       "      <th>SalariesEtcPriorYear</th>\n",
       "      <th>SavingsAndTempCashInvestments</th>\n",
       "      <th>SavingsAndTempCashInvstGrp</th>\n",
       "      <th>ScheduleBRequired</th>\n",
       "      <th>ScheduleBRequiredInd</th>\n",
       "      <th>ScheduleJRequired</th>\n",
       "      <th>ScheduleJRequiredInd</th>\n",
       "      <th>ScheduleORequired</th>\n",
       "      <th>ScheduleORequiredInd</th>\n",
       "      <th>School</th>\n",
       "      <th>SchoolOperatingInd</th>\n",
       "      <th>SignificantChange</th>\n",
       "      <th>SignificantChangeInd</th>\n",
       "      <th>SignificantNewProgramServices</th>\n",
       "      <th>SignificantNewProgramSrvcInd</th>\n",
       "      <th>SpecialConditionDescription</th>\n",
       "      <th>StateLegalDomicile</th>\n",
       "      <th>StateRequiredReservesAmt</th>\n",
       "      <th>StatesWhereCopyOfReturnIsFiled</th>\n",
       "      <th>StatesWhereCopyOfReturnIsFldCd</th>\n",
       "      <th>SubjectToProxyTax</th>\n",
       "      <th>SubjectToProxyTaxInd</th>\n",
       "      <th>SubmittedOn</th>\n",
       "      <th>TaxExemptBondLiabilities</th>\n",
       "      <th>TaxExemptBondLiabilitiesGrp</th>\n",
       "      <th>TaxExemptBonds</th>\n",
       "      <th>TaxExemptBondsInd</th>\n",
       "      <th>TaxPeriod</th>\n",
       "      <th>TaxableDistributions</th>\n",
       "      <th>TaxableDistributionsInd</th>\n",
       "      <th>TaxablePartyNotification</th>\n",
       "      <th>TaxablePartyNotificationInd</th>\n",
       "      <th>TempOrPermanentEndowmentsInd</th>\n",
       "      <th>TemporarilyRestrictedNetAssets</th>\n",
       "      <th>TemporarilyRstrNetAssetsGrp</th>\n",
       "      <th>TermOrPermanentEndowments</th>\n",
       "      <th>TerminateOperationsInd</th>\n",
       "      <th>Terminated</th>\n",
       "      <th>TerminationOrContraction</th>\n",
       "      <th>TheBooksAreInCareOf</th>\n",
       "      <th>TotLiabNetAssetsFundBalanceGrp</th>\n",
       "      <th>TotReportableCompRltdOrgAmt</th>\n",
       "      <th>TotalAssets</th>\n",
       "      <th>TotalAssetsBOY</th>\n",
       "      <th>TotalAssetsBOYAmt</th>\n",
       "      <th>TotalAssetsEOY</th>\n",
       "      <th>TotalAssetsEOYAmt</th>\n",
       "      <th>TotalAssetsGrp</th>\n",
       "      <th>TotalCompGT150K</th>\n",
       "      <th>TotalCompGreaterThan150KInd</th>\n",
       "      <th>TotalContributions</th>\n",
       "      <th>TotalContributionsAmt</th>\n",
       "      <th>TotalEmployeeCnt</th>\n",
       "      <th>TotalExpensesCurrentYear</th>\n",
       "      <th>TotalExpensesPriorYear</th>\n",
       "      <th>TotalFunctionalExpenses</th>\n",
       "      <th>TotalFunctionalExpensesGrp</th>\n",
       "      <th>TotalFundrsngExpCurrentYear</th>\n",
       "      <th>TotalGrossUBI</th>\n",
       "      <th>TotalGrossUBIAmt</th>\n",
       "      <th>TotalJointCosts</th>\n",
       "      <th>TotalJointCostsGrp</th>\n",
       "      <th>TotalLiabNetAssetsFundBalances</th>\n",
       "      <th>TotalLiabilities</th>\n",
       "      <th>TotalLiabilitiesBOY</th>\n",
       "      <th>TotalLiabilitiesBOYAmt</th>\n",
       "      <th>TotalLiabilitiesEOY</th>\n",
       "      <th>TotalLiabilitiesEOYAmt</th>\n",
       "      <th>TotalLiabilitiesGrp</th>\n",
       "      <th>TotalNbrEmployees</th>\n",
       "      <th>TotalNbrVolunteers</th>\n",
       "      <th>TotalNetAssetsFundBalanceGrp</th>\n",
       "      <th>TotalNetAssetsFundBalances</th>\n",
       "      <th>TotalOfOtherProgramServiceExp</th>\n",
       "      <th>TotalOfOtherProgramServiceGrnt</th>\n",
       "      <th>TotalOfOtherProgramServiceRev</th>\n",
       "      <th>TotalOthProgramServiceRevGrp</th>\n",
       "      <th>TotalOthProgramServiceRevenue</th>\n",
       "      <th>TotalOtherCompensation</th>\n",
       "      <th>TotalOtherCompensationAmt</th>\n",
       "      <th>TotalOtherProgSrvcExpenseAmt</th>\n",
       "      <th>TotalOtherProgSrvcGrantAmt</th>\n",
       "      <th>TotalOtherProgSrvcRevenueAmt</th>\n",
       "      <th>TotalOtherRevenue</th>\n",
       "      <th>TotalProfFundrsngExpCY</th>\n",
       "      <th>TotalProfFundrsngExpPriorYear</th>\n",
       "      <th>TotalProgramServiceExpense</th>\n",
       "      <th>TotalProgramServiceExpensesAmt</th>\n",
       "      <th>TotalProgramServiceRevenue</th>\n",
       "      <th>TotalProgramServiceRevenueAmt</th>\n",
       "      <th>TotalReportableCompFrmRltdOrgs</th>\n",
       "      <th>TotalReportableCompFromOrg</th>\n",
       "      <th>TotalReportableCompFromOrgAmt</th>\n",
       "      <th>TotalRevenue</th>\n",
       "      <th>TotalRevenueCurrentYear</th>\n",
       "      <th>TotalRevenueGrp</th>\n",
       "      <th>TotalRevenuePriorYear</th>\n",
       "      <th>TotalVolunteersCnt</th>\n",
       "      <th>TransactionRelatedEntity</th>\n",
       "      <th>TransactionWithControlEntInd</th>\n",
       "      <th>TransfersToExemptNonChrtblOrg</th>\n",
       "      <th>Travel</th>\n",
       "      <th>TravelEntrtnmntPublicOfficials</th>\n",
       "      <th>TravelGrp</th>\n",
       "      <th>TrnsfrExmptNonChrtblRltdOrgInd</th>\n",
       "      <th>TypeOfOrgOtherDescription</th>\n",
       "      <th>TypeOfOrganizationAssocInd</th>\n",
       "      <th>TypeOfOrganizationAssociation</th>\n",
       "      <th>TypeOfOrganizationCorpInd</th>\n",
       "      <th>TypeOfOrganizationCorporation</th>\n",
       "      <th>TypeOfOrganizationOther</th>\n",
       "      <th>TypeOfOrganizationOtherInd</th>\n",
       "      <th>TypeOfOrganizationTrust</th>\n",
       "      <th>TypeOfOrganizationTrustInd</th>\n",
       "      <th>URL</th>\n",
       "      <th>USAddress</th>\n",
       "      <th>UnrelatedBusIncmOverLimitInd</th>\n",
       "      <th>UnrelatedBusinessIncome</th>\n",
       "      <th>UnrestrictedNetAssets</th>\n",
       "      <th>UnrestrictedNetAssetsGrp</th>\n",
       "      <th>UnsecuredNotesLoansPayable</th>\n",
       "      <th>UnsecuredNotesLoansPayableGrp</th>\n",
       "      <th>UponRequest</th>\n",
       "      <th>UponRequestInd</th>\n",
       "      <th>VotingMembersGoverningBodyCnt</th>\n",
       "      <th>VotingMembersIndependentCnt</th>\n",
       "      <th>WebSite</th>\n",
       "      <th>WebsiteAddressTxt</th>\n",
       "      <th>WhistleblowerPolicy</th>\n",
       "      <th>WhistleblowerPolicyInd</th>\n",
       "      <th>WrittenPolicyOrProcedure</th>\n",
       "      <th>WrittenPolicyOrProcedureInd</th>\n",
       "      <th>YearFormation</th>\n",
       "      <th>_id</th>\n",
       "      <th>FeesForServicesAccountingGrp_v2</th>\n",
       "      <th>FYE</th>\n",
       "      <th>FeesForServicesAccounting_TotalAmt</th>\n",
       "      <th>FeesForServicesAccounting_binary</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>18819</th>\n",
       "      <td>RetDoc1038000001</td>\n",
       "      <td>RetDoc1044400001</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0</td>\n",
       "      <td>{u'BOYAmt': u'506341', u'EOYAmt': u'557069'}</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>{u'BOYAmt': u'315681', u'EOYAmt': u'161137'}</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>TO DEVELOP SOLUTIONS TO COMPLEX HUMAN &amp; ENVIRONMENTAL HEALTH PROBLEMS THROUGH RESEARCH &amp; EDUCATION.</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>2334773</td>\n",
       "      <td>NaN</td>\n",
       "      <td>{u'TotalAmt': u'660207', u'ProgramServicesAmt': u'515736', u'ManagementAndGeneralAmt': u'95934', u'FundraisingAmt': u'48537'}</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>1</td>\n",
       "      <td>NaN</td>\n",
       "      <td>1</td>\n",
       "      <td>NaN</td>\n",
       "      <td>1</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>{u'PersonNm': u'CLAUDINE D LURVEY', u'PhoneNum': u'2072889880', u'USAddress': {u'CityNm': u'SALISBURY COVE', u'StateAbbreviationCd': u'ME', u'ZIPCd': u'04672', u'AddressLine1Txt': u'OLD BAR HARBOR ROAD'}}</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>{u'#text': u'1', u'@referenceDocumentId': u'RetDoc1042800001'}</td>\n",
       "      <td>{u'#text': u'1', u'@referenceDocumentId': u'RetDoc1042800001'}</td>\n",
       "      <td>{u'#text': u'1', u'@referenceDocumentId': u'RetDoc1042800001'}</td>\n",
       "      <td>0</td>\n",
       "      <td>10272455</td>\n",
       "      <td>0</td>\n",
       "      <td>982272</td>\n",
       "      <td>6661282</td>\n",
       "      <td>1404</td>\n",
       "      <td>1711837</td>\n",
       "      <td>2136547</td>\n",
       "      <td>4155139</td>\n",
       "      <td>10831421</td>\n",
       "      <td>554608</td>\n",
       "      <td>15000</td>\n",
       "      <td>12967968</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>{u'BOYAmt': u'1121795', u'EOYAmt': u'2540456'}</td>\n",
       "      <td>0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>1</td>\n",
       "      <td>NaN</td>\n",
       "      <td>{u'#text': u'0', u'@referenceDocumentId': u'RetDoc1040000001'}</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>1</td>\n",
       "      <td>NaN</td>\n",
       "      <td>1</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>1</td>\n",
       "      <td>NaN</td>\n",
       "      <td>{u'#text': u'0', u'@referenceDocumentId': u'RetDoc1040000001'}</td>\n",
       "      <td>NaN</td>\n",
       "      <td>{u'#text': u'0', u'@referenceDocumentId': u'RetDoc1040000001'}</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>{u'ContractorName': {u'BusinessName': {u'BusinessNameLine1Txt': u'GOODIE'S ELECTRICAL SERVICE'}}, u'ServicesDesc': u'ELECTRICIAN', u'ContractorAddress': {u'USAddress': {u'CityNm': u'BAR HARBOR', u'StateAbbreviationCd': u'ME', u'ZIPCd': u'04609', ...</td>\n",
       "      <td>800</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>{u'#text': u'0', u'@referenceDocumentId': u'RetDoc1040000001'}</td>\n",
       "      <td>NaN</td>\n",
       "      <td>93493320080035</td>\n",
       "      <td>0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>{u'#text': u'0', u'@referenceDocumentId': u'RetDoc1042900001'}</td>\n",
       "      <td>NaN</td>\n",
       "      <td>{u'#text': u'1', u'@referenceDocumentId': u'RetDoc1042900001'}</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>{u'BOYAmt': u'529444', u'EOYAmt': u'835605'}</td>\n",
       "      <td>NaN</td>\n",
       "      <td>1</td>\n",
       "      <td>NaN</td>\n",
       "      <td>{u'TotalAmt': u'1121059', u'ProgramServicesAmt': u'896847', u'ManagementAndGeneralAmt': u'168159', u'FundraisingAmt': u'56053'}</td>\n",
       "      <td>THE MDI BIOLOGICAL LABORATORY IS RECOGNIZED BY THE NATIONAL INSTITUTES OF HEALTH AS A CENTER OF BIOMEDICAL RESEARCH EXCELLENCE (COBRE) FOR OUR INNOVATIVE RESEARCH PROGRAM IN REGENERATIVE AND AGING BIOLOGY AND MEDICINE.OUR APPROACH SPEEDS THE DISC...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>{u'#text': u'1', u'@referenceDocumentId': u'RetDoc1039100001'}</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>1</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>{u'#text': u'0', u'@referenceDocumentId': u'RetDoc1040000001'}</td>\n",
       "      <td>NaN</td>\n",
       "      <td>010202467</td>\n",
       "      <td>NaN</td>\n",
       "      <td>1</td>\n",
       "      <td>96</td>\n",
       "      <td>NaN</td>\n",
       "      <td>1</td>\n",
       "      <td>{u'#text': u'0', u'@referenceDocumentId': u'RetDoc1042800001'}</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>8605100</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>{u'SeparateBasisFinclStmtInd': u'X'}</td>\n",
       "      <td>1</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>1</td>\n",
       "      <td>NaN</td>\n",
       "      <td>1</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>{u'TotalAmt': u'23700', u'ManagementAndGeneralAmt': u'23700'}</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>{u'TotalAmt': u'3792', u'ManagementAndGeneralAmt': u'3792'}</td>\n",
       "      <td>NaN</td>\n",
       "      <td>{u'TotalAmt': u'89167', u'ManagementAndGeneralAmt': u'3379', u'FundraisingAmt': u'85788'}</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>{u'TotalAmt': u'201518', u'ProgramServicesAmt': u'127950', u'ManagementAndGeneralAmt': u'64909', u'FundraisingAmt': u'8659'}</td>\n",
       "      <td>{u'TotalAmt': u'15000', u'FundraisingAmt': u'15000'}</td>\n",
       "      <td>{u'TotalAmt': u'18902', u'ManagementAndGeneralAmt': u'18902'}</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>[{u'PersonNm': u'DR EDWARD J BENZ', u'IndividualTrusteeOrDirectorInd': u'X', u'AverageHoursPerWeekRt': u'2.00', u'ReportableCompFromRltdOrgAmt': u'0', u'OtherCompensationAmt': u'0', u'TitleTxt': u'VICE CHAIRMAN OF THE BOARD', u'OfficerInd': u'X',...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>1</td>\n",
       "      <td>NaN</td>\n",
       "      <td>990</td>\n",
       "      <td>1898</td>\n",
       "      <td>0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>{u'#text': u'1', u'@referenceDocumentId': u'RetDoc1041300001'}</td>\n",
       "      <td>800</td>\n",
       "      <td>NaN</td>\n",
       "      <td>37598</td>\n",
       "      <td>NaN</td>\n",
       "      <td>38964</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>{u'OtherAmt': u'322752'}</td>\n",
       "      <td>NaN</td>\n",
       "      <td>{u'#text': u'0', u'@referenceDocumentId': u'RetDoc1041300001'}</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>22</td>\n",
       "      <td>NaN</td>\n",
       "      <td>7932282</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>{u'#text': u'0', u'@referenceDocumentId': u'RetDoc1042800001'}</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>{u'OtherAmt': u'340000'}</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>13022814</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>38</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>158</td>\n",
       "      <td>0</td>\n",
       "      <td>{u'#text': u'1', u'@referenceDocumentId': u'RetDoc1040000001'}</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>{u'#text': u'1', u'@referenceDocumentId': u'RetDoc1040000001'}</td>\n",
       "      <td>21</td>\n",
       "      <td>3</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>X</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>X</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>X</td>\n",
       "      <td>X</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>{u'TotalAmt': u'49382', u'ProgramServicesAmt': u'8673', u'ManagementAndGeneralAmt': u'40709'}</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>{u'TotalAmt': u'92396', u'ProgramServicesAmt': u'10397', u'ManagementAndGeneralAmt': u'81746', u'FundraisingAmt': u'253'}</td>\n",
       "      <td>NaN</td>\n",
       "      <td>{u'BOYAmt': u'147691', u'EOYAmt': u'141271'}</td>\n",
       "      <td>NaN</td>\n",
       "      <td>{u'TotalAmt': u'255985', u'ProgramServicesAmt': u'191989', u'ManagementAndGeneralAmt': u'63996'}</td>\n",
       "      <td>NaN</td>\n",
       "      <td>{u'BOYAmt': u'8329'}</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>{u'TotalRevenueColumnAmt': u'659520', u'ExclusionAmt': u'659520'}</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>{u'BOYAmt': u'4031973', u'EOYAmt': u'4747115'}</td>\n",
       "      <td>NaN</td>\n",
       "      <td>True</td>\n",
       "      <td>True</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>11322245</td>\n",
       "      <td>{u'BOYAmt': u'14025403', u'EOYAmt': u'14020382'}</td>\n",
       "      <td>25342627</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>2016-04-29T13:40:20</td>\n",
       "      <td>ME</td>\n",
       "      <td>NaN</td>\n",
       "      <td>{u'OtherAmt': u'17248'}</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>{u'#text': u'0', u'@referenceDocumentId': u'RetDoc1042800001'}</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>{u'#text': u'1', u'@referenceDocumentId': u'RetDoc1039700001'}</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>1</td>\n",
       "      <td>NaN</td>\n",
       "      <td>4600</td>\n",
       "      <td>NaN</td>\n",
       "      <td>X</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>1</td>\n",
       "      <td>NaN</td>\n",
       "      <td>1</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>THE MDI BIOLOGICAL LABORATORY IS A RAPIDLY GROWING, INDEPENDENT NON-PROFIT BIOMEDICAL RESEARCH INSTITUTION. ITS MISSION IS TO IMPROVE HUMAN HEALTH AND WELL-BEING THROUGH RESEARCH, EDUCATION,AND DEVELOPMENT VENTURES THAT TRANSFORM DISCOVERIES INTO...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>21657900</td>\n",
       "      <td>NaN</td>\n",
       "      <td>23690097</td>\n",
       "      <td>NaN</td>\n",
       "      <td>{u'TotalRevenueColumnAmt': u'322752', u'ExclusionAmt': u'322752'}</td>\n",
       "      <td>{u'TotalRevenueColumnAmt': u'1366', u'ExclusionAmt': u'1366'}</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>{u'TotalRevenueColumnAmt': u'38', u'ExclusionAmt': u'38'}</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>-104353</td>\n",
       "      <td>NaN</td>\n",
       "      <td>X</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>57474</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>201533209349308003</td>\n",
       "      <td>NaN</td>\n",
       "      <td>{u'TotalAmt': u'587738', u'ProgramServicesAmt': u'494365', u'ManagementAndGeneralAmt': u'91234', u'FundraisingAmt': u'2139'}</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>X</td>\n",
       "      <td>NaN</td>\n",
       "      <td>X</td>\n",
       "      <td>MOUNT DESERT ISLAND BIOLOGICAL LABORATORY</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>3</td>\n",
       "      <td>NaN</td>\n",
       "      <td>{u'TotalAmt': u'370740', u'ProgramServicesAmt': u'284086', u'ManagementAndGeneralAmt': u'70910', u'FundraisingAmt': u'15744'}</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>[{u'TotalAmt': u'2124748', u'ProgramServicesAmt': u'2124748', u'Desc': u'SUBAWARDS'}, {u'TotalAmt': u'682378', u'ProgramServicesAmt': u'627645', u'ManagementAndGeneralAmt': u'47547', u'FundraisingAmt': u'7186', u'Desc': u'SUPPLIES'}, {u'TotalAmt'...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>{u'BOYAmt': u'4630000', u'EOYAmt': u'4525000'}</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>{u'TotalAmt': u'3353971', u'ProgramServicesAmt': u'2478596', u'ManagementAndGeneralAmt': u'683266', u'FundraisingAmt': u'192109'}</td>\n",
       "      <td>NaN</td>\n",
       "      <td>X</td>\n",
       "      <td>NaN</td>\n",
       "      <td>X</td>\n",
       "      <td>0</td>\n",
       "      <td>7802139</td>\n",
       "      <td>{u'#text': u'0', u'@referenceDocumentId': u'RetDoc1042800001'}</td>\n",
       "      <td>0</td>\n",
       "      <td>785633</td>\n",
       "      <td>6736413</td>\n",
       "      <td>-13375</td>\n",
       "      <td>1591204</td>\n",
       "      <td>-449665</td>\n",
       "      <td>3878853</td>\n",
       "      <td>10615266</td>\n",
       "      <td>0</td>\n",
       "      <td>10165601</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>{u'TotalAmt': u'257923', u'ProgramServicesAmt': u'189245', u'ManagementAndGeneralAmt': u'52592', u'FundraisingAmt': u'16086'}</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>{u'TotalAmt': u'172505', u'ProgramServicesAmt': u'125037', u'ManagementAndGeneralAmt': u'37384', u'FundraisingAmt': u'10084'}</td>\n",
       "      <td>NaN</td>\n",
       "      <td>{u'BOYAmt': u'4193257', u'EOYAmt': u'4433997'}</td>\n",
       "      <td>NaN</td>\n",
       "      <td>{u'BOYAmt': u'797504', u'EOYAmt': u'986621'}</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>{u'#text': u'0', u'@referenceDocumentId': u'RetDoc1039700001'}</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>{u'BOYAmt': u'4791', u'EOYAmt': u'5301'}</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>DR KEVIN STRANGE PHD</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>{u'#text': u'0', u'@referenceDocumentId': u'RetDoc1041300001'}</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>[{u'TotalRevenueColumnAmt': u'469300', u'BusinessCd': u'611710', u'RelatedOrExemptFuncIncomeAmt': u'469300', u'Desc': u'CONFER &amp; COURSE FEES'}, {u'TotalRevenueColumnAmt': u'353507', u'BusinessCd': u'541700', u'RelatedOrExemptFuncIncomeAmt': u'353...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>2136547</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>1</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>{u'#text': u'0', u'@referenceDocumentId': u'RetDoc1040000001'}</td>\n",
       "      <td>NaN</td>\n",
       "      <td>{u'#text': u'1', u'@referenceDocumentId': u'RetDoc1040000001'}</td>\n",
       "      <td>NaN</td>\n",
       "      <td>{u'#text': u'0', u'@referenceDocumentId': u'RetDoc1040000001'}</td>\n",
       "      <td>NaN</td>\n",
       "      <td>{u'#text': u'1', u'@referenceDocumentId': u'RetDoc1040000001'}</td>\n",
       "      <td>NaN</td>\n",
       "      <td>{u'#text': u'0', u'@referenceDocumentId': u'RetDoc1040000001'}</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>1733437</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>{u'BOYAmt': u'6870518', u'EOYAmt': u'7005488'}</td>\n",
       "      <td>NaN</td>\n",
       "      <td>{u'#text': u'1', u'@referenceDocumentId': u'RetDoc1234500001'}</td>\n",
       "      <td>NaN</td>\n",
       "      <td>{u'#text': u'1', u'@referenceDocumentId': u'RetDoc1042400001'}</td>\n",
       "      <td>NaN</td>\n",
       "      <td>1</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>ME</td>\n",
       "      <td>NaN</td>\n",
       "      <td>{u'#text': u'0', u'@referenceDocumentId': u'RetDoc1039700001'}</td>\n",
       "      <td>2016-02-16</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>{u'#text': u'1', u'@referenceDocumentId': u'RetDoc1053100001'}</td>\n",
       "      <td>201412</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0</td>\n",
       "      <td>{u'#text': u'1', u'@referenceDocumentId': u'RetDoc1040000001'}</td>\n",
       "      <td>NaN</td>\n",
       "      <td>{u'BOYAmt': u'5880227', u'EOYAmt': u'7351259'}</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>{u'BOYAmt': u'27323685', u'EOYAmt': u'29607771'}</td>\n",
       "      <td>0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>27323685</td>\n",
       "      <td>NaN</td>\n",
       "      <td>29607771</td>\n",
       "      <td>{u'BOYAmt': u'27323685', u'EOYAmt': u'29607771'}</td>\n",
       "      <td>NaN</td>\n",
       "      <td>1</td>\n",
       "      <td>NaN</td>\n",
       "      <td>10272455</td>\n",
       "      <td>96</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>{u'TotalAmt': u'10831421', u'ProgramServicesAmt': u'8605100', u'ManagementAndGeneralAmt': u'1671713', u'FundraisingAmt': u'554608'}</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>5665785</td>\n",
       "      <td>NaN</td>\n",
       "      <td>5917674</td>\n",
       "      <td>{u'BOYAmt': u'5665785', u'EOYAmt': u'5917674'}</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>{u'BOYAmt': u'21657900', u'EOYAmt': u'23690097'}</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>{u'TotalRevenueColumnAmt': u'38356', u'RelatedOrExemptFuncIncomeAmt': u'38356'}</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>67100</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>8605100</td>\n",
       "      <td>NaN</td>\n",
       "      <td>1711837</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>496334</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>{u'UnrelatedBusinessRevenueAmt': u'0', u'TotalRevenueColumnAmt': u'12967968', u'RelatedOrExemptFuncIncomeAmt': u'1711837', u'ExclusionAmt': u'983676'}</td>\n",
       "      <td>NaN</td>\n",
       "      <td>125</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>{u'TotalAmt': u'128657', u'ProgramServicesAmt': u'84416', u'ManagementAndGeneralAmt': u'33781', u'FundraisingAmt': u'10460'}</td>\n",
       "      <td>0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>X</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>https://s3.amazonaws.com/irs-form-990/201533209349308003_public.xml</td>\n",
       "      <td>{u'CityNm': u'SALISBURY COVE', u'StateAbbreviationCd': u'ME', u'ZIPCd': u'046720035', u'AddressLine1Txt': u'PO BOX 35'}</td>\n",
       "      <td>0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>{u'BOYAmt': u'11584416', u'EOYAmt': u'11904841'}</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>X</td>\n",
       "      <td>22</td>\n",
       "      <td>21</td>\n",
       "      <td>NaN</td>\n",
       "      <td>WWW.MDIBL.ORG</td>\n",
       "      <td>NaN</td>\n",
       "      <td>1</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>58c33e003ffc5a1664e96b73</td>\n",
       "      <td>{u'TotalAmt': u'23700', u'ManagementAndGeneralAmt': u'23700'}</td>\n",
       "      <td>FY2014</td>\n",
       "      <td>23700.0</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "            @documentId @referenceDocumentId @referenceDocumentName  \\\n",
       "18819  RetDoc1038000001     RetDoc1044400001                    NaN   \n",
       "\n",
       "      @softwareId @softwareVersion @softwareVersionNum  \\\n",
       "18819         NaN              NaN                 NaN   \n",
       "\n",
       "      AccountantCompileOrReview AccountantCompileOrReviewBasis  \\\n",
       "18819                       NaN                            NaN   \n",
       "\n",
       "      AccountantCompileOrReviewInd  \\\n",
       "18819                            0   \n",
       "\n",
       "                       AccountsPayableAccrExpnssGrp  \\\n",
       "18819  {u'BOYAmt': u'506341', u'EOYAmt': u'557069'}   \n",
       "\n",
       "      AccountsPayableAccruedExpenses AccountsReceivable  \\\n",
       "18819                            NaN                NaN   \n",
       "\n",
       "                              AccountsReceivableGrp  \\\n",
       "18819  {u'BOYAmt': u'315681', u'EOYAmt': u'161137'}   \n",
       "\n",
       "      AcctCompileOrReviewBasisGrp ActivitiesConductedPartnership  \\\n",
       "18819                         NaN                            NaN   \n",
       "\n",
       "      ActivitiesConductedPrtshpInd Activity2 Activity3 ActivityCd  \\\n",
       "18819                            0       NaN       NaN        NaN   \n",
       "\n",
       "      ActivityCode  \\\n",
       "18819          NaN   \n",
       "\n",
       "                                                                                     ActivityOrMissionDesc  \\\n",
       "18819  TO DEVELOP SOLUTIONS TO COMPLEX HUMAN & ENVIRONMENTAL HEALTH PROBLEMS THROUGH RESEARCH & EDUCATION.   \n",
       "\n",
       "      ActivityOrMissionDescription ActivityOther AddressChange  \\\n",
       "18819                          NaN           NaN           NaN   \n",
       "\n",
       "      AddressChangeInd AddressPrincipalOfficerForeign  \\\n",
       "18819              NaN                            NaN   \n",
       "\n",
       "      AddressPrincipalOfficerUS Advertising AdvertisingGrp  \\\n",
       "18819                       NaN         NaN            NaN   \n",
       "\n",
       "      AllAffiliatesIncluded AllAffiliatesIncludedInd AllOtherContributions  \\\n",
       "18819                   NaN                      NaN                   NaN   \n",
       "\n",
       "      AllOtherContributionsAmt AllOtherExpenses  \\\n",
       "18819                  2334773              NaN   \n",
       "\n",
       "                                                                                                                 AllOtherExpensesGrp  \\\n",
       "18819  {u'TotalAmt': u'660207', u'ProgramServicesAmt': u'515736', u'ManagementAndGeneralAmt': u'95934', u'FundraisingAmt': u'48537'}   \n",
       "\n",
       "      AmendedReturn AmendedReturnInd AnnualDisclosureCoveredPersons  \\\n",
       "18819           NaN              NaN                            NaN   \n",
       "\n",
       "      AnnualDisclosureCoveredPrsnInd AuditCommittee AuditCommitteeInd  \\\n",
       "18819                              1            NaN                 1   \n",
       "\n",
       "      AuditedFinancialStmtAttInd BackupWthldComplianceInd  \\\n",
       "18819                        NaN                        1   \n",
       "\n",
       "      BalanceSheetAmountsReported BenefitsPaidToMembersCY  \\\n",
       "18819                         NaN                     NaN   \n",
       "\n",
       "      BenefitsPaidToMembersPriorYear BenefitsToMembers BenefitsToMembersGrp  \\\n",
       "18819                            NaN               NaN                  NaN   \n",
       "\n",
       "                                                                                                                                                                                                BooksInCareOfDetail  \\\n",
       "18819  {u'PersonNm': u'CLAUDINE D LURVEY', u'PhoneNum': u'2072889880', u'USAddress': {u'CityNm': u'SALISBURY COVE', u'StateAbbreviationCd': u'ME', u'ZIPCd': u'04672', u'AddressLine1Txt': u'OLD BAR HARBOR ROAD'}}   \n",
       "\n",
       "      BsnssRltnshpThruFamilyMember BsnssRltnshpWithOrganization  \\\n",
       "18819                          NaN                          NaN   \n",
       "\n",
       "                                             BusinessRlnWithFamMemInd  \\\n",
       "18819  {u'#text': u'1', u'@referenceDocumentId': u'RetDoc1042800001'}   \n",
       "\n",
       "                                         BusinessRlnWithOfficerEntInd  \\\n",
       "18819  {u'#text': u'1', u'@referenceDocumentId': u'RetDoc1042800001'}   \n",
       "\n",
       "                                             BusinessRlnWithOrgMemInd  \\\n",
       "18819  {u'#text': u'1', u'@referenceDocumentId': u'RetDoc1042800001'}   \n",
       "\n",
       "      CYBenefitsPaidToMembersAmt CYContributionsGrantsAmt  \\\n",
       "18819                          0                 10272455   \n",
       "\n",
       "      CYGrantsAndSimilarPaidAmt CYInvestmentIncomeAmt CYOtherExpensesAmt  \\\n",
       "18819                         0                982272            6661282   \n",
       "\n",
       "      CYOtherRevenueAmt CYProgramServiceRevenueAmt CYRevenuesLessExpensesAmt  \\\n",
       "18819              1404                    1711837                   2136547   \n",
       "\n",
       "      CYSalariesCompEmpBnftPaidAmt CYTotalExpensesAmt  \\\n",
       "18819                      4155139           10831421   \n",
       "\n",
       "      CYTotalFundraisingExpenseAmt CYTotalProfFndrsngExpnsAmt  \\\n",
       "18819                       554608                      15000   \n",
       "\n",
       "      CYTotalRevenueAmt CapStckTrstPrinCurrentFunds  \\\n",
       "18819          12967968                         NaN   \n",
       "\n",
       "      CapStkTrPrinCurrentFundsGrp CashNonInterestBearing  \\\n",
       "18819                         NaN                    NaN   \n",
       "\n",
       "                            CashNonInterestBearingGrp ChangeToOrgDocumentsInd  \\\n",
       "18819  {u'BOYAmt': u'1121795', u'EOYAmt': u'2540456'}                       0   \n",
       "\n",
       "      ChangesToOrganizingDocs CntrbtnsRprtdFundraisingEvents  \\\n",
       "18819                     NaN                            NaN   \n",
       "\n",
       "      CntrctRcvdGreaterThan100KCnt CollectionsOfArt  \\\n",
       "18819                            1              NaN   \n",
       "\n",
       "                                                  CollectionsOfArtInd  \\\n",
       "18819  {u'#text': u'0', u'@referenceDocumentId': u'RetDoc1040000001'}   \n",
       "\n",
       "      CompCurrentOfcrDirectorsGrp CompCurrentOfficersDirectors  \\\n",
       "18819                         NaN                          NaN   \n",
       "\n",
       "      CompDisqualPersons CompDisqualPersonsGrp CompensationFromOtherSources  \\\n",
       "18819                NaN                   NaN                          NaN   \n",
       "\n",
       "      CompensationFromOtherSrcsInd CompensationProcessCEO  \\\n",
       "18819                            0                    NaN   \n",
       "\n",
       "      CompensationProcessCEOInd CompensationProcessOther  \\\n",
       "18819                         1                      NaN   \n",
       "\n",
       "      CompensationProcessOtherInd ComplianceWithBackupWitholding  \\\n",
       "18819                           1                            NaN   \n",
       "\n",
       "      ConferencesMeetings ConferencesMeetingsGrp ConflictOfInterestPolicy  \\\n",
       "18819                 NaN                    NaN                      NaN   \n",
       "\n",
       "      ConflictOfInterestPolicyInd ConservationEasements  \\\n",
       "18819                           1                   NaN   \n",
       "\n",
       "                                             ConservationEasementsInd  \\\n",
       "18819  {u'#text': u'0', u'@referenceDocumentId': u'RetDoc1040000001'}   \n",
       "\n",
       "      ConsolidatedAuditFinancialStmt  \\\n",
       "18819                            NaN   \n",
       "\n",
       "                                        ConsolidatedAuditFinclStmtInd  \\\n",
       "18819  {u'#text': u'0', u'@referenceDocumentId': u'RetDoc1040000001'}   \n",
       "\n",
       "      ContractTerminationInd ContractorCompensation  \\\n",
       "18819                    NaN                    NaN   \n",
       "\n",
       "                                                                                                                                                                                                                                       ContractorCompensationGrp  \\\n",
       "18819  {u'ContractorName': {u'BusinessName': {u'BusinessNameLine1Txt': u'GOODIE'S ELECTRICAL SERVICE'}}, u'ServicesDesc': u'ELECTRICIAN', u'ContractorAddress': {u'USAddress': {u'CityNm': u'BAR HARBOR', u'StateAbbreviationCd': u'ME', u'ZIPCd': u'04609', ...   \n",
       "\n",
       "      ContriRptFundraisingEventAmt ContributionsGrantsCurrentYear  \\\n",
       "18819                          800                            NaN   \n",
       "\n",
       "      ContributionsGrantsPriorYear CostOfGoodsSold CostOfGoodsSoldAmt  \\\n",
       "18819                          NaN             NaN                  0   \n",
       "\n",
       "      CountryLegalDomicile CreditCounseling  \\\n",
       "18819                  NaN              NaN   \n",
       "\n",
       "                                                  CreditCounselingInd  \\\n",
       "18819  {u'#text': u'0', u'@referenceDocumentId': u'RetDoc1040000001'}   \n",
       "\n",
       "      DAFExcessBusinessHoldingsInd             DLN  \\\n",
       "18819                          NaN  93493320080035   \n",
       "\n",
       "      DecisionsSubjectToApprovaInd DecisionsSubjectToApproval  \\\n",
       "18819                            0                        NaN   \n",
       "\n",
       "                                         DeductibleArtContributionInd  \\\n",
       "18819  {u'#text': u'0', u'@referenceDocumentId': u'RetDoc1042900001'}   \n",
       "\n",
       "      DeductibleContributionsOfArt  \\\n",
       "18819                          NaN   \n",
       "\n",
       "                                           DeductibleNonCashContriInd  \\\n",
       "18819  {u'#text': u'1', u'@referenceDocumentId': u'RetDoc1042900001'}   \n",
       "\n",
       "      DeductibleNonCashContributions DeferredRevenue  \\\n",
       "18819                            NaN             NaN   \n",
       "\n",
       "                                 DeferredRevenueGrp  \\\n",
       "18819  {u'BOYAmt': u'529444', u'EOYAmt': u'835605'}   \n",
       "\n",
       "      DelegationOfManagementDuties DelegationOfMgmtDutiesInd  \\\n",
       "18819                          NaN                         1   \n",
       "\n",
       "      DepreciationDepletion  \\\n",
       "18819                   NaN   \n",
       "\n",
       "                                                                                                              DepreciationDepletionGrp  \\\n",
       "18819  {u'TotalAmt': u'1121059', u'ProgramServicesAmt': u'896847', u'ManagementAndGeneralAmt': u'168159', u'FundraisingAmt': u'56053'}   \n",
       "\n",
       "                                                                                                                                                                                                                                                            Desc  \\\n",
       "18819  THE MDI BIOLOGICAL LABORATORY IS RECOGNIZED BY THE NATIONAL INSTITUTES OF HEALTH AS A CENTER OF BIOMEDICAL RESEARCH EXCELLENCE (COBRE) FOR OUR INNOVATIVE RESEARCH PROGRAM IN REGENERATIVE AND AGING BIOLOGY AND MEDICINE.OUR APPROACH SPEEDS THE DISC...   \n",
       "\n",
       "      DescribedIn501C3  \\\n",
       "18819              NaN   \n",
       "\n",
       "                                           DescribedInSection501c3Ind  \\\n",
       "18819  {u'#text': u'1', u'@referenceDocumentId': u'RetDoc1039100001'}   \n",
       "\n",
       "      Description DisregardedEntity DisregardedEntityInd DistributionToDonor  \\\n",
       "18819         NaN               NaN                    0                 NaN   \n",
       "\n",
       "      DistributionToDonorInd DoNotFollowSFAS117 DocumentRetentionPolicy  \\\n",
       "18819                    NaN                NaN                     NaN   \n",
       "\n",
       "      DocumentRetentionPolicyInd DoingBusinessAs DoingBusinessAsName  \\\n",
       "18819                          1             NaN                 NaN   \n",
       "\n",
       "      DonatedServicesAndUseFcltsAmt  \\\n",
       "18819                           NaN   \n",
       "\n",
       "                                                  DonorAdvisedFundInd  \\\n",
       "18819  {u'#text': u'0', u'@referenceDocumentId': u'RetDoc1040000001'}   \n",
       "\n",
       "      DonorAdvisedFunds        EIN ElectionOfBoardMembers  \\\n",
       "18819               NaN  010202467                    NaN   \n",
       "\n",
       "      ElectionOfBoardMembersInd EmployeeCnt EmploymentTaxReturnsFiled  \\\n",
       "18819                         1          96                       NaN   \n",
       "\n",
       "      EmploymentTaxReturnsFiledInd  \\\n",
       "18819                            1   \n",
       "\n",
       "                                       EngagedInExcessBenefitTransInd  \\\n",
       "18819  {u'#text': u'0', u'@referenceDocumentId': u'RetDoc1042800001'}   \n",
       "\n",
       "      EscrowAccount EscrowAccountInd EscrowAccountLiability  \\\n",
       "18819           NaN                0                    NaN   \n",
       "\n",
       "      EscrowAccountLiabilityGrp ExcessBenefitTransaction  \\\n",
       "18819                       NaN                      NaN   \n",
       "\n",
       "      ExcessBusinessHoldings ExcessBusinessHoldingsInd Expense ExpenseAmt  \\\n",
       "18819                    NaN                       NaN     NaN    8605100   \n",
       "\n",
       "      FSAudited FSAuditedBasis                     FSAuditedBasisGrp  \\\n",
       "18819       NaN            NaN  {u'SeparateBasisFinclStmtInd': u'X'}   \n",
       "\n",
       "      FSAuditedInd FamilyOrBusinessRelationship FamilyOrBusinessRlnInd  \\\n",
       "18819            1                          NaN                      0   \n",
       "\n",
       "      FederalGrantAuditPerformed FederalGrantAuditPerformedInd  \\\n",
       "18819                        NaN                             1   \n",
       "\n",
       "      FederalGrantAuditRequired FederalGrantAuditRequiredInd  \\\n",
       "18819                       NaN                            1   \n",
       "\n",
       "      FederatedCampaigns FederatedCampaignsAmt FeesForServicesAccounting  \\\n",
       "18819                NaN                   NaN                       NaN   \n",
       "\n",
       "                                        FeesForServicesAccountingGrp  \\\n",
       "18819  {u'TotalAmt': u'23700', u'ManagementAndGeneralAmt': u'23700'}   \n",
       "\n",
       "      FeesForServicesInvstMgmntFees FeesForServicesLegal  \\\n",
       "18819                           NaN                  NaN   \n",
       "\n",
       "                                           FeesForServicesLegalGrp  \\\n",
       "18819  {u'TotalAmt': u'3792', u'ManagementAndGeneralAmt': u'3792'}   \n",
       "\n",
       "      FeesForServicesLobbying  \\\n",
       "18819                     NaN   \n",
       "\n",
       "                                                                      FeesForServicesLobbyingGrp  \\\n",
       "18819  {u'TotalAmt': u'89167', u'ManagementAndGeneralAmt': u'3379', u'FundraisingAmt': u'85788'}   \n",
       "\n",
       "      FeesForServicesManagement FeesForServicesManagementGrp  \\\n",
       "18819                       NaN                          NaN   \n",
       "\n",
       "      FeesForServicesOther  \\\n",
       "18819                  NaN   \n",
       "\n",
       "                                                                                                            FeesForServicesOtherGrp  \\\n",
       "18819  {u'TotalAmt': u'201518', u'ProgramServicesAmt': u'127950', u'ManagementAndGeneralAmt': u'64909', u'FundraisingAmt': u'8659'}   \n",
       "\n",
       "                             FeesForServicesProfFundraising  \\\n",
       "18819  {u'TotalAmt': u'15000', u'FundraisingAmt': u'15000'}   \n",
       "\n",
       "                                        FeesForSrvcInvstMgmntFeesGrp  \\\n",
       "18819  {u'TotalAmt': u'18902', u'ManagementAndGeneralAmt': u'18902'}   \n",
       "\n",
       "      FiledLieu1041 FinalReturnInd FinancialStatementBoth  \\\n",
       "18819           NaN            NaN                    NaN   \n",
       "\n",
       "      FinancialStatementConsolidated FinancialStatementSeparate  \\\n",
       "18819                            NaN                        NaN   \n",
       "\n",
       "      FinancialStmtAttached FollowSFAS117 ForeignActivities  \\\n",
       "18819                   NaN           NaN               NaN   \n",
       "\n",
       "      ForeignActivitiesInd ForeignAddress ForeignCountryCd  \\\n",
       "18819                    0            NaN              NaN   \n",
       "\n",
       "      ForeignFinancialAccount ForeignFinancialAccountInd ForeignGrants  \\\n",
       "18819                     NaN                          0           NaN   \n",
       "\n",
       "      ForeignGrantsGrp ForeignOffice ForeignOfficeInd Form1098CFiled  \\\n",
       "18819              NaN           NaN                0            NaN   \n",
       "\n",
       "      Form1098CFiledInd Form720Filed Form8282FiledCnt  \\\n",
       "18819               NaN          NaN              NaN   \n",
       "\n",
       "      Form8282PropertyDisposedOf Form8282PropertyDisposedOfInd Form8886TFiled  \\\n",
       "18819                        NaN                             0            NaN   \n",
       "\n",
       "      Form8886TFiledInd Form8899Filed Form8899Filedind Form990-TFiled  \\\n",
       "18819               NaN           NaN              NaN            NaN   \n",
       "\n",
       "      Form990PartVIISectionA  \\\n",
       "18819                    NaN   \n",
       "\n",
       "                                                                                                                                                                                                                                       Form990PartVIISectionAGrp  \\\n",
       "18819  [{u'PersonNm': u'DR EDWARD J BENZ', u'IndividualTrusteeOrDirectorInd': u'X', u'AverageHoursPerWeekRt': u'2.00', u'ReportableCompFromRltdOrgAmt': u'0', u'OtherCompensationAmt': u'0', u'TitleTxt': u'VICE CHAIRMAN OF THE BOARD', u'OfficerInd': u'X',...   \n",
       "\n",
       "      Form990ProvidedToGoverningBody Form990ProvidedToGvrnBodyInd  \\\n",
       "18819                            NaN                            1   \n",
       "\n",
       "      Form990TFiledInd FormType FormationYr FormerOfcrEmployeesListedInd  \\\n",
       "18819              NaN      990        1898                            0   \n",
       "\n",
       "      FormersListed FundraisingActivities  \\\n",
       "18819           NaN                   NaN   \n",
       "\n",
       "                                             FundraisingActivitiesInd  \\\n",
       "18819  {u'#text': u'1', u'@referenceDocumentId': u'RetDoc1041300001'}   \n",
       "\n",
       "      FundraisingAmt FundraisingDirectExpenses FundraisingDirectExpensesAmt  \\\n",
       "18819            800                       NaN                        37598   \n",
       "\n",
       "      FundraisingEvents FundraisingGrossIncomeAmt FundsToPayPremiums  \\\n",
       "18819               NaN                     38964                NaN   \n",
       "\n",
       "      GainOrLoss             GainOrLossGrp Gaming  \\\n",
       "18819        NaN  {u'OtherAmt': u'322752'}    NaN   \n",
       "\n",
       "                                                  GamingActivitiesInd  \\\n",
       "18819  {u'#text': u'0', u'@referenceDocumentId': u'RetDoc1041300001'}   \n",
       "\n",
       "      GamingDirectExpenses GamingDirectExpensesAmt GamingGrossIncomeAmt  \\\n",
       "18819                  NaN                     NaN                  NaN   \n",
       "\n",
       "      GoverningBodyVotingMembersCnt GovernmentGrants GovernmentGrantsAmt  \\\n",
       "18819                            22              NaN             7932282   \n",
       "\n",
       "      GrantAmt GrantToRelatedPerson  \\\n",
       "18819      NaN                  NaN   \n",
       "\n",
       "                                              GrantToRelatedPersonInd Grants  \\\n",
       "18819  {u'#text': u'0', u'@referenceDocumentId': u'RetDoc1042800001'}    NaN   \n",
       "\n",
       "      GrantsAndSimilarAmntsCY GrantsAndSimilarAmntsPriorYear GrantsPayable  \\\n",
       "18819                     NaN                            NaN           NaN   \n",
       "\n",
       "      GrantsPayableGrp GrantsToDomesticIndividuals  \\\n",
       "18819              NaN                         NaN   \n",
       "\n",
       "      GrantsToDomesticIndividualsGrp GrantsToDomesticOrgs  \\\n",
       "18819                            NaN                  NaN   \n",
       "\n",
       "      GrantsToDomesticOrgsGrp GrantsToIndividuals GrantsToIndividualsInd  \\\n",
       "18819                     NaN                 NaN                      0   \n",
       "\n",
       "      GrantsToOrganizations GrantsToOrganizationsInd GrossAmountSalesAssets  \\\n",
       "18819                   NaN                        0                    NaN   \n",
       "\n",
       "      GrossAmountSalesAssetsGrp GrossIncomeFundraisingEvents  \\\n",
       "18819  {u'OtherAmt': u'340000'}                          NaN   \n",
       "\n",
       "      GrossIncomeGaming GrossReceipts GrossReceiptsAmt GrossRents  \\\n",
       "18819               NaN           NaN         13022814        NaN   \n",
       "\n",
       "      GrossRentsGrp GrossSalesOfInventory GrossSalesOfInventoryAmt  \\\n",
       "18819           NaN                   NaN                       38   \n",
       "\n",
       "      GroupExemptionNum GroupExemptionNumber GroupReturnForAffiliates  \\\n",
       "18819               NaN                  NaN                      NaN   \n",
       "\n",
       "      GroupReturnForAffiliatesInd Hospital IRPDocumentCnt IRPDocumentW2GCnt  \\\n",
       "18819                           0      NaN            158                 0   \n",
       "\n",
       "                                              IncludeFIN48FootnoteInd  \\\n",
       "18819  {u'#text': u'1', u'@referenceDocumentId': u'RetDoc1040000001'}   \n",
       "\n",
       "      IncmFromInvestBondProceedsGrp IncomeFromInvestBondProceeds  \\\n",
       "18819                           NaN                          NaN   \n",
       "\n",
       "      IndependentAuditFinancialStmt  \\\n",
       "18819                           NaN   \n",
       "\n",
       "                                         IndependentAuditFinclStmtInd  \\\n",
       "18819  {u'#text': u'1', u'@referenceDocumentId': u'RetDoc1040000001'}   \n",
       "\n",
       "      IndependentVotingMemberCnt IndivRcvdGreaterThan100KCnt  \\\n",
       "18819                         21                           3   \n",
       "\n",
       "      IndoorTanningServices IndoorTanningServicesInd InfoInScheduleOPartIII  \\\n",
       "18819                   NaN                        0                    NaN   \n",
       "\n",
       "      InfoInScheduleOPartIIIInd InfoInScheduleOPartIX  \\\n",
       "18819                         X                   NaN   \n",
       "\n",
       "      InfoInScheduleOPartIXInd InfoInScheduleOPartV InfoInScheduleOPartVI  \\\n",
       "18819                      NaN                  NaN                   NaN   \n",
       "\n",
       "      InfoInScheduleOPartVII InfoInScheduleOPartVIII  \\\n",
       "18819                    NaN                     NaN   \n",
       "\n",
       "      InfoInScheduleOPartVIIIInd InfoInScheduleOPartVIIInd  \\\n",
       "18819                        NaN                       NaN   \n",
       "\n",
       "      InfoInScheduleOPartVIInd InfoInScheduleOPartVInd InfoInScheduleOPartX  \\\n",
       "18819                        X                     NaN                  NaN   \n",
       "\n",
       "      InfoInScheduleOPartXI InfoInScheduleOPartXII InfoInScheduleOPartXIIInd  \\\n",
       "18819                   NaN                    NaN                         X   \n",
       "\n",
       "      InfoInScheduleOPartXIInd InfoInScheduleOPartXInd InformationTechnology  \\\n",
       "18819                        X                     NaN                   NaN   \n",
       "\n",
       "                                                                            InformationTechnologyGrp  \\\n",
       "18819  {u'TotalAmt': u'49382', u'ProgramServicesAmt': u'8673', u'ManagementAndGeneralAmt': u'40709'}   \n",
       "\n",
       "      InitialReturn InitialReturnInd Insurance  \\\n",
       "18819           NaN              NaN       NaN   \n",
       "\n",
       "                                                                                                                    InsuranceGrp  \\\n",
       "18819  {u'TotalAmt': u'92396', u'ProgramServicesAmt': u'10397', u'ManagementAndGeneralAmt': u'81746', u'FundraisingAmt': u'253'}   \n",
       "\n",
       "      IntangibleAssets                           IntangibleAssetsGrp Interest  \\\n",
       "18819              NaN  {u'BOYAmt': u'147691', u'EOYAmt': u'141271'}      NaN   \n",
       "\n",
       "                                                                                            InterestGrp  \\\n",
       "18819  {u'TotalAmt': u'255985', u'ProgramServicesAmt': u'191989', u'ManagementAndGeneralAmt': u'63996'}   \n",
       "\n",
       "      InventoriesForSaleOrUse InventoriesForSaleOrUseGrp InvestTaxExemptBonds  \\\n",
       "18819                     NaN       {u'BOYAmt': u'8329'}                  NaN   \n",
       "\n",
       "      InvestTaxExemptBondsInd InvestmentExpenseAmt InvestmentInJointVenture  \\\n",
       "18819                       0                  NaN                      NaN   \n",
       "\n",
       "      InvestmentInJointVentureInd InvestmentIncome  \\\n",
       "18819                           0              NaN   \n",
       "\n",
       "      InvestmentIncomeCurrentYear  \\\n",
       "18819                         NaN   \n",
       "\n",
       "                                                     InvestmentIncomeGrp  \\\n",
       "18819  {u'TotalRevenueColumnAmt': u'659520', u'ExclusionAmt': u'659520'}   \n",
       "\n",
       "      InvestmentIncomePriorYear InvestmentsOtherSecurities  \\\n",
       "18819                       NaN                        NaN   \n",
       "\n",
       "      InvestmentsOtherSecuritiesGrp InvestmentsProgramRelated  \\\n",
       "18819                           NaN                       NaN   \n",
       "\n",
       "      InvestmentsProgramRelatedGrp  \\\n",
       "18819                          NaN   \n",
       "\n",
       "                           InvestmentsPubTradedSecGrp  \\\n",
       "18819  {u'BOYAmt': u'4031973', u'EOYAmt': u'4747115'}   \n",
       "\n",
       "      InvestmentsPubTradedSecurities IsAvailable IsElectronic JointCosts  \\\n",
       "18819                            NaN        True         True        NaN   \n",
       "\n",
       "      JointCostsInd LandBldgEquipAccumDeprecAmt  \\\n",
       "18819           NaN                    11322245   \n",
       "\n",
       "                               LandBldgEquipBasisNetGrp  \\\n",
       "18819  {u'BOYAmt': u'14025403', u'EOYAmt': u'14020382'}   \n",
       "\n",
       "      LandBldgEquipCostOrOtherBssAmt LandBldgEquipmentAccumDeprec  \\\n",
       "18819                       25342627                          NaN   \n",
       "\n",
       "      LandBuildingsEquipmentBasis LandBuildingsEquipmentBasisNet  \\\n",
       "18819                         NaN                            NaN   \n",
       "\n",
       "               LastUpdated LegalDomicileStateCd LessCostOthBasisSalesExpenses  \\\n",
       "18819  2016-04-29T13:40:20                   ME                           NaN   \n",
       "\n",
       "      LessCostOthBasisSalesExpnssGrp LessRentalExpenses LessRentalExpensesGrp  \\\n",
       "18819        {u'OtherAmt': u'17248'}                NaN                   NaN   \n",
       "\n",
       "      LicensedMoreThanOneState LicensedMoreThanOneStateInd  \\\n",
       "18819                      NaN                         NaN   \n",
       "\n",
       "                                                   LoanOutstandingInd  \\\n",
       "18819  {u'#text': u'0', u'@referenceDocumentId': u'RetDoc1042800001'}   \n",
       "\n",
       "      LoanToOfficerOrDQP LoansFromOfficersDirectors  \\\n",
       "18819                NaN                        NaN   \n",
       "\n",
       "      LoansFromOfficersDirectorsGrp LobbyingActivities  \\\n",
       "18819                           NaN                NaN   \n",
       "\n",
       "                                                LobbyingActivitiesInd  \\\n",
       "18819  {u'#text': u'1', u'@referenceDocumentId': u'RetDoc1039700001'}   \n",
       "\n",
       "      LocalChapters LocalChaptersInd MaterialDiversionOrMisuse  \\\n",
       "18819           NaN                0                       NaN   \n",
       "\n",
       "      MaterialDiversionOrMisuseInd MembersOrStockholders  \\\n",
       "18819                            0                   NaN   \n",
       "\n",
       "      MembersOrStockholdersInd MembershipDues MembershipDuesAmt  \\\n",
       "18819                        1            NaN              4600   \n",
       "\n",
       "      MethodOfAccountingAccrual MethodOfAccountingAccrualInd  \\\n",
       "18819                       NaN                            X   \n",
       "\n",
       "      MethodOfAccountingCash MethodOfAccountingCashInd  \\\n",
       "18819                    NaN                       NaN   \n",
       "\n",
       "      MethodOfAccountingOther MethodOfAccountingOtherInd MinutesOfCommittees  \\\n",
       "18819                     NaN                        NaN                 NaN   \n",
       "\n",
       "      MinutesOfCommitteesInd MinutesOfGoverningBody MinutesOfGoverningBodyInd  \\\n",
       "18819                      1                    NaN                         1   \n",
       "\n",
       "      MiscellaneousRevenue MiscellaneousRevenueGrp  \\\n",
       "18819                  NaN                     NaN   \n",
       "\n",
       "                                                                                                                                                                                                                                                     MissionDesc  \\\n",
       "18819  THE MDI BIOLOGICAL LABORATORY IS A RAPIDLY GROWING, INDEPENDENT NON-PROFIT BIOMEDICAL RESEARCH INSTITUTION. ITS MISSION IS TO IMPROVE HUMAN HEALTH AND WELL-BEING THROUGH RESEARCH, EDUCATION,AND DEVELOPMENT VENTURES THAT TRANSFORM DISCOVERIES INTO...   \n",
       "\n",
       "      MissionDescription MoreThan5000KToIndividuals  \\\n",
       "18819                NaN                        NaN   \n",
       "\n",
       "      MoreThan5000KToIndividualsInd MoreThan5000KToOrgInd  \\\n",
       "18819                             0                     0   \n",
       "\n",
       "      MoreThan5000KToOrganizations MortNotesPyblSecuredInvestProp  \\\n",
       "18819                          NaN                            NaN   \n",
       "\n",
       "      MortgNotesPyblScrdInvstPropGrp NameOfForeignCountry  \\\n",
       "18819                            NaN                  NaN   \n",
       "\n",
       "      NameOfPrincipalOfficerBusiness NameOfPrincipalOfficerPerson  \\\n",
       "18819                            NaN                          NaN   \n",
       "\n",
       "      NbrIndependentVotingMembers NbrVotingGoverningBodyMembers  \\\n",
       "18819                         NaN                           NaN   \n",
       "\n",
       "      NbrVotingMembersGoverningBody NetAssetsOrFundBalancesBOY  \\\n",
       "18819                           NaN                        NaN   \n",
       "\n",
       "      NetAssetsOrFundBalancesBOYAmt NetAssetsOrFundBalancesEOY  \\\n",
       "18819                      21657900                        NaN   \n",
       "\n",
       "      NetAssetsOrFundBalancesEOYAmt NetGainOrLossInvestments  \\\n",
       "18819                      23690097                      NaN   \n",
       "\n",
       "                                             NetGainOrLossInvestmentsGrp  \\\n",
       "18819  {u'TotalRevenueColumnAmt': u'322752', u'ExclusionAmt': u'322752'}   \n",
       "\n",
       "                                        NetIncmFromFundraisingEvtGrp  \\\n",
       "18819  {u'TotalRevenueColumnAmt': u'1366', u'ExclusionAmt': u'1366'}   \n",
       "\n",
       "      NetIncomeFromFundraisingEvents NetIncomeFromGaming  \\\n",
       "18819                            NaN                 NaN   \n",
       "\n",
       "      NetIncomeFromGamingGrp NetIncomeOrLoss  \\\n",
       "18819                    NaN             NaN   \n",
       "\n",
       "                                              NetIncomeOrLossGrp  \\\n",
       "18819  {u'TotalRevenueColumnAmt': u'38', u'ExclusionAmt': u'38'}   \n",
       "\n",
       "      NetRentalIncomeOrLoss NetRentalIncomeOrLossGrp  \\\n",
       "18819                   NaN                      NaN   \n",
       "\n",
       "      NetUnrelatedBusTxblIncmAmt NetUnrelatedBusinessTxblIncome  \\\n",
       "18819                          0                            NaN   \n",
       "\n",
       "      NetUnrlzdGainsLossesInvstAmt NoListedPersonsCompensated  \\\n",
       "18819                      -104353                        NaN   \n",
       "\n",
       "      NoListedPersonsCompensatedInd NonDeductibleContributions  \\\n",
       "18819                             X                        NaN   \n",
       "\n",
       "      NonDeductibleDisclosure NoncashContributions NoncashContributionsAmt  \\\n",
       "18819                     NaN                  NaN                   57474   \n",
       "\n",
       "      NondeductibleContriDisclInd NondeductibleContributionsInd  \\\n",
       "18819                         NaN                             0   \n",
       "\n",
       "      NumberFormsTransmittedWith1096 NumberIndependentVotingMembers  \\\n",
       "18819                            NaN                            NaN   \n",
       "\n",
       "      NumberIndividualsGT100K NumberOf8282Filed NumberOfContractorsGT100K  \\\n",
       "18819                     NaN               NaN                       NaN   \n",
       "\n",
       "      NumberOfEmployees NumberW2GIncluded            ObjectId Occupancy  \\\n",
       "18819               NaN               NaN  201533209349308003       NaN   \n",
       "\n",
       "                                                                                                                       OccupancyGrp  \\\n",
       "18819  {u'TotalAmt': u'587738', u'ProgramServicesAmt': u'494365', u'ManagementAndGeneralAmt': u'91234', u'FundraisingAmt': u'2139'}   \n",
       "\n",
       "      OfficeExpenses OfficeExpensesGrp OfficerEntityWithBsnssRltnshp  \\\n",
       "18819            NaN               NaN                           NaN   \n",
       "\n",
       "      OfficerMailingAddress OfficerMailingAddressInd OnBehalfOfIssuer  \\\n",
       "18819                   NaN                        0              NaN   \n",
       "\n",
       "      OnBehalfOfIssuerInd OperateHospitalInd OrgDoesNotFollowSFAS117Ind  \\\n",
       "18819                   0                  0                        NaN   \n",
       "\n",
       "      OrgFiledInLieuOfForm1041Ind Organization501c Organization501c3  \\\n",
       "18819                         NaN              NaN               NaN   \n",
       "\n",
       "      Organization501c3Ind Organization501cInd OrganizationFollowsSFAS117Ind  \\\n",
       "18819                    X                 NaN                             X   \n",
       "\n",
       "                                OrganizationName  \\\n",
       "18819  MOUNT DESERT ISLAND BIOLOGICAL LABORATORY   \n",
       "\n",
       "      OthNotesLoansReceivableNetGrp OtherAssetsTotal OtherAssetsTotalGrp  \\\n",
       "18819                           NaN              NaN                 NaN   \n",
       "\n",
       "      OtherChangesInNetAssetsAmt OtherEmployeeBenefits  \\\n",
       "18819                          3                   NaN   \n",
       "\n",
       "                                                                                                            OtherEmployeeBenefitsGrp  \\\n",
       "18819  {u'TotalAmt': u'370740', u'ProgramServicesAmt': u'284086', u'ManagementAndGeneralAmt': u'70910', u'FundraisingAmt': u'15744'}   \n",
       "\n",
       "      OtherExpensePriorYear OtherExpenses OtherExpensesCurrentYear  \\\n",
       "18819                   NaN           NaN                      NaN   \n",
       "\n",
       "                                                                                                                                                                                                                                                OtherExpensesGrp  \\\n",
       "18819  [{u'TotalAmt': u'2124748', u'ProgramServicesAmt': u'2124748', u'Desc': u'SUBAWARDS'}, {u'TotalAmt': u'682378', u'ProgramServicesAmt': u'627645', u'ManagementAndGeneralAmt': u'47547', u'FundraisingAmt': u'7186', u'Desc': u'SUPPLIES'}, {u'TotalAmt'...   \n",
       "\n",
       "      OtherExplainInSchO OtherInd OtherLiabilities  \\\n",
       "18819                NaN      NaN              NaN   \n",
       "\n",
       "                                  OtherLiabilitiesGrp  \\\n",
       "18819  {u'BOYAmt': u'4630000', u'EOYAmt': u'4525000'}   \n",
       "\n",
       "      OtherNotesLoansReceivableNet OtherOrganizationDsc  \\\n",
       "18819                          NaN                  NaN   \n",
       "\n",
       "      OtherRevenueCurrentYear OtherRevenueMisc OtherRevenueMiscGrp  \\\n",
       "18819                     NaN              NaN                 NaN   \n",
       "\n",
       "      OtherRevenuePriorYear OtherRevenueTotalAmt OtherSalariesAndWages  \\\n",
       "18819                   NaN                  NaN                   NaN   \n",
       "\n",
       "                                                                                                                OtherSalariesAndWagesGrp  \\\n",
       "18819  {u'TotalAmt': u'3353971', u'ProgramServicesAmt': u'2478596', u'ManagementAndGeneralAmt': u'683266', u'FundraisingAmt': u'192109'}   \n",
       "\n",
       "      OtherWebsite OtherWebsiteInd OwnWebsite OwnWebsiteInd  \\\n",
       "18819          NaN               X        NaN             X   \n",
       "\n",
       "      PYBenefitsPaidToMembersAmt PYContributionsGrantsAmt  \\\n",
       "18819                          0                  7802139   \n",
       "\n",
       "                                              PYExcessBenefitTransInd  \\\n",
       "18819  {u'#text': u'0', u'@referenceDocumentId': u'RetDoc1042800001'}   \n",
       "\n",
       "      PYGrantsAndSimilarPaidAmt PYInvestmentIncomeAmt PYOtherExpensesAmt  \\\n",
       "18819                         0                785633            6736413   \n",
       "\n",
       "      PYOtherRevenueAmt PYProgramServiceRevenueAmt PYRevenuesLessExpensesAmt  \\\n",
       "18819            -13375                    1591204                   -449665   \n",
       "\n",
       "      PYSalariesCompEmpBnftPaidAmt PYTotalExpensesAmt  \\\n",
       "18819                      3878853           10615266   \n",
       "\n",
       "      PYTotalProfFndrsngExpnsAmt PYTotalRevenueAmt  \\\n",
       "18819                          0          10165601   \n",
       "\n",
       "      PaidInCapSrplsLandBldgEqpFund PartialLiquidation PartialLiquidationInd  \\\n",
       "18819                           NaN                NaN                     0   \n",
       "\n",
       "      PayPremiumsPrsnlBnftCntrctInd PaymentsToAffiliates  \\\n",
       "18819                           NaN                  NaN   \n",
       "\n",
       "      PaymentsToAffiliatesGrp PayrollTaxes  \\\n",
       "18819                     NaN          NaN   \n",
       "\n",
       "                                                                                                                     PayrollTaxesGrp  \\\n",
       "18819  {u'TotalAmt': u'257923', u'ProgramServicesAmt': u'189245', u'ManagementAndGeneralAmt': u'52592', u'FundraisingAmt': u'16086'}   \n",
       "\n",
       "      PdInCapSrplsLandBldgEqpFundGrp PensionPlanContributions  \\\n",
       "18819                            NaN                      NaN   \n",
       "\n",
       "                                                                                                         PensionPlanContributionsGrp  \\\n",
       "18819  {u'TotalAmt': u'172505', u'ProgramServicesAmt': u'125037', u'ManagementAndGeneralAmt': u'37384', u'FundraisingAmt': u'10084'}   \n",
       "\n",
       "      PermanentlyRestrictedNetAssets  \\\n",
       "18819                            NaN   \n",
       "\n",
       "                          PermanentlyRstrNetAssetsGrp  \\\n",
       "18819  {u'BOYAmt': u'4193257', u'EOYAmt': u'4433997'}   \n",
       "\n",
       "      PledgesAndGrantsReceivable  \\\n",
       "18819                        NaN   \n",
       "\n",
       "                      PledgesAndGrantsReceivableGrp PoliciesReferenceChapters  \\\n",
       "18819  {u'BOYAmt': u'797504', u'EOYAmt': u'986621'}                       NaN   \n",
       "\n",
       "      PoliciesReferenceChaptersInd PoliticalActivities  \\\n",
       "18819                          NaN                 NaN   \n",
       "\n",
       "                                             PoliticalCampaignActyInd  \\\n",
       "18819  {u'#text': u'0', u'@referenceDocumentId': u'RetDoc1039700001'}   \n",
       "\n",
       "      PremiumsPaid PrepaidExpensesDeferredCharges  \\\n",
       "18819          NaN                            NaN   \n",
       "\n",
       "                 PrepaidExpensesDefrdChargesGrp PrincipalOfcrBusinessAddress  \\\n",
       "18819  {u'BOYAmt': u'4791', u'EOYAmt': u'5301'}                          NaN   \n",
       "\n",
       "      PrincipalOfcrBusinessName    PrincipalOfficerNm  \\\n",
       "18819                       NaN  DR KEVIN STRANGE PHD   \n",
       "\n",
       "      PriorExcessBenefitTransaction PriorPeriodAdjustmentsAmt  \\\n",
       "18819                           NaN                       NaN   \n",
       "\n",
       "      ProfessionalFundraising  \\\n",
       "18819                     NaN   \n",
       "\n",
       "                                           ProfessionalFundraisingInd  \\\n",
       "18819  {u'#text': u'0', u'@referenceDocumentId': u'RetDoc1041300001'}   \n",
       "\n",
       "      ProgSrvcAccomActy2Grp ProgSrvcAccomActy3Grp ProgSrvcAccomActyOtherGrp  \\\n",
       "18819                   NaN                   NaN                       NaN   \n",
       "\n",
       "      ProgramServiceRevenue ProgramServiceRevenueCY  \\\n",
       "18819                   NaN                     NaN   \n",
       "\n",
       "                                                                                                                                                                                                                                        ProgramServiceRevenueGrp  \\\n",
       "18819  [{u'TotalRevenueColumnAmt': u'469300', u'BusinessCd': u'611710', u'RelatedOrExemptFuncIncomeAmt': u'469300', u'Desc': u'CONFER & COURSE FEES'}, {u'TotalRevenueColumnAmt': u'353507', u'BusinessCd': u'541700', u'RelatedOrExemptFuncIncomeAmt': u'353...   \n",
       "\n",
       "      ProgramServiceRevenuePriorYear ProhibitedTaxShelterTrans  \\\n",
       "18819                            NaN                       NaN   \n",
       "\n",
       "      ProhibitedTaxShelterTransInd PymtTravelEntrtnmntPubOfclGrp  \\\n",
       "18819                            0                           NaN   \n",
       "\n",
       "      QuidProQuoContriDisclInd QuidProQuoContributions  \\\n",
       "18819                      NaN                     NaN   \n",
       "\n",
       "      QuidProQuoContributionsInd QuidProQuoDisclosure  \\\n",
       "18819                          0                  NaN   \n",
       "\n",
       "      RcvFndsToPayPrsnlBnftCntrctInd RcvblFromDisqualifiedPrsnGrp  \\\n",
       "18819                            NaN                          NaN   \n",
       "\n",
       "      ReceivablesFromDisqualPersons ReceivablesFromOfficersEtc  \\\n",
       "18819                           NaN                        NaN   \n",
       "\n",
       "      ReceivablesFromOfficersEtcGrp ReconcilationDonatedServices  \\\n",
       "18819                           NaN                          NaN   \n",
       "\n",
       "      ReconcilationInvestExpenses ReconcilationOtherChanges  \\\n",
       "18819                         NaN                       NaN   \n",
       "\n",
       "      ReconcilationPriorAdjustment ReconcilationRevenueExpenses  \\\n",
       "18819                          NaN                          NaN   \n",
       "\n",
       "      ReconcilationRevenueExpnssAmt ReconciliationUnrealizedInvest  \\\n",
       "18819                       2136547                            NaN   \n",
       "\n",
       "      RegularMonitoringEnforcement RegularMonitoringEnfrcInd RelatedEntity  \\\n",
       "18819                          NaN                         1           NaN   \n",
       "\n",
       "      RelatedEntityInd RelatedOrgControlledEntity  \\\n",
       "18819                0                        NaN   \n",
       "\n",
       "      RelatedOrganizationCtrlEntInd RelatedOrganizations  \\\n",
       "18819                             0                  NaN   \n",
       "\n",
       "      RelatedOrganizationsAmt RentalIncomeOrLoss RentalIncomeOrLossGrp  \\\n",
       "18819                     NaN                NaN                   NaN   \n",
       "\n",
       "      ReportFin48Footnote ReportInvestOthSecurities  \\\n",
       "18819                 NaN                       NaN   \n",
       "\n",
       "                                         ReportInvestmentsOtherSecInd  \\\n",
       "18819  {u'#text': u'0', u'@referenceDocumentId': u'RetDoc1040000001'}   \n",
       "\n",
       "      ReportLandBldgEquip  \\\n",
       "18819                 NaN   \n",
       "\n",
       "                                       ReportLandBuildingEquipmentInd  \\\n",
       "18819  {u'#text': u'1', u'@referenceDocumentId': u'RetDoc1040000001'}   \n",
       "\n",
       "      ReportOtherAssets  \\\n",
       "18819               NaN   \n",
       "\n",
       "                                                 ReportOtherAssetsInd  \\\n",
       "18819  {u'#text': u'0', u'@referenceDocumentId': u'RetDoc1040000001'}   \n",
       "\n",
       "      ReportOtherLiabilities  \\\n",
       "18819                    NaN   \n",
       "\n",
       "                                            ReportOtherLiabilitiesInd  \\\n",
       "18819  {u'#text': u'1', u'@referenceDocumentId': u'RetDoc1040000001'}   \n",
       "\n",
       "      ReportProgRelInvest  \\\n",
       "18819                 NaN   \n",
       "\n",
       "                                         ReportProgramRelatedInvstInd  \\\n",
       "18819  {u'#text': u'0', u'@referenceDocumentId': u'RetDoc1040000001'}   \n",
       "\n",
       "      ReservesMaintainedAmt RetainedEarningsEndowmentEtc Revenue RevenueAmt  \\\n",
       "18819                   NaN                          NaN     NaN    1733437   \n",
       "\n",
       "      RevenuesLessExpensesCY RevenuesLessExpensesPriorYear Royalties  \\\n",
       "18819                    NaN                           NaN       NaN   \n",
       "\n",
       "      RoyaltiesGrp RoyaltiesRevenue RoyaltiesRevenueGrp  \\\n",
       "18819          NaN              NaN                 NaN   \n",
       "\n",
       "      RtnEarnEndowmentIncmOthFndsGrp SalariesEtcCurrentYear  \\\n",
       "18819                            NaN                    NaN   \n",
       "\n",
       "      SalariesEtcPriorYear SavingsAndTempCashInvestments  \\\n",
       "18819                  NaN                           NaN   \n",
       "\n",
       "                           SavingsAndTempCashInvstGrp ScheduleBRequired  \\\n",
       "18819  {u'BOYAmt': u'6870518', u'EOYAmt': u'7005488'}               NaN   \n",
       "\n",
       "                                                 ScheduleBRequiredInd  \\\n",
       "18819  {u'#text': u'1', u'@referenceDocumentId': u'RetDoc1234500001'}   \n",
       "\n",
       "      ScheduleJRequired  \\\n",
       "18819               NaN   \n",
       "\n",
       "                                                 ScheduleJRequiredInd  \\\n",
       "18819  {u'#text': u'1', u'@referenceDocumentId': u'RetDoc1042400001'}   \n",
       "\n",
       "      ScheduleORequired ScheduleORequiredInd School SchoolOperatingInd  \\\n",
       "18819               NaN                    1    NaN                  0   \n",
       "\n",
       "      SignificantChange SignificantChangeInd SignificantNewProgramServices  \\\n",
       "18819               NaN                    0                           NaN   \n",
       "\n",
       "      SignificantNewProgramSrvcInd SpecialConditionDescription  \\\n",
       "18819                            0                         NaN   \n",
       "\n",
       "      StateLegalDomicile StateRequiredReservesAmt  \\\n",
       "18819                NaN                      NaN   \n",
       "\n",
       "      StatesWhereCopyOfReturnIsFiled StatesWhereCopyOfReturnIsFldCd  \\\n",
       "18819                            NaN                             ME   \n",
       "\n",
       "      SubjectToProxyTax  \\\n",
       "18819               NaN   \n",
       "\n",
       "                                                 SubjectToProxyTaxInd  \\\n",
       "18819  {u'#text': u'0', u'@referenceDocumentId': u'RetDoc1039700001'}   \n",
       "\n",
       "      SubmittedOn TaxExemptBondLiabilities TaxExemptBondLiabilitiesGrp  \\\n",
       "18819  2016-02-16                      NaN                         NaN   \n",
       "\n",
       "      TaxExemptBonds  \\\n",
       "18819            NaN   \n",
       "\n",
       "                                                    TaxExemptBondsInd  \\\n",
       "18819  {u'#text': u'1', u'@referenceDocumentId': u'RetDoc1053100001'}   \n",
       "\n",
       "      TaxPeriod TaxableDistributions TaxableDistributionsInd  \\\n",
       "18819    201412                  NaN                     NaN   \n",
       "\n",
       "      TaxablePartyNotification TaxablePartyNotificationInd  \\\n",
       "18819                      NaN                           0   \n",
       "\n",
       "                                         TempOrPermanentEndowmentsInd  \\\n",
       "18819  {u'#text': u'1', u'@referenceDocumentId': u'RetDoc1040000001'}   \n",
       "\n",
       "      TemporarilyRestrictedNetAssets  \\\n",
       "18819                            NaN   \n",
       "\n",
       "                          TemporarilyRstrNetAssetsGrp  \\\n",
       "18819  {u'BOYAmt': u'5880227', u'EOYAmt': u'7351259'}   \n",
       "\n",
       "      TermOrPermanentEndowments TerminateOperationsInd Terminated  \\\n",
       "18819                       NaN                      0        NaN   \n",
       "\n",
       "      TerminationOrContraction TheBooksAreInCareOf  \\\n",
       "18819                      NaN                 NaN   \n",
       "\n",
       "                         TotLiabNetAssetsFundBalanceGrp  \\\n",
       "18819  {u'BOYAmt': u'27323685', u'EOYAmt': u'29607771'}   \n",
       "\n",
       "      TotReportableCompRltdOrgAmt TotalAssets TotalAssetsBOY  \\\n",
       "18819                           0         NaN            NaN   \n",
       "\n",
       "      TotalAssetsBOYAmt TotalAssetsEOY TotalAssetsEOYAmt  \\\n",
       "18819          27323685            NaN          29607771   \n",
       "\n",
       "                                         TotalAssetsGrp TotalCompGT150K  \\\n",
       "18819  {u'BOYAmt': u'27323685', u'EOYAmt': u'29607771'}             NaN   \n",
       "\n",
       "      TotalCompGreaterThan150KInd TotalContributions TotalContributionsAmt  \\\n",
       "18819                           1                NaN              10272455   \n",
       "\n",
       "      TotalEmployeeCnt TotalExpensesCurrentYear TotalExpensesPriorYear  \\\n",
       "18819               96                      NaN                    NaN   \n",
       "\n",
       "      TotalFunctionalExpenses  \\\n",
       "18819                     NaN   \n",
       "\n",
       "                                                                                                                TotalFunctionalExpensesGrp  \\\n",
       "18819  {u'TotalAmt': u'10831421', u'ProgramServicesAmt': u'8605100', u'ManagementAndGeneralAmt': u'1671713', u'FundraisingAmt': u'554608'}   \n",
       "\n",
       "      TotalFundrsngExpCurrentYear TotalGrossUBI TotalGrossUBIAmt  \\\n",
       "18819                         NaN           NaN                0   \n",
       "\n",
       "      TotalJointCosts TotalJointCostsGrp TotalLiabNetAssetsFundBalances  \\\n",
       "18819             NaN                NaN                            NaN   \n",
       "\n",
       "      TotalLiabilities TotalLiabilitiesBOY TotalLiabilitiesBOYAmt  \\\n",
       "18819              NaN                 NaN                5665785   \n",
       "\n",
       "      TotalLiabilitiesEOY TotalLiabilitiesEOYAmt  \\\n",
       "18819                 NaN                5917674   \n",
       "\n",
       "                                  TotalLiabilitiesGrp TotalNbrEmployees  \\\n",
       "18819  {u'BOYAmt': u'5665785', u'EOYAmt': u'5917674'}               NaN   \n",
       "\n",
       "      TotalNbrVolunteers                      TotalNetAssetsFundBalanceGrp  \\\n",
       "18819                NaN  {u'BOYAmt': u'21657900', u'EOYAmt': u'23690097'}   \n",
       "\n",
       "      TotalNetAssetsFundBalances TotalOfOtherProgramServiceExp  \\\n",
       "18819                        NaN                           NaN   \n",
       "\n",
       "      TotalOfOtherProgramServiceGrnt TotalOfOtherProgramServiceRev  \\\n",
       "18819                            NaN                           NaN   \n",
       "\n",
       "                                                          TotalOthProgramServiceRevGrp  \\\n",
       "18819  {u'TotalRevenueColumnAmt': u'38356', u'RelatedOrExemptFuncIncomeAmt': u'38356'}   \n",
       "\n",
       "      TotalOthProgramServiceRevenue TotalOtherCompensation  \\\n",
       "18819                           NaN                    NaN   \n",
       "\n",
       "      TotalOtherCompensationAmt TotalOtherProgSrvcExpenseAmt  \\\n",
       "18819                     67100                          NaN   \n",
       "\n",
       "      TotalOtherProgSrvcGrantAmt TotalOtherProgSrvcRevenueAmt  \\\n",
       "18819                        NaN                          NaN   \n",
       "\n",
       "      TotalOtherRevenue TotalProfFundrsngExpCY TotalProfFundrsngExpPriorYear  \\\n",
       "18819               NaN                    NaN                           NaN   \n",
       "\n",
       "      TotalProgramServiceExpense TotalProgramServiceExpensesAmt  \\\n",
       "18819                        NaN                        8605100   \n",
       "\n",
       "      TotalProgramServiceRevenue TotalProgramServiceRevenueAmt  \\\n",
       "18819                        NaN                       1711837   \n",
       "\n",
       "      TotalReportableCompFrmRltdOrgs TotalReportableCompFromOrg  \\\n",
       "18819                            NaN                        NaN   \n",
       "\n",
       "      TotalReportableCompFromOrgAmt TotalRevenue TotalRevenueCurrentYear  \\\n",
       "18819                        496334          NaN                     NaN   \n",
       "\n",
       "                                                                                                                                              TotalRevenueGrp  \\\n",
       "18819  {u'UnrelatedBusinessRevenueAmt': u'0', u'TotalRevenueColumnAmt': u'12967968', u'RelatedOrExemptFuncIncomeAmt': u'1711837', u'ExclusionAmt': u'983676'}   \n",
       "\n",
       "      TotalRevenuePriorYear TotalVolunteersCnt TransactionRelatedEntity  \\\n",
       "18819                   NaN                125                      NaN   \n",
       "\n",
       "      TransactionWithControlEntInd TransfersToExemptNonChrtblOrg Travel  \\\n",
       "18819                          NaN                           NaN    NaN   \n",
       "\n",
       "      TravelEntrtnmntPublicOfficials  \\\n",
       "18819                            NaN   \n",
       "\n",
       "                                                                                                                          TravelGrp  \\\n",
       "18819  {u'TotalAmt': u'128657', u'ProgramServicesAmt': u'84416', u'ManagementAndGeneralAmt': u'33781', u'FundraisingAmt': u'10460'}   \n",
       "\n",
       "      TrnsfrExmptNonChrtblRltdOrgInd TypeOfOrgOtherDescription  \\\n",
       "18819                              0                       NaN   \n",
       "\n",
       "      TypeOfOrganizationAssocInd TypeOfOrganizationAssociation  \\\n",
       "18819                        NaN                           NaN   \n",
       "\n",
       "      TypeOfOrganizationCorpInd TypeOfOrganizationCorporation  \\\n",
       "18819                         X                           NaN   \n",
       "\n",
       "      TypeOfOrganizationOther TypeOfOrganizationOtherInd  \\\n",
       "18819                     NaN                        NaN   \n",
       "\n",
       "      TypeOfOrganizationTrust TypeOfOrganizationTrustInd  \\\n",
       "18819                     NaN                        NaN   \n",
       "\n",
       "                                                                       URL  \\\n",
       "18819  https://s3.amazonaws.com/irs-form-990/201533209349308003_public.xml   \n",
       "\n",
       "                                                                                                                     USAddress  \\\n",
       "18819  {u'CityNm': u'SALISBURY COVE', u'StateAbbreviationCd': u'ME', u'ZIPCd': u'046720035', u'AddressLine1Txt': u'PO BOX 35'}   \n",
       "\n",
       "      UnrelatedBusIncmOverLimitInd UnrelatedBusinessIncome  \\\n",
       "18819                            0                     NaN   \n",
       "\n",
       "      UnrestrictedNetAssets                          UnrestrictedNetAssetsGrp  \\\n",
       "18819                   NaN  {u'BOYAmt': u'11584416', u'EOYAmt': u'11904841'}   \n",
       "\n",
       "      UnsecuredNotesLoansPayable UnsecuredNotesLoansPayableGrp UponRequest  \\\n",
       "18819                        NaN                           NaN         NaN   \n",
       "\n",
       "      UponRequestInd VotingMembersGoverningBodyCnt  \\\n",
       "18819              X                            22   \n",
       "\n",
       "      VotingMembersIndependentCnt WebSite WebsiteAddressTxt  \\\n",
       "18819                          21     NaN     WWW.MDIBL.ORG   \n",
       "\n",
       "      WhistleblowerPolicy WhistleblowerPolicyInd WrittenPolicyOrProcedure  \\\n",
       "18819                 NaN                      1                      NaN   \n",
       "\n",
       "      WrittenPolicyOrProcedureInd YearFormation                       _id  \\\n",
       "18819                         NaN           NaN  58c33e003ffc5a1664e96b73   \n",
       "\n",
       "                                     FeesForServicesAccountingGrp_v2     FYE  \\\n",
       "18819  {u'TotalAmt': u'23700', u'ManagementAndGeneralAmt': u'23700'}  FY2014   \n",
       "\n",
       "       FeesForServicesAccounting_TotalAmt  FeesForServicesAccounting_binary  \n",
       "18819                             23700.0                                 1  "
      ]
     },
     "execution_count": 8,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_all = pd.read_pickle('e-file data for 7,133 of 8,304 2016 EINs.pkl')\n",
    "print '# of columns:', len(df_all.columns)\n",
    "print '# of observations:', len(df_all)\n",
    "df_all.head(1)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Identify Governance Variables"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      " # of columns: 4\n",
      "# of observations: 652\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>variable</th>\n",
       "      <th>description</th>\n",
       "      <th>line_number</th>\n",
       "      <th>first year</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>AccountantCompileOrReview</td>\n",
       "      <td>Accountant provide compilation or review?</td>\n",
       "      <td>Part XII Line 2a</td>\n",
       "      <td>2010</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                    variable                                description  \\\n",
       "0  AccountantCompileOrReview  Accountant provide compilation or review?   \n",
       "\n",
       "        line_number  first year  \n",
       "0  Part XII Line 2a        2010  "
      ]
     },
     "execution_count": 17,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "vars = pd.read_pickle('variable descriptions 2010-2015 990 e-file data (first year only).pkl')\n",
    "print '# of columns:', len(vars.columns)\n",
    "print '# of observations:', len(vars)\n",
    "vars.head(1)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "##### Independent and voting directors"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 37,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>FYE</th>\n",
       "      <th>EIN</th>\n",
       "      <th>NbrIndependentVotingMembers</th>\n",
       "      <th>NbrVotingGoverningBodyMembers</th>\n",
       "      <th>NbrVotingMembersGoverningBody</th>\n",
       "      <th>NumberIndependentVotingMembers</th>\n",
       "      <th>GoverningBodyVotingMembersCnt</th>\n",
       "      <th>IndependentVotingMemberCnt</th>\n",
       "      <th>VotingMembersGoverningBodyCnt</th>\n",
       "      <th>VotingMembersIndependentCnt</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>18819</th>\n",
       "      <td>FY2014</td>\n",
       "      <td>010202467</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>22</td>\n",
       "      <td>21</td>\n",
       "      <td>22</td>\n",
       "      <td>21</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7965</th>\n",
       "      <td>FY2013</td>\n",
       "      <td>010202467</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>21</td>\n",
       "      <td>21</td>\n",
       "      <td>21</td>\n",
       "      <td>21</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8989</th>\n",
       "      <td>FY2012</td>\n",
       "      <td>010202467</td>\n",
       "      <td>20</td>\n",
       "      <td>20</td>\n",
       "      <td>20</td>\n",
       "      <td>20</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>22904</th>\n",
       "      <td>FY2011</td>\n",
       "      <td>010202467</td>\n",
       "      <td>23</td>\n",
       "      <td>23</td>\n",
       "      <td>23</td>\n",
       "      <td>23</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>19649</th>\n",
       "      <td>FY2010</td>\n",
       "      <td>010202467</td>\n",
       "      <td>24</td>\n",
       "      <td>24</td>\n",
       "      <td>24</td>\n",
       "      <td>24</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>28581</th>\n",
       "      <td>FY2015</td>\n",
       "      <td>010211478</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>18</td>\n",
       "      <td>18</td>\n",
       "      <td>18</td>\n",
       "      <td>18</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>20683</th>\n",
       "      <td>FY2014</td>\n",
       "      <td>010211478</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>18</td>\n",
       "      <td>18</td>\n",
       "      <td>18</td>\n",
       "      <td>18</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4797</th>\n",
       "      <td>FY2013</td>\n",
       "      <td>010211478</td>\n",
       "      <td>18</td>\n",
       "      <td>18</td>\n",
       "      <td>18</td>\n",
       "      <td>18</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>15877</th>\n",
       "      <td>FY2012</td>\n",
       "      <td>010211478</td>\n",
       "      <td>18</td>\n",
       "      <td>18</td>\n",
       "      <td>18</td>\n",
       "      <td>18</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>18981</th>\n",
       "      <td>FY2011</td>\n",
       "      <td>010211478</td>\n",
       "      <td>19</td>\n",
       "      <td>19</td>\n",
       "      <td>19</td>\n",
       "      <td>19</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>589</th>\n",
       "      <td>FY2014</td>\n",
       "      <td>010211513</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>23</td>\n",
       "      <td>21</td>\n",
       "      <td>23</td>\n",
       "      <td>21</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>21460</th>\n",
       "      <td>FY2013</td>\n",
       "      <td>010211513</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>22</td>\n",
       "      <td>20</td>\n",
       "      <td>22</td>\n",
       "      <td>20</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4502</th>\n",
       "      <td>FY2012</td>\n",
       "      <td>010211513</td>\n",
       "      <td>20</td>\n",
       "      <td>22</td>\n",
       "      <td>22</td>\n",
       "      <td>20</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>13619</th>\n",
       "      <td>FY2011</td>\n",
       "      <td>010211513</td>\n",
       "      <td>21</td>\n",
       "      <td>21</td>\n",
       "      <td>21</td>\n",
       "      <td>21</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>17018</th>\n",
       "      <td>FY2010</td>\n",
       "      <td>010211513</td>\n",
       "      <td>24</td>\n",
       "      <td>25</td>\n",
       "      <td>25</td>\n",
       "      <td>24</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>28147</th>\n",
       "      <td>FY2014</td>\n",
       "      <td>010211530</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>28</td>\n",
       "      <td>28</td>\n",
       "      <td>28</td>\n",
       "      <td>28</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6614</th>\n",
       "      <td>FY2013</td>\n",
       "      <td>010211530</td>\n",
       "      <td>29</td>\n",
       "      <td>29</td>\n",
       "      <td>29</td>\n",
       "      <td>29</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>27887</th>\n",
       "      <td>FY2012</td>\n",
       "      <td>010211530</td>\n",
       "      <td>27</td>\n",
       "      <td>27</td>\n",
       "      <td>27</td>\n",
       "      <td>27</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>13438</th>\n",
       "      <td>FY2011</td>\n",
       "      <td>010211530</td>\n",
       "      <td>29</td>\n",
       "      <td>29</td>\n",
       "      <td>29</td>\n",
       "      <td>29</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>17003</th>\n",
       "      <td>FY2010</td>\n",
       "      <td>010211530</td>\n",
       "      <td>29</td>\n",
       "      <td>29</td>\n",
       "      <td>29</td>\n",
       "      <td>29</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "          FYE        EIN NbrIndependentVotingMembers  \\\n",
       "18819  FY2014  010202467                         NaN   \n",
       "7965   FY2013  010202467                         NaN   \n",
       "8989   FY2012  010202467                          20   \n",
       "22904  FY2011  010202467                          23   \n",
       "19649  FY2010  010202467                          24   \n",
       "28581  FY2015  010211478                         NaN   \n",
       "20683  FY2014  010211478                         NaN   \n",
       "4797   FY2013  010211478                          18   \n",
       "15877  FY2012  010211478                          18   \n",
       "18981  FY2011  010211478                          19   \n",
       "589    FY2014  010211513                         NaN   \n",
       "21460  FY2013  010211513                         NaN   \n",
       "4502   FY2012  010211513                          20   \n",
       "13619  FY2011  010211513                          21   \n",
       "17018  FY2010  010211513                          24   \n",
       "28147  FY2014  010211530                         NaN   \n",
       "6614   FY2013  010211530                          29   \n",
       "27887  FY2012  010211530                          27   \n",
       "13438  FY2011  010211530                          29   \n",
       "17003  FY2010  010211530                          29   \n",
       "\n",
       "      NbrVotingGoverningBodyMembers NbrVotingMembersGoverningBody  \\\n",
       "18819                           NaN                           NaN   \n",
       "7965                            NaN                           NaN   \n",
       "8989                             20                            20   \n",
       "22904                            23                            23   \n",
       "19649                            24                            24   \n",
       "28581                           NaN                           NaN   \n",
       "20683                           NaN                           NaN   \n",
       "4797                             18                            18   \n",
       "15877                            18                            18   \n",
       "18981                            19                            19   \n",
       "589                             NaN                           NaN   \n",
       "21460                           NaN                           NaN   \n",
       "4502                             22                            22   \n",
       "13619                            21                            21   \n",
       "17018                            25                            25   \n",
       "28147                           NaN                           NaN   \n",
       "6614                             29                            29   \n",
       "27887                            27                            27   \n",
       "13438                            29                            29   \n",
       "17003                            29                            29   \n",
       "\n",
       "      NumberIndependentVotingMembers GoverningBodyVotingMembersCnt  \\\n",
       "18819                            NaN                            22   \n",
       "7965                             NaN                            21   \n",
       "8989                              20                           NaN   \n",
       "22904                             23                           NaN   \n",
       "19649                             24                           NaN   \n",
       "28581                            NaN                            18   \n",
       "20683                            NaN                            18   \n",
       "4797                              18                           NaN   \n",
       "15877                             18                           NaN   \n",
       "18981                             19                           NaN   \n",
       "589                              NaN                            23   \n",
       "21460                            NaN                            22   \n",
       "4502                              20                           NaN   \n",
       "13619                             21                           NaN   \n",
       "17018                             24                           NaN   \n",
       "28147                            NaN                            28   \n",
       "6614                              29                           NaN   \n",
       "27887                             27                           NaN   \n",
       "13438                             29                           NaN   \n",
       "17003                             29                           NaN   \n",
       "\n",
       "      IndependentVotingMemberCnt VotingMembersGoverningBodyCnt  \\\n",
       "18819                         21                            22   \n",
       "7965                          21                            21   \n",
       "8989                         NaN                           NaN   \n",
       "22904                        NaN                           NaN   \n",
       "19649                        NaN                           NaN   \n",
       "28581                         18                            18   \n",
       "20683                         18                            18   \n",
       "4797                         NaN                           NaN   \n",
       "15877                        NaN                           NaN   \n",
       "18981                        NaN                           NaN   \n",
       "589                           21                            23   \n",
       "21460                         20                            22   \n",
       "4502                         NaN                           NaN   \n",
       "13619                        NaN                           NaN   \n",
       "17018                        NaN                           NaN   \n",
       "28147                         28                            28   \n",
       "6614                         NaN                           NaN   \n",
       "27887                        NaN                           NaN   \n",
       "13438                        NaN                           NaN   \n",
       "17003                        NaN                           NaN   \n",
       "\n",
       "      VotingMembersIndependentCnt  \n",
       "18819                          21  \n",
       "7965                           21  \n",
       "8989                          NaN  \n",
       "22904                         NaN  \n",
       "19649                         NaN  \n",
       "28581                          18  \n",
       "20683                          18  \n",
       "4797                          NaN  \n",
       "15877                         NaN  \n",
       "18981                         NaN  \n",
       "589                            21  \n",
       "21460                          20  \n",
       "4502                          NaN  \n",
       "13619                         NaN  \n",
       "17018                         NaN  \n",
       "28147                          28  \n",
       "6614                          NaN  \n",
       "27887                         NaN  \n",
       "13438                         NaN  \n",
       "17003                         NaN  "
      ]
     },
     "execution_count": 37,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "cols_2013 = ['GoverningBodyVotingMembersCnt', 'IndependentVotingMemberCnt',\n",
    "               'VotingMembersGoverningBodyCnt', 'VotingMembersIndependentCnt']\n",
    "               \n",
    "cols_2010 = ['NbrIndependentVotingMembers', 'NbrVotingGoverningBodyMembers',\n",
    "             'NbrVotingMembersGoverningBody', 'NumberIndependentVotingMembers']\n",
    "df_all[['FYE', 'EIN'] + cols_2010+cols_2013][:20]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 43,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "11143\n",
      "11143\n",
      "0\n",
      "0 \n",
      "\n",
      "0\n",
      "0\n"
     ]
    }
   ],
   "source": [
    "print len(df_all[(df_all['IndependentVotingMemberCnt'].notnull())])\n",
    "print len(df_all[(df_all['VotingMembersIndependentCnt'].notnull())])\n",
    "print len(df_all[(df_all['IndependentVotingMemberCnt'].isnull()) & (df_all['VotingMembersIndependentCnt'].notnull())])\n",
    "print len(df_all[(df_all['VotingMembersIndependentCnt'].isnull()) & (df_all['IndependentVotingMemberCnt'].notnull())]), '\\n'\n",
    "\n",
    "print len(df_all[(df_all['GoverningBodyVotingMembersCnt'].isnull()) & (df_all['VotingMembersGoverningBodyCnt'].notnull())])\n",
    "print len(df_all[(df_all['VotingMembersGoverningBodyCnt'].isnull()) & (df_all['GoverningBodyVotingMembersCnt'].notnull())])"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "<br>Based on the above we can drop the first two for each year -- we'll go with the Section I, Line 3 and 4 versions, which seem to be the same"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 242,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "['EIN', 'FYE', 'VotingMembersGoverningBodyCnt', 'NbrVotingMembersGoverningBody', 'VotingMembersIndependentCnt', 'NbrIndependentVotingMembers']\n"
     ]
    }
   ],
   "source": [
    "cols = ['EIN', 'FYE']\n",
    "#cols_2013 = ['VotingMembersGoverningBodyCnt', 'VotingMembersIndependentCnt']\n",
    "#cols_2010 = ['NbrIndependentVotingMembers', 'NbrVotingMembersGoverningBody']\n",
    "cols = cols + ['VotingMembersGoverningBodyCnt', 'NbrVotingMembersGoverningBody',\n",
    "        'VotingMembersIndependentCnt', 'NbrIndependentVotingMembers']\n",
    "print cols"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "##### Outsourced Management"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 243,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "['EIN', 'FYE', 'VotingMembersGoverningBodyCnt', 'NbrVotingMembersGoverningBody', 'VotingMembersIndependentCnt', 'NbrIndependentVotingMembers', 'DelegationOfMgmtDutiesInd', 'DelegationOfManagementDuties']\n"
     ]
    }
   ],
   "source": [
    "cols = cols + ['DelegationOfMgmtDutiesInd', 'DelegationOfManagementDuties']\n",
    "print cols"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "##### IRS 990 Review"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 244,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "['EIN', 'FYE', 'VotingMembersGoverningBodyCnt', 'NbrVotingMembersGoverningBody', 'VotingMembersIndependentCnt', 'NbrIndependentVotingMembers', 'DelegationOfMgmtDutiesInd', 'DelegationOfManagementDuties', 'Form990ProvidedToGvrnBodyInd', 'Form990ProvidedToGoverningBody']\n"
     ]
    }
   ],
   "source": [
    "cols = cols + ['Form990ProvidedToGvrnBodyInd', 'Form990ProvidedToGoverningBody']\n",
    "print cols "
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "##### Audit Committee"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 245,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "['EIN', 'FYE', 'VotingMembersGoverningBodyCnt', 'NbrVotingMembersGoverningBody', 'VotingMembersIndependentCnt', 'NbrIndependentVotingMembers', 'DelegationOfMgmtDutiesInd', 'DelegationOfManagementDuties', 'Form990ProvidedToGvrnBodyInd', 'Form990ProvidedToGoverningBody', 'AuditCommitteeInd', 'AuditCommittee']\n"
     ]
    }
   ],
   "source": [
    "cols = cols + ['AuditCommitteeInd', 'AuditCommittee']\n",
    "print cols "
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "##### Donor Restrictions"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 54,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "23690097"
      ]
     },
     "execution_count": 54,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "7351259+4433997+11904841"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "<br>What we see here is that *TemporarilyRstrNetAssetsGrp* + *PermanentlyRstrNetAssetsGrp* + *UnrestrictedNetAssetsGrp* = *TotalNetAssetsFundBalanceGrp* = *NetAssetsOrFundBalancesEOYAmt*\n",
    "\n",
    "- Given that *TotalNetAssetsFundBalanceGrp* is a 'group' variable, I should gather EOY values from the first two variables, take the sum, and then divide by *NetAssetsOrFundBalancesEOYAmt* in order to get a *DONOR RESTRICTIONS* VARIABLE SIMILAR TO YETMAN & YETMAN"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 53,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>EIN</th>\n",
       "      <th>FYE</th>\n",
       "      <th>TemporarilyRstrNetAssetsGrp</th>\n",
       "      <th>TemporarilyRestrictedNetAssets</th>\n",
       "      <th>PermanentlyRstrNetAssetsGrp</th>\n",
       "      <th>PermanentlyRestrictedNetAssets</th>\n",
       "      <th>UnrestrictedNetAssetsGrp</th>\n",
       "      <th>UnrestrictedNetAssets</th>\n",
       "      <th>TotalNetAssetsFundBalanceGrp</th>\n",
       "      <th>TotalNetAssetsFundBalances</th>\n",
       "      <th>NetAssetsOrFundBalancesBOYAmt</th>\n",
       "      <th>NetAssetsOrFundBalancesBOY</th>\n",
       "      <th>NetAssetsOrFundBalancesEOYAmt</th>\n",
       "      <th>NetAssetsOrFundBalancesEOY</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>18819</th>\n",
       "      <td>010202467</td>\n",
       "      <td>FY2014</td>\n",
       "      <td>{u'BOYAmt': u'5880227', u'EOYAmt': u'7351259'}</td>\n",
       "      <td>NaN</td>\n",
       "      <td>{u'BOYAmt': u'4193257', u'EOYAmt': u'4433997'}</td>\n",
       "      <td>NaN</td>\n",
       "      <td>{u'BOYAmt': u'11584416', u'EOYAmt': u'11904841'}</td>\n",
       "      <td>NaN</td>\n",
       "      <td>{u'BOYAmt': u'21657900', u'EOYAmt': u'23690097'}</td>\n",
       "      <td>NaN</td>\n",
       "      <td>21657900</td>\n",
       "      <td>NaN</td>\n",
       "      <td>23690097</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7965</th>\n",
       "      <td>010202467</td>\n",
       "      <td>FY2013</td>\n",
       "      <td>{u'BOYAmt': u'5773597', u'EOYAmt': u'5880227'}</td>\n",
       "      <td>NaN</td>\n",
       "      <td>{u'BOYAmt': u'4153627', u'EOYAmt': u'4193257'}</td>\n",
       "      <td>NaN</td>\n",
       "      <td>{u'BOYAmt': u'12203695', u'EOYAmt': u'11584416'}</td>\n",
       "      <td>NaN</td>\n",
       "      <td>{u'BOYAmt': u'22130919', u'EOYAmt': u'21657900'}</td>\n",
       "      <td>NaN</td>\n",
       "      <td>22130919</td>\n",
       "      <td>NaN</td>\n",
       "      <td>21657900</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8989</th>\n",
       "      <td>010202467</td>\n",
       "      <td>FY2012</td>\n",
       "      <td>NaN</td>\n",
       "      <td>{u'BOY': u'6554891', u'EOY': u'5773597'}</td>\n",
       "      <td>NaN</td>\n",
       "      <td>{u'BOY': u'3096087', u'EOY': u'4153627'}</td>\n",
       "      <td>NaN</td>\n",
       "      <td>{u'BOY': u'10835720', u'EOY': u'12203695'}</td>\n",
       "      <td>NaN</td>\n",
       "      <td>{u'BOY': u'20486698', u'EOY': u'22130919'}</td>\n",
       "      <td>NaN</td>\n",
       "      <td>20486698</td>\n",
       "      <td>NaN</td>\n",
       "      <td>22130919</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>22904</th>\n",
       "      <td>010202467</td>\n",
       "      <td>FY2011</td>\n",
       "      <td>NaN</td>\n",
       "      <td>{u'BOY': u'7110362', u'EOY': u'6554891'}</td>\n",
       "      <td>NaN</td>\n",
       "      <td>{u'BOY': u'2873816', u'EOY': u'3096087'}</td>\n",
       "      <td>NaN</td>\n",
       "      <td>{u'BOY': u'7951722', u'EOY': u'10835720'}</td>\n",
       "      <td>NaN</td>\n",
       "      <td>{u'BOY': u'17935900', u'EOY': u'20486698'}</td>\n",
       "      <td>NaN</td>\n",
       "      <td>17935900</td>\n",
       "      <td>NaN</td>\n",
       "      <td>20486698</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>19649</th>\n",
       "      <td>010202467</td>\n",
       "      <td>FY2010</td>\n",
       "      <td>NaN</td>\n",
       "      <td>{u'BOY': u'7291747', u'EOY': u'7110362'}</td>\n",
       "      <td>NaN</td>\n",
       "      <td>{u'BOY': u'2824416', u'EOY': u'2873816'}</td>\n",
       "      <td>NaN</td>\n",
       "      <td>{u'BOY': u'7878008', u'EOY': u'7951722'}</td>\n",
       "      <td>NaN</td>\n",
       "      <td>{u'BOY': u'17994171', u'EOY': u'17935900'}</td>\n",
       "      <td>NaN</td>\n",
       "      <td>17994171</td>\n",
       "      <td>NaN</td>\n",
       "      <td>17935900</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>28581</th>\n",
       "      <td>010211478</td>\n",
       "      <td>FY2015</td>\n",
       "      <td>{u'BOYAmt': u'889233', u'EOYAmt': u'908556'}</td>\n",
       "      <td>NaN</td>\n",
       "      <td>{u'BOYAmt': u'2717476', u'EOYAmt': u'2717000'}</td>\n",
       "      <td>NaN</td>\n",
       "      <td>{u'BOYAmt': u'1254972', u'EOYAmt': u'1179056'}</td>\n",
       "      <td>NaN</td>\n",
       "      <td>{u'BOYAmt': u'4861681', u'EOYAmt': u'4804612'}</td>\n",
       "      <td>NaN</td>\n",
       "      <td>4861681</td>\n",
       "      <td>NaN</td>\n",
       "      <td>4804612</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>20683</th>\n",
       "      <td>010211478</td>\n",
       "      <td>FY2014</td>\n",
       "      <td>{u'BOYAmt': u'614778', u'EOYAmt': u'889233'}</td>\n",
       "      <td>NaN</td>\n",
       "      <td>{u'BOYAmt': u'2669412', u'EOYAmt': u'2717476'}</td>\n",
       "      <td>NaN</td>\n",
       "      <td>{u'BOYAmt': u'1291239', u'EOYAmt': u'1254972'}</td>\n",
       "      <td>NaN</td>\n",
       "      <td>{u'BOYAmt': u'4575429', u'EOYAmt': u'4861681'}</td>\n",
       "      <td>NaN</td>\n",
       "      <td>4575429</td>\n",
       "      <td>NaN</td>\n",
       "      <td>4861681</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4797</th>\n",
       "      <td>010211478</td>\n",
       "      <td>FY2013</td>\n",
       "      <td>NaN</td>\n",
       "      <td>{u'BOY': u'563731', u'EOY': u'614778'}</td>\n",
       "      <td>NaN</td>\n",
       "      <td>{u'BOY': u'2652247', u'EOY': u'2669412'}</td>\n",
       "      <td>NaN</td>\n",
       "      <td>{u'BOY': u'1362377', u'EOY': u'1291239'}</td>\n",
       "      <td>NaN</td>\n",
       "      <td>{u'BOY': u'4578355', u'EOY': u'4575429'}</td>\n",
       "      <td>NaN</td>\n",
       "      <td>4578355</td>\n",
       "      <td>NaN</td>\n",
       "      <td>4575429</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>15877</th>\n",
       "      <td>010211478</td>\n",
       "      <td>FY2012</td>\n",
       "      <td>NaN</td>\n",
       "      <td>{u'BOY': u'511917', u'EOY': u'563731'}</td>\n",
       "      <td>NaN</td>\n",
       "      <td>{u'BOY': u'2662228', u'EOY': u'2652247'}</td>\n",
       "      <td>NaN</td>\n",
       "      <td>{u'BOY': u'1383839', u'EOY': u'1362377'}</td>\n",
       "      <td>NaN</td>\n",
       "      <td>{u'BOY': u'4557984', u'EOY': u'4578355'}</td>\n",
       "      <td>NaN</td>\n",
       "      <td>4557984</td>\n",
       "      <td>NaN</td>\n",
       "      <td>4578355</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>18981</th>\n",
       "      <td>010211478</td>\n",
       "      <td>FY2011</td>\n",
       "      <td>NaN</td>\n",
       "      <td>{u'BOY': u'586298', u'EOY': u'511917'}</td>\n",
       "      <td>NaN</td>\n",
       "      <td>{u'BOY': u'406957', u'EOY': u'2662228'}</td>\n",
       "      <td>NaN</td>\n",
       "      <td>{u'BOY': u'928373', u'EOY': u'1383839'}</td>\n",
       "      <td>NaN</td>\n",
       "      <td>{u'BOY': u'1921628', u'EOY': u'4557984'}</td>\n",
       "      <td>NaN</td>\n",
       "      <td>1921628</td>\n",
       "      <td>NaN</td>\n",
       "      <td>4557984</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "             EIN     FYE                     TemporarilyRstrNetAssetsGrp  \\\n",
       "18819  010202467  FY2014  {u'BOYAmt': u'5880227', u'EOYAmt': u'7351259'}   \n",
       "7965   010202467  FY2013  {u'BOYAmt': u'5773597', u'EOYAmt': u'5880227'}   \n",
       "8989   010202467  FY2012                                             NaN   \n",
       "22904  010202467  FY2011                                             NaN   \n",
       "19649  010202467  FY2010                                             NaN   \n",
       "28581  010211478  FY2015    {u'BOYAmt': u'889233', u'EOYAmt': u'908556'}   \n",
       "20683  010211478  FY2014    {u'BOYAmt': u'614778', u'EOYAmt': u'889233'}   \n",
       "4797   010211478  FY2013                                             NaN   \n",
       "15877  010211478  FY2012                                             NaN   \n",
       "18981  010211478  FY2011                                             NaN   \n",
       "\n",
       "                 TemporarilyRestrictedNetAssets  \\\n",
       "18819                                       NaN   \n",
       "7965                                        NaN   \n",
       "8989   {u'BOY': u'6554891', u'EOY': u'5773597'}   \n",
       "22904  {u'BOY': u'7110362', u'EOY': u'6554891'}   \n",
       "19649  {u'BOY': u'7291747', u'EOY': u'7110362'}   \n",
       "28581                                       NaN   \n",
       "20683                                       NaN   \n",
       "4797     {u'BOY': u'563731', u'EOY': u'614778'}   \n",
       "15877    {u'BOY': u'511917', u'EOY': u'563731'}   \n",
       "18981    {u'BOY': u'586298', u'EOY': u'511917'}   \n",
       "\n",
       "                          PermanentlyRstrNetAssetsGrp  \\\n",
       "18819  {u'BOYAmt': u'4193257', u'EOYAmt': u'4433997'}   \n",
       "7965   {u'BOYAmt': u'4153627', u'EOYAmt': u'4193257'}   \n",
       "8989                                              NaN   \n",
       "22904                                             NaN   \n",
       "19649                                             NaN   \n",
       "28581  {u'BOYAmt': u'2717476', u'EOYAmt': u'2717000'}   \n",
       "20683  {u'BOYAmt': u'2669412', u'EOYAmt': u'2717476'}   \n",
       "4797                                              NaN   \n",
       "15877                                             NaN   \n",
       "18981                                             NaN   \n",
       "\n",
       "                 PermanentlyRestrictedNetAssets  \\\n",
       "18819                                       NaN   \n",
       "7965                                        NaN   \n",
       "8989   {u'BOY': u'3096087', u'EOY': u'4153627'}   \n",
       "22904  {u'BOY': u'2873816', u'EOY': u'3096087'}   \n",
       "19649  {u'BOY': u'2824416', u'EOY': u'2873816'}   \n",
       "28581                                       NaN   \n",
       "20683                                       NaN   \n",
       "4797   {u'BOY': u'2652247', u'EOY': u'2669412'}   \n",
       "15877  {u'BOY': u'2662228', u'EOY': u'2652247'}   \n",
       "18981   {u'BOY': u'406957', u'EOY': u'2662228'}   \n",
       "\n",
       "                               UnrestrictedNetAssetsGrp  \\\n",
       "18819  {u'BOYAmt': u'11584416', u'EOYAmt': u'11904841'}   \n",
       "7965   {u'BOYAmt': u'12203695', u'EOYAmt': u'11584416'}   \n",
       "8989                                                NaN   \n",
       "22904                                               NaN   \n",
       "19649                                               NaN   \n",
       "28581    {u'BOYAmt': u'1254972', u'EOYAmt': u'1179056'}   \n",
       "20683    {u'BOYAmt': u'1291239', u'EOYAmt': u'1254972'}   \n",
       "4797                                                NaN   \n",
       "15877                                               NaN   \n",
       "18981                                               NaN   \n",
       "\n",
       "                            UnrestrictedNetAssets  \\\n",
       "18819                                         NaN   \n",
       "7965                                          NaN   \n",
       "8989   {u'BOY': u'10835720', u'EOY': u'12203695'}   \n",
       "22904   {u'BOY': u'7951722', u'EOY': u'10835720'}   \n",
       "19649    {u'BOY': u'7878008', u'EOY': u'7951722'}   \n",
       "28581                                         NaN   \n",
       "20683                                         NaN   \n",
       "4797     {u'BOY': u'1362377', u'EOY': u'1291239'}   \n",
       "15877    {u'BOY': u'1383839', u'EOY': u'1362377'}   \n",
       "18981     {u'BOY': u'928373', u'EOY': u'1383839'}   \n",
       "\n",
       "                           TotalNetAssetsFundBalanceGrp  \\\n",
       "18819  {u'BOYAmt': u'21657900', u'EOYAmt': u'23690097'}   \n",
       "7965   {u'BOYAmt': u'22130919', u'EOYAmt': u'21657900'}   \n",
       "8989                                                NaN   \n",
       "22904                                               NaN   \n",
       "19649                                               NaN   \n",
       "28581    {u'BOYAmt': u'4861681', u'EOYAmt': u'4804612'}   \n",
       "20683    {u'BOYAmt': u'4575429', u'EOYAmt': u'4861681'}   \n",
       "4797                                                NaN   \n",
       "15877                                               NaN   \n",
       "18981                                               NaN   \n",
       "\n",
       "                       TotalNetAssetsFundBalances  \\\n",
       "18819                                         NaN   \n",
       "7965                                          NaN   \n",
       "8989   {u'BOY': u'20486698', u'EOY': u'22130919'}   \n",
       "22904  {u'BOY': u'17935900', u'EOY': u'20486698'}   \n",
       "19649  {u'BOY': u'17994171', u'EOY': u'17935900'}   \n",
       "28581                                         NaN   \n",
       "20683                                         NaN   \n",
       "4797     {u'BOY': u'4578355', u'EOY': u'4575429'}   \n",
       "15877    {u'BOY': u'4557984', u'EOY': u'4578355'}   \n",
       "18981    {u'BOY': u'1921628', u'EOY': u'4557984'}   \n",
       "\n",
       "      NetAssetsOrFundBalancesBOYAmt NetAssetsOrFundBalancesBOY  \\\n",
       "18819                      21657900                        NaN   \n",
       "7965                       22130919                        NaN   \n",
       "8989                            NaN                   20486698   \n",
       "22904                           NaN                   17935900   \n",
       "19649                           NaN                   17994171   \n",
       "28581                       4861681                        NaN   \n",
       "20683                       4575429                        NaN   \n",
       "4797                            NaN                    4578355   \n",
       "15877                           NaN                    4557984   \n",
       "18981                           NaN                    1921628   \n",
       "\n",
       "      NetAssetsOrFundBalancesEOYAmt NetAssetsOrFundBalancesEOY  \n",
       "18819                      23690097                        NaN  \n",
       "7965                       21657900                        NaN  \n",
       "8989                            NaN                   22130919  \n",
       "22904                           NaN                   20486698  \n",
       "19649                           NaN                   17935900  \n",
       "28581                       4804612                        NaN  \n",
       "20683                       4861681                        NaN  \n",
       "4797                            NaN                    4575429  \n",
       "15877                           NaN                    4578355  \n",
       "18981                           NaN                    4557984  "
      ]
     },
     "execution_count": 53,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "asset_cols = ['TemporarilyRstrNetAssetsGrp', 'TemporarilyRestrictedNetAssets',\n",
    "              'PermanentlyRstrNetAssetsGrp', 'PermanentlyRestrictedNetAssets',\n",
    "              'UnrestrictedNetAssetsGrp', 'UnrestrictedNetAssets', \n",
    "              'TotalNetAssetsFundBalanceGrp', 'TotalNetAssetsFundBalances', \n",
    "              'NetAssetsOrFundBalancesBOYAmt', 'NetAssetsOrFundBalancesBOY', \n",
    "              'NetAssetsOrFundBalancesEOYAmt', 'NetAssetsOrFundBalancesEOY']\n",
    "df_all[['EIN', 'FYE']+asset_cols][:10]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 62,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "['EIN', 'FYE', 'VotingMembersGoverningBodyCnt', 'NbrVotingMembersGoverningBody', 'VotingMembersIndependentCnt', 'NbrIndependentVotingMembers', 'DelegationOfMgmtDutiesInd', 'DelegationOfManagementDuties', 'Form990ProvidedToGvrnBodyInd', 'Form990ProvidedToGoverningBody', 'AuditCommitteeInd', 'AuditCommittee', 'TemporarilyRstrNetAssetsGrp', 'TemporarilyRestrictedNetAssets', 'PermanentlyRstrNetAssetsGrp', 'PermanentlyRestrictedNetAssets', 'NetAssetsOrFundBalancesEOYAmt', 'NetAssetsOrFundBalancesEOY']\n"
     ]
    }
   ],
   "source": [
    "#cols = cols + ['TemporarilyRstrNetAssetsGrp', 'TemporarilyRestrictedNetAssets',\n",
    "              'PermanentlyRstrNetAssetsGrp', 'PermanentlyRestrictedNetAssets',\n",
    "              #'UnrestrictedNetAssetsGrp', 'UnrestrictedNetAssets', \n",
    "              #'TotalNetAssetsFundBalanceGrp', 'TotalNetAssetsFundBalances', \n",
    "              #'NetAssetsOrFundBalancesBOYAmt', 'NetAssetsOrFundBalancesBOY', \n",
    "              'NetAssetsOrFundBalancesEOYAmt', 'NetAssetsOrFundBalancesEOY']\n",
    "#print cols"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 247,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[u'UponRequest', u'UponRequestInd', u'VotingMembersGoverningBodyCnt', u'VotingMembersIndependentCnt', u'WebSite', u'WebsiteAddressTxt', u'WhistleblowerPolicy', u'WhistleblowerPolicyInd', u'WrittenPolicyOrProcedure', u'WrittenPolicyOrProcedureInd', u'YearFormation', u'_id', 'FeesForServicesAccountingGrp_v2', 'FYE', 'FeesForServicesAccounting_TotalAmt', 'FeesForServicesAccounting_binary', 'perm_rest_assets', 'temp_rest_assets', 'net_assets', 'donor_restrictions']\n"
     ]
    }
   ],
   "source": [
    "print df_all.columns.tolist()[-20:]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 249,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "cols = cols + ['perm_rest_assets', 'temp_rest_assets', 'net_assets', 'donor_restrictions']"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "##### Tax-Exempt Bonds (municipal bonds)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 250,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "['EIN', 'FYE', 'VotingMembersGoverningBodyCnt', 'NbrVotingMembersGoverningBody', 'VotingMembersIndependentCnt', 'NbrIndependentVotingMembers', 'DelegationOfMgmtDutiesInd', 'DelegationOfManagementDuties', 'Form990ProvidedToGvrnBodyInd', 'Form990ProvidedToGoverningBody', 'AuditCommitteeInd', 'AuditCommittee', 'perm_rest_assets', 'temp_rest_assets', 'net_assets', 'donor_restrictions', 'TaxExemptBondsInd', 'TaxExemptBonds']\n"
     ]
    }
   ],
   "source": [
    "cols = cols + ['TaxExemptBondsInd', 'TaxExemptBonds']\n",
    "print cols"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "##### Taxable Revenues"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 251,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "['EIN', 'FYE', 'VotingMembersGoverningBodyCnt', 'NbrVotingMembersGoverningBody', 'VotingMembersIndependentCnt', 'NbrIndependentVotingMembers', 'DelegationOfMgmtDutiesInd', 'DelegationOfManagementDuties', 'Form990ProvidedToGvrnBodyInd', 'Form990ProvidedToGoverningBody', 'AuditCommitteeInd', 'AuditCommittee', 'perm_rest_assets', 'temp_rest_assets', 'net_assets', 'donor_restrictions', 'TaxExemptBondsInd', 'TaxExemptBonds', 'NetUnrelatedBusTxblIncmAmt', 'NetUnrelatedBusinessTxblIncome']\n"
     ]
    }
   ],
   "source": [
    "cols = cols + ['NetUnrelatedBusTxblIncmAmt', 'NetUnrelatedBusinessTxblIncome']\n",
    "print cols"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "##### Audited Financials and Federal Audits"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 69,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{u'#text': u'true', u'@referenceDocumentId': u'BinaryAttach BinaryAttachN1 BinaryAttachN2'}    2\n",
       "{u'#text': u'true', u'@referenceDocumentId': u'BinaryAttach BinaryAttachN1'}                   2\n",
       "{u'#text': u'true', u'@referenceDocumentId': u'990:103'}                                       2\n",
       "{u'#text': u'true', u'@referenceDocumentId': u'RetDoc3'}                                       2\n",
       "1                                                                                              1\n",
       "{u'#text': u'1', u'@referenceDocumentId': u'RetDoc2317200001'}                                 1\n",
       "Name: AuditedFinancialStmtAttInd, dtype: int64"
      ]
     },
     "execution_count": 69,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "#df_all['AuditedFinancialStmtAttInd'].value_counts()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 76,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>EIN</th>\n",
       "      <th>FYE</th>\n",
       "      <th>FSAuditedInd</th>\n",
       "      <th>FSAudited</th>\n",
       "      <th>FederalGrantAuditPerformedInd</th>\n",
       "      <th>FederalGrantAuditPerformed</th>\n",
       "      <th>FederalGrantAuditRequiredInd</th>\n",
       "      <th>FederalGrantAuditRequired</th>\n",
       "      <th>IndependentAuditFinclStmtInd</th>\n",
       "      <th>IndependentAuditFinancialStmt</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>18819</th>\n",
       "      <td>010202467</td>\n",
       "      <td>FY2014</td>\n",
       "      <td>1</td>\n",
       "      <td>NaN</td>\n",
       "      <td>1</td>\n",
       "      <td>NaN</td>\n",
       "      <td>1</td>\n",
       "      <td>NaN</td>\n",
       "      <td>{u'#text': u'1', u'@referenceDocumentId': u'RetDoc1040000001'}</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7965</th>\n",
       "      <td>010202467</td>\n",
       "      <td>FY2013</td>\n",
       "      <td>1</td>\n",
       "      <td>NaN</td>\n",
       "      <td>1</td>\n",
       "      <td>NaN</td>\n",
       "      <td>1</td>\n",
       "      <td>NaN</td>\n",
       "      <td>{u'#text': u'1', u'@referenceDocumentId': u'RetDoc1040000001'}</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8989</th>\n",
       "      <td>010202467</td>\n",
       "      <td>FY2012</td>\n",
       "      <td>NaN</td>\n",
       "      <td>1</td>\n",
       "      <td>NaN</td>\n",
       "      <td>1</td>\n",
       "      <td>NaN</td>\n",
       "      <td>1</td>\n",
       "      <td>NaN</td>\n",
       "      <td>{u'#text': u'1', u'@referenceDocumentId': u'RetDoc1040000001'}</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>22904</th>\n",
       "      <td>010202467</td>\n",
       "      <td>FY2011</td>\n",
       "      <td>NaN</td>\n",
       "      <td>1</td>\n",
       "      <td>NaN</td>\n",
       "      <td>1</td>\n",
       "      <td>NaN</td>\n",
       "      <td>1</td>\n",
       "      <td>NaN</td>\n",
       "      <td>{u'#text': u'1', u'@referenceDocumentId': u'RetDoc1040000001'}</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>19649</th>\n",
       "      <td>010202467</td>\n",
       "      <td>FY2010</td>\n",
       "      <td>NaN</td>\n",
       "      <td>1</td>\n",
       "      <td>NaN</td>\n",
       "      <td>1</td>\n",
       "      <td>NaN</td>\n",
       "      <td>1</td>\n",
       "      <td>NaN</td>\n",
       "      <td>{u'#text': u'1', u'@referenceDocumentId': u'RetDoc1039900001'}</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>28581</th>\n",
       "      <td>010211478</td>\n",
       "      <td>FY2015</td>\n",
       "      <td>true</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>false</td>\n",
       "      <td>NaN</td>\n",
       "      <td>{u'#text': u'true', u'@referenceDocumentId': u'RetDoc3'}</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>20683</th>\n",
       "      <td>010211478</td>\n",
       "      <td>FY2014</td>\n",
       "      <td>true</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>false</td>\n",
       "      <td>NaN</td>\n",
       "      <td>{u'#text': u'true', u'@referenceDocumentId': u'RetDoc4'}</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4797</th>\n",
       "      <td>010211478</td>\n",
       "      <td>FY2013</td>\n",
       "      <td>NaN</td>\n",
       "      <td>true</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>false</td>\n",
       "      <td>NaN</td>\n",
       "      <td>{u'#text': u'true', u'@referenceDocumentId': u'RetDoc3'}</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "             EIN     FYE FSAuditedInd FSAudited FederalGrantAuditPerformedInd  \\\n",
       "18819  010202467  FY2014            1       NaN                             1   \n",
       "7965   010202467  FY2013            1       NaN                             1   \n",
       "8989   010202467  FY2012          NaN         1                           NaN   \n",
       "22904  010202467  FY2011          NaN         1                           NaN   \n",
       "19649  010202467  FY2010          NaN         1                           NaN   \n",
       "28581  010211478  FY2015         true       NaN                           NaN   \n",
       "20683  010211478  FY2014         true       NaN                           NaN   \n",
       "4797   010211478  FY2013          NaN      true                           NaN   \n",
       "\n",
       "      FederalGrantAuditPerformed FederalGrantAuditRequiredInd  \\\n",
       "18819                        NaN                            1   \n",
       "7965                         NaN                            1   \n",
       "8989                           1                          NaN   \n",
       "22904                          1                          NaN   \n",
       "19649                          1                          NaN   \n",
       "28581                        NaN                        false   \n",
       "20683                        NaN                        false   \n",
       "4797                         NaN                          NaN   \n",
       "\n",
       "      FederalGrantAuditRequired  \\\n",
       "18819                       NaN   \n",
       "7965                        NaN   \n",
       "8989                          1   \n",
       "22904                         1   \n",
       "19649                         1   \n",
       "28581                       NaN   \n",
       "20683                       NaN   \n",
       "4797                      false   \n",
       "\n",
       "                                         IndependentAuditFinclStmtInd  \\\n",
       "18819  {u'#text': u'1', u'@referenceDocumentId': u'RetDoc1040000001'}   \n",
       "7965   {u'#text': u'1', u'@referenceDocumentId': u'RetDoc1040000001'}   \n",
       "8989                                                              NaN   \n",
       "22904                                                             NaN   \n",
       "19649                                                             NaN   \n",
       "28581        {u'#text': u'true', u'@referenceDocumentId': u'RetDoc3'}   \n",
       "20683        {u'#text': u'true', u'@referenceDocumentId': u'RetDoc4'}   \n",
       "4797                                                              NaN   \n",
       "\n",
       "                                        IndependentAuditFinancialStmt  \n",
       "18819                                                             NaN  \n",
       "7965                                                              NaN  \n",
       "8989   {u'#text': u'1', u'@referenceDocumentId': u'RetDoc1040000001'}  \n",
       "22904  {u'#text': u'1', u'@referenceDocumentId': u'RetDoc1040000001'}  \n",
       "19649  {u'#text': u'1', u'@referenceDocumentId': u'RetDoc1039900001'}  \n",
       "28581                                                             NaN  \n",
       "20683                                                             NaN  \n",
       "4797         {u'#text': u'true', u'@referenceDocumentId': u'RetDoc3'}  "
      ]
     },
     "execution_count": 76,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "audit_cols = [#'AuditedFinancialStmtAttInd', \n",
    "              #'ConsolidatedAuditFinclStmtInd', 'ConsolidatedAuditFinancialStmt', \n",
    "              'FSAuditedInd', 'FSAudited', #'FSAuditedBasisGrp', 'FSAuditedBasis',\n",
    "              'FederalGrantAuditPerformedInd', 'FederalGrantAuditPerformed',  \n",
    "              'FederalGrantAuditRequiredInd', 'FederalGrantAuditRequired', \n",
    "              'IndependentAuditFinclStmtInd', 'IndependentAuditFinancialStmt', ]\n",
    "df_all[['EIN', 'FYE']+ audit_cols][:8]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 252,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "['EIN', 'FYE', 'VotingMembersGoverningBodyCnt', 'NbrVotingMembersGoverningBody', 'VotingMembersIndependentCnt', 'NbrIndependentVotingMembers', 'DelegationOfMgmtDutiesInd', 'DelegationOfManagementDuties', 'Form990ProvidedToGvrnBodyInd', 'Form990ProvidedToGoverningBody', 'AuditCommitteeInd', 'AuditCommittee', 'perm_rest_assets', 'temp_rest_assets', 'net_assets', 'donor_restrictions', 'TaxExemptBondsInd', 'TaxExemptBonds', 'NetUnrelatedBusTxblIncmAmt', 'NetUnrelatedBusinessTxblIncome', 'FSAuditedInd', 'FSAudited', 'FederalGrantAuditPerformedInd', 'FederalGrantAuditPerformed', 'FederalGrantAuditRequiredInd', 'FederalGrantAuditRequired', 'IndependentAuditFinclStmtInd', 'IndependentAuditFinancialStmt']\n"
     ]
    }
   ],
   "source": [
    "cols = cols + ['FSAuditedInd', 'FSAudited', \n",
    "              'FederalGrantAuditPerformedInd', 'FederalGrantAuditPerformed',  \n",
    "              'FederalGrantAuditRequiredInd', 'FederalGrantAuditRequired', \n",
    "              'IndependentAuditFinclStmtInd', 'IndependentAuditFinancialStmt', ]\n",
    "print cols"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Create *Donor Restrictions* variable as in Yetman & Yetman (2012)\n",
    "I NEED TO DO IT *BEFORE* COLLAPSING -- OTHERWISE IT WILL CAUSE PROBLEMS FOR AN EIN THAT HAS NO PERMANENTLY RESTRICTED ASSETS IN ONE YEAR, SAY, BUT IT DOES THE NEXT -- THE GROUPBY FUNCTION WILL BRING THOSE INTO THE SAME ROW."
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "##### Donor Restrictions -- Part (a) - Permanently Restricted Assets"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 199,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "('Index: ', 26647, 'Perm. Restr Assets:', 0) \n",
      "\n",
      "# of minutes:  1.63736418486 \n",
      "\n",
      "\n",
      "29440\n"
     ]
    }
   ],
   "source": [
    "from IPython.display import display, clear_output   ##### FOR USE WITH STDOUT (DYNAMIC, SINGLE-LINE PRINTING)\n",
    "\n",
    "import timeit\n",
    "start_time = timeit.default_timer()\n",
    "\n",
    "for index, row in df_all[:].iterrows():\n",
    "    perm_rest = 0\n",
    "    if pd.notnull(row['PermanentlyRstrNetAssetsGrp']):\n",
    "        #print 'not null!'\n",
    "        if 'EOYAmt' in row['PermanentlyRstrNetAssetsGrp']:\n",
    "            perm_rest = row['PermanentlyRstrNetAssetsGrp']['EOYAmt']\n",
    "        elif 'BOYAmt' in row['PermanentlyRstrNetAssetsGrp']:\n",
    "            perm_rest = row['PermanentlyRstrNetAssetsGrp']['BOYAmt']            \n",
    "    elif pd.notnull(row['PermanentlyRestrictedNetAssets']):\n",
    "        print 'going to second variable'\n",
    "        if 'EOY' in row['PermanentlyRestrictedNetAssets']:\n",
    "            perm_rest = row['PermanentlyRestrictedNetAssets']['EOY']\n",
    "        elif 'BOY' in row['PermanentlyRestrictedNetAssets']:\n",
    "            perm_rest = row['PermanentlyRestrictedNetAssets']['BOY']\n",
    "            \n",
    "    df_all.ix[index, 'perm_rest_assets'] = perm_rest\n",
    "        \n",
    "    clear_output()\n",
    "    print ('Index: ', index, 'Perm. Restr Assets:', perm_rest), '\\n'\n",
    "    sys.stdout.flush()        \n",
    "        \n",
    "        \n",
    "elapsed = timeit.default_timer() - start_time\n",
    "print '# of minutes: ', elapsed/60, '\\n', '\\n'         \n",
    "print df_all['perm_rest_assets'].value_counts().sum()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 200,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "29440\n",
      "0\n",
      "count     29440\n",
      "unique    10252\n",
      "top           0\n",
      "freq      13659\n",
      "Name: perm_rest_assets, dtype: int64 \n",
      "\n",
      "count    2.944000e+04\n",
      "mean     5.853230e+06\n",
      "std      3.626274e+07\n",
      "min      0.000000e+00\n",
      "25%      0.000000e+00\n",
      "50%      2.500000e+03\n",
      "75%      1.215208e+06\n",
      "max      1.097021e+09\n",
      "Name: perm_rest_assets, dtype: float64\n"
     ]
    }
   ],
   "source": [
    "print len(df_all[df_all['perm_rest_assets'].notnull()])\n",
    "print len(df_all[df_all['perm_rest_assets'].isnull()])\n",
    "print df_all['perm_rest_assets'].describe(), '\\n'\n",
    "df_all['perm_rest_assets'] = df_all['perm_rest_assets'].astype('int')\n",
    "print df_all['perm_rest_assets'].describe()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "##### Donor Restrictions -- Part (b) - Temporarily Restricted Assets"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 201,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "('Index: ', 26647, 'Temp. Restr Assets:', u'96956') \n",
      "\n",
      "# of minutes:  1.76544401646 \n",
      "\n",
      "\n",
      "29440\n"
     ]
    }
   ],
   "source": [
    "from IPython.display import display, clear_output   ##### FOR USE WITH STDOUT (DYNAMIC, SINGLE-LINE PRINTING)\n",
    "\n",
    "import timeit\n",
    "start_time = timeit.default_timer()\n",
    "\n",
    "for index, row in df_all[:].iterrows():\n",
    "    temp_rest = 0\n",
    "    if pd.notnull(row['TemporarilyRstrNetAssetsGrp']):\n",
    "        #print 'not null!'\n",
    "        if 'EOYAmt' in row['TemporarilyRstrNetAssetsGrp']:\n",
    "            temp_rest = row['TemporarilyRstrNetAssetsGrp']['EOYAmt']\n",
    "        elif 'BOYAmt' in row['TemporarilyRstrNetAssetsGrp']:\n",
    "            temp_rest = row['TemporarilyRstrNetAssetsGrp']['BOYAmt']            \n",
    "    elif pd.notnull(row['TemporarilyRestrictedNetAssets']):\n",
    "        print 'going to second'\n",
    "        if 'EOY' in row['TemporarilyRestrictedNetAssets']:\n",
    "            temp_rest = row['TemporarilyRestrictedNetAssets']['EOY']\n",
    "        elif 'BOY' in row['TemporarilyRestrictedNetAssets']:\n",
    "            temp_rest = row['TemporarilyRestrictedNetAssets']['BOY']\n",
    "            \n",
    "    df_all.ix[index, 'temp_rest_assets'] = temp_rest\n",
    "        \n",
    "    clear_output()\n",
    "    print ('Index: ', index, 'Temp. Restr Assets:', temp_rest), '\\n'\n",
    "    sys.stdout.flush()        \n",
    "        \n",
    "        \n",
    "elapsed = timeit.default_timer() - start_time\n",
    "print '# of minutes: ', elapsed/60, '\\n', '\\n'         \n",
    "print df_all['temp_rest_assets'].value_counts().sum()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 202,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "29440\n",
      "0\n",
      "count     29440\n",
      "unique    24978\n",
      "top           0\n",
      "freq       2849\n",
      "Name: temp_rest_assets, dtype: int64 \n",
      "\n",
      "count    2.944000e+04\n",
      "mean     7.365323e+06\n",
      "std      3.905838e+07\n",
      "min     -2.158587e+06\n",
      "25%      1.274518e+05\n",
      "50%      6.781395e+05\n",
      "75%      2.880510e+06\n",
      "max      1.473751e+09\n",
      "Name: temp_rest_assets, dtype: float64\n"
     ]
    }
   ],
   "source": [
    "print len(df_all[df_all['temp_rest_assets'].notnull()])\n",
    "print len(df_all[df_all['temp_rest_assets'].isnull()])\n",
    "print df_all['temp_rest_assets'].describe(), '\\n'\n",
    "df_all['temp_rest_assets'] = df_all['temp_rest_assets'].astype('int')\n",
    "print df_all['temp_rest_assets'].describe()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "##### Donor Restrictions -- Part (c) - Total Assets"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 203,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "count     29440\n",
      "unique    28932\n",
      "top           0\n",
      "freq         41\n",
      "Name: net_assets, dtype: object \n",
      "\n",
      "count    2.944000e+04\n",
      "mean     3.175411e+07\n",
      "std      1.712160e+08\n",
      "min     -1.399548e+08\n",
      "25%      1.734802e+06\n",
      "50%      5.309920e+06\n",
      "75%      1.683359e+07\n",
      "max      8.116399e+09\n",
      "Name: net_assets, dtype: float64\n"
     ]
    }
   ],
   "source": [
    "df_all['net_assets'] = df_all['NetAssetsOrFundBalancesEOYAmt']\n",
    "df_all['net_assets'] = np.where(df_all['net_assets'].isnull(),\n",
    "                                          df_all['NetAssetsOrFundBalancesEOY'], df_all['net_assets'])\n",
    "print df_all['net_assets'].describe(), '\\n'\n",
    "df_all['net_assets'] = df_all['net_assets'].astype('int')\n",
    "print df_all['net_assets'].describe()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "##### Donor Restrictions -- Part (d) - Calculations"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 204,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "count    2.940000e+04\n",
       "mean              inf\n",
       "std               NaN\n",
       "min     -1.564426e+02\n",
       "25%      6.059707e-02\n",
       "50%      2.498250e-01\n",
       "75%      5.590346e-01\n",
       "max               inf\n",
       "Name: donor_restrictions, dtype: float64"
      ]
     },
     "execution_count": 204,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_all['donor_restrictions'] = (df_all['temp_rest_assets']+df_all['perm_rest_assets'])/df_all['net_assets']\n",
    "df_all['donor_restrictions'].describe()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 205,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "FeesForServicesAccounting_binary      int64\n",
       "perm_rest_assets                      int64\n",
       "temp_rest_assets                      int64\n",
       "net_assets                            int64\n",
       "donor_restrictions                  float64\n",
       "dtype: object"
      ]
     },
     "execution_count": 205,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_all.dtypes[-5:]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "<br>THERE ARE 40 MISSING CASES -- ASSIGN THEM A VALUE OF ZERO"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 207,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "40"
      ]
     },
     "execution_count": 207,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "print len(df_all[df_all['donor_restrictions'].isnull()])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 208,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "count    2.944000e+04\n",
       "mean              inf\n",
       "std               NaN\n",
       "min     -1.564426e+02\n",
       "25%      5.999956e-02\n",
       "50%      2.490086e-01\n",
       "75%      5.585148e-01\n",
       "max               inf\n",
       "Name: donor_restrictions, dtype: float64"
      ]
     },
     "execution_count": 208,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_all['donor_restrictions'] = np.where(df_all['donor_restrictions'].isnull(), 0, df_all['donor_restrictions'])\n",
    "df_all['donor_restrictions'].describe()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 209,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0\n"
     ]
    }
   ],
   "source": [
    "print len(df_all[df_all['donor_restrictions'].isnull()])"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "<br>CHECK A FEW >1 ROWS"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 221,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>FYE</th>\n",
       "      <th>donor_restrictions</th>\n",
       "      <th>temp_rest_assets</th>\n",
       "      <th>TemporarilyRstrNetAssetsGrp</th>\n",
       "      <th>TemporarilyRestrictedNetAssets</th>\n",
       "      <th>perm_rest_assets</th>\n",
       "      <th>PermanentlyRstrNetAssetsGrp</th>\n",
       "      <th>PermanentlyRestrictedNetAssets</th>\n",
       "      <th>net_assets</th>\n",
       "      <th>NetAssetsOrFundBalancesEOYAmt</th>\n",
       "      <th>NetAssetsOrFundBalancesEOY</th>\n",
       "      <th>TotalNetAssetsFundBalanceGrp</th>\n",
       "      <th>TotalNetAssetsFundBalances</th>\n",
       "      <th>NetAssetsOrFundBalancesEOYAmt</th>\n",
       "      <th>NetAssetsOrFundBalancesEOY</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>13277</th>\n",
       "      <td>FY2011</td>\n",
       "      <td>0.011886</td>\n",
       "      <td>45000</td>\n",
       "      <td>NaN</td>\n",
       "      <td>{u'BOY': u'73257', u'EOY': u'45000'}</td>\n",
       "      <td>0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>3785879</td>\n",
       "      <td>NaN</td>\n",
       "      <td>3785879</td>\n",
       "      <td>NaN</td>\n",
       "      <td>{u'BOY': u'3228716', u'EOY': u'3785879'}</td>\n",
       "      <td>NaN</td>\n",
       "      <td>3785879</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>616</th>\n",
       "      <td>FY2014</td>\n",
       "      <td>0.143153</td>\n",
       "      <td>52310</td>\n",
       "      <td>{u'BOYAmt': u'91351', u'EOYAmt': u'52310'}</td>\n",
       "      <td>NaN</td>\n",
       "      <td>284363</td>\n",
       "      <td>{u'BOYAmt': u'207970', u'EOYAmt': u'284363'}</td>\n",
       "      <td>NaN</td>\n",
       "      <td>2351842</td>\n",
       "      <td>2351842</td>\n",
       "      <td>NaN</td>\n",
       "      <td>{u'BOYAmt': u'2321970', u'EOYAmt': u'2351842'}</td>\n",
       "      <td>NaN</td>\n",
       "      <td>2351842</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8573</th>\n",
       "      <td>FY2013</td>\n",
       "      <td>0.128908</td>\n",
       "      <td>91351</td>\n",
       "      <td>{u'BOYAmt': u'467396', u'EOYAmt': u'91351'}</td>\n",
       "      <td>NaN</td>\n",
       "      <td>207970</td>\n",
       "      <td>{u'BOYAmt': u'100000', u'EOYAmt': u'207970'}</td>\n",
       "      <td>NaN</td>\n",
       "      <td>2321970</td>\n",
       "      <td>2321970</td>\n",
       "      <td>NaN</td>\n",
       "      <td>{u'BOYAmt': u'2441674', u'EOYAmt': u'2321970'}</td>\n",
       "      <td>NaN</td>\n",
       "      <td>2321970</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>23577</th>\n",
       "      <td>FY2012</td>\n",
       "      <td>0.232380</td>\n",
       "      <td>467396</td>\n",
       "      <td>NaN</td>\n",
       "      <td>{u'BOY': u'171880', u'EOY': u'467396'}</td>\n",
       "      <td>100000</td>\n",
       "      <td>NaN</td>\n",
       "      <td>{u'BOY': u'0', u'EOY': u'100000'}</td>\n",
       "      <td>2441674</td>\n",
       "      <td>NaN</td>\n",
       "      <td>2441674</td>\n",
       "      <td>NaN</td>\n",
       "      <td>{u'BOY': u'1953372', u'EOY': u'2441674'}</td>\n",
       "      <td>NaN</td>\n",
       "      <td>2441674</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>16294</th>\n",
       "      <td>FY2011</td>\n",
       "      <td>0.087991</td>\n",
       "      <td>171880</td>\n",
       "      <td>NaN</td>\n",
       "      <td>{u'BOY': u'96956', u'EOY': u'171880'}</td>\n",
       "      <td>0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>1953372</td>\n",
       "      <td>NaN</td>\n",
       "      <td>1953372</td>\n",
       "      <td>NaN</td>\n",
       "      <td>{u'BOY': u'1720152', u'EOY': u'1953372'}</td>\n",
       "      <td>NaN</td>\n",
       "      <td>1953372</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>26647</th>\n",
       "      <td>FY2010</td>\n",
       "      <td>0.056365</td>\n",
       "      <td>96956</td>\n",
       "      <td>NaN</td>\n",
       "      <td>{u'BOY': u'79000', u'EOY': u'96956'}</td>\n",
       "      <td>0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>1720152</td>\n",
       "      <td>NaN</td>\n",
       "      <td>1720152</td>\n",
       "      <td>NaN</td>\n",
       "      <td>{u'BOY': u'1413351', u'EOY': u'1720152'}</td>\n",
       "      <td>NaN</td>\n",
       "      <td>1720152</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "          FYE  donor_restrictions  temp_rest_assets  \\\n",
       "13277  FY2011            0.011886             45000   \n",
       "616    FY2014            0.143153             52310   \n",
       "8573   FY2013            0.128908             91351   \n",
       "23577  FY2012            0.232380            467396   \n",
       "16294  FY2011            0.087991            171880   \n",
       "26647  FY2010            0.056365             96956   \n",
       "\n",
       "                       TemporarilyRstrNetAssetsGrp  \\\n",
       "13277                                          NaN   \n",
       "616     {u'BOYAmt': u'91351', u'EOYAmt': u'52310'}   \n",
       "8573   {u'BOYAmt': u'467396', u'EOYAmt': u'91351'}   \n",
       "23577                                          NaN   \n",
       "16294                                          NaN   \n",
       "26647                                          NaN   \n",
       "\n",
       "               TemporarilyRestrictedNetAssets  perm_rest_assets  \\\n",
       "13277    {u'BOY': u'73257', u'EOY': u'45000'}                 0   \n",
       "616                                       NaN            284363   \n",
       "8573                                      NaN            207970   \n",
       "23577  {u'BOY': u'171880', u'EOY': u'467396'}            100000   \n",
       "16294   {u'BOY': u'96956', u'EOY': u'171880'}                 0   \n",
       "26647    {u'BOY': u'79000', u'EOY': u'96956'}                 0   \n",
       "\n",
       "                        PermanentlyRstrNetAssetsGrp  \\\n",
       "13277                                           NaN   \n",
       "616    {u'BOYAmt': u'207970', u'EOYAmt': u'284363'}   \n",
       "8573   {u'BOYAmt': u'100000', u'EOYAmt': u'207970'}   \n",
       "23577                                           NaN   \n",
       "16294                                           NaN   \n",
       "26647                                           NaN   \n",
       "\n",
       "          PermanentlyRestrictedNetAssets  net_assets  \\\n",
       "13277                                NaN     3785879   \n",
       "616                                  NaN     2351842   \n",
       "8573                                 NaN     2321970   \n",
       "23577  {u'BOY': u'0', u'EOY': u'100000'}     2441674   \n",
       "16294                                NaN     1953372   \n",
       "26647                                NaN     1720152   \n",
       "\n",
       "      NetAssetsOrFundBalancesEOYAmt NetAssetsOrFundBalancesEOY  \\\n",
       "13277                           NaN                    3785879   \n",
       "616                         2351842                        NaN   \n",
       "8573                        2321970                        NaN   \n",
       "23577                           NaN                    2441674   \n",
       "16294                           NaN                    1953372   \n",
       "26647                           NaN                    1720152   \n",
       "\n",
       "                         TotalNetAssetsFundBalanceGrp  \\\n",
       "13277                                             NaN   \n",
       "616    {u'BOYAmt': u'2321970', u'EOYAmt': u'2351842'}   \n",
       "8573   {u'BOYAmt': u'2441674', u'EOYAmt': u'2321970'}   \n",
       "23577                                             NaN   \n",
       "16294                                             NaN   \n",
       "26647                                             NaN   \n",
       "\n",
       "                     TotalNetAssetsFundBalances NetAssetsOrFundBalancesEOYAmt  \\\n",
       "13277  {u'BOY': u'3228716', u'EOY': u'3785879'}                           NaN   \n",
       "616                                         NaN                       2351842   \n",
       "8573                                        NaN                       2321970   \n",
       "23577  {u'BOY': u'1953372', u'EOY': u'2441674'}                           NaN   \n",
       "16294  {u'BOY': u'1720152', u'EOY': u'1953372'}                           NaN   \n",
       "26647  {u'BOY': u'1413351', u'EOY': u'1720152'}                           NaN   \n",
       "\n",
       "      NetAssetsOrFundBalancesEOY  \n",
       "13277                    3785879  \n",
       "616                          NaN  \n",
       "8573                         NaN  \n",
       "23577                    2441674  \n",
       "16294                    1953372  \n",
       "26647                    1720152  "
      ]
     },
     "execution_count": 221,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_all[['FYE', 'donor_restrictions', 'temp_rest_assets', 'TemporarilyRstrNetAssetsGrp', 'TemporarilyRestrictedNetAssets',\n",
    "        'perm_rest_assets', 'PermanentlyRstrNetAssetsGrp', 'PermanentlyRestrictedNetAssets',\n",
    "        'net_assets', 'NetAssetsOrFundBalancesEOYAmt', 'NetAssetsOrFundBalancesEOY',\n",
    "                      #'UnrestrictedNetAssetsGrp', 'UnrestrictedNetAssets', \n",
    "              'TotalNetAssetsFundBalanceGrp', 'TotalNetAssetsFundBalances', \n",
    "              #'NetAssetsOrFundBalancesBOYAmt', 'NetAssetsOrFundBalancesBOY', \n",
    "              'NetAssetsOrFundBalancesEOYAmt', 'NetAssetsOrFundBalancesEOY']][-6:]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 211,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>FYE</th>\n",
       "      <th>donor_restrictions</th>\n",
       "      <th>temp_rest_assets</th>\n",
       "      <th>perm_rest_assets</th>\n",
       "      <th>net_assets</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>13277</th>\n",
       "      <td>FY2011</td>\n",
       "      <td>0.011886</td>\n",
       "      <td>45000</td>\n",
       "      <td>0</td>\n",
       "      <td>3785879</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>616</th>\n",
       "      <td>FY2014</td>\n",
       "      <td>0.143153</td>\n",
       "      <td>52310</td>\n",
       "      <td>284363</td>\n",
       "      <td>2351842</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8573</th>\n",
       "      <td>FY2013</td>\n",
       "      <td>0.128908</td>\n",
       "      <td>91351</td>\n",
       "      <td>207970</td>\n",
       "      <td>2321970</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>23577</th>\n",
       "      <td>FY2012</td>\n",
       "      <td>0.232380</td>\n",
       "      <td>467396</td>\n",
       "      <td>100000</td>\n",
       "      <td>2441674</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>16294</th>\n",
       "      <td>FY2011</td>\n",
       "      <td>0.087991</td>\n",
       "      <td>171880</td>\n",
       "      <td>0</td>\n",
       "      <td>1953372</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>26647</th>\n",
       "      <td>FY2010</td>\n",
       "      <td>0.056365</td>\n",
       "      <td>96956</td>\n",
       "      <td>0</td>\n",
       "      <td>1720152</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "          FYE  donor_restrictions  temp_rest_assets  perm_rest_assets  \\\n",
       "13277  FY2011            0.011886             45000                 0   \n",
       "616    FY2014            0.143153             52310            284363   \n",
       "8573   FY2013            0.128908             91351            207970   \n",
       "23577  FY2012            0.232380            467396            100000   \n",
       "16294  FY2011            0.087991            171880                 0   \n",
       "26647  FY2010            0.056365             96956                 0   \n",
       "\n",
       "       net_assets  \n",
       "13277     3785879  \n",
       "616       2351842  \n",
       "8573      2321970  \n",
       "23577     2441674  \n",
       "16294     1953372  \n",
       "26647     1720152  "
      ]
     },
     "execution_count": 211,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_all[['FYE', 'donor_restrictions', 'temp_rest_assets', 'perm_rest_assets', 'net_assets']][-6:]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 224,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "1230\n",
      "1117\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>FYE</th>\n",
       "      <th>donor_restrictions</th>\n",
       "      <th>temp_rest_assets</th>\n",
       "      <th>TemporarilyRstrNetAssetsGrp</th>\n",
       "      <th>TemporarilyRestrictedNetAssets</th>\n",
       "      <th>perm_rest_assets</th>\n",
       "      <th>PermanentlyRstrNetAssetsGrp</th>\n",
       "      <th>PermanentlyRestrictedNetAssets</th>\n",
       "      <th>net_assets</th>\n",
       "      <th>NetAssetsOrFundBalancesEOYAmt</th>\n",
       "      <th>NetAssetsOrFundBalancesEOY</th>\n",
       "      <th>TotalNetAssetsFundBalanceGrp</th>\n",
       "      <th>TotalNetAssetsFundBalances</th>\n",
       "      <th>NetAssetsOrFundBalancesEOYAmt</th>\n",
       "      <th>NetAssetsOrFundBalancesEOY</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>4768</th>\n",
       "      <td>FY2012</td>\n",
       "      <td>1.029663</td>\n",
       "      <td>261414</td>\n",
       "      <td>NaN</td>\n",
       "      <td>{u'BOY': u'589365', u'EOY': u'261414'}</td>\n",
       "      <td>0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>253883</td>\n",
       "      <td>NaN</td>\n",
       "      <td>253883</td>\n",
       "      <td>NaN</td>\n",
       "      <td>{u'BOY': u'610579', u'EOY': u'253883'}</td>\n",
       "      <td>NaN</td>\n",
       "      <td>253883</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3425</th>\n",
       "      <td>FY2014</td>\n",
       "      <td>1.162178</td>\n",
       "      <td>124675</td>\n",
       "      <td>{u'BOYAmt': u'271888', u'EOYAmt': u'124675'}</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>107277</td>\n",
       "      <td>107277</td>\n",
       "      <td>NaN</td>\n",
       "      <td>{u'BOYAmt': u'440094', u'EOYAmt': u'107277'}</td>\n",
       "      <td>NaN</td>\n",
       "      <td>107277</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>25435</th>\n",
       "      <td>FY2014</td>\n",
       "      <td>1.426897</td>\n",
       "      <td>312031</td>\n",
       "      <td>{u'BOYAmt': u'94893', u'EOYAmt': u'312031'}</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>218678</td>\n",
       "      <td>218678</td>\n",
       "      <td>NaN</td>\n",
       "      <td>{u'BOYAmt': u'608360', u'EOYAmt': u'218678'}</td>\n",
       "      <td>NaN</td>\n",
       "      <td>218678</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1853</th>\n",
       "      <td>FY2014</td>\n",
       "      <td>1.013958</td>\n",
       "      <td>685532</td>\n",
       "      <td>{u'BOYAmt': u'499406', u'EOYAmt': u'685532'}</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>676095</td>\n",
       "      <td>676095</td>\n",
       "      <td>NaN</td>\n",
       "      <td>{u'BOYAmt': u'426629', u'EOYAmt': u'676095'}</td>\n",
       "      <td>NaN</td>\n",
       "      <td>676095</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1872</th>\n",
       "      <td>FY2014</td>\n",
       "      <td>1.001405</td>\n",
       "      <td>2144621</td>\n",
       "      <td>{u'BOYAmt': u'1579409', u'EOYAmt': u'2144621'}</td>\n",
       "      <td>NaN</td>\n",
       "      <td>6644457</td>\n",
       "      <td>{u'BOYAmt': u'6501148', u'EOYAmt': u'6644457'}</td>\n",
       "      <td>NaN</td>\n",
       "      <td>8776750</td>\n",
       "      <td>8776750</td>\n",
       "      <td>NaN</td>\n",
       "      <td>{u'BOYAmt': u'8480604', u'EOYAmt': u'8776750'}</td>\n",
       "      <td>NaN</td>\n",
       "      <td>8776750</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1938</th>\n",
       "      <td>FY2014</td>\n",
       "      <td>1.001405</td>\n",
       "      <td>2144621</td>\n",
       "      <td>{u'BOYAmt': u'1579409', u'EOYAmt': u'2144621'}</td>\n",
       "      <td>NaN</td>\n",
       "      <td>6644457</td>\n",
       "      <td>{u'BOYAmt': u'6501148', u'EOYAmt': u'6644457'}</td>\n",
       "      <td>NaN</td>\n",
       "      <td>8776750</td>\n",
       "      <td>8776750</td>\n",
       "      <td>NaN</td>\n",
       "      <td>{u'BOYAmt': u'8480604', u'EOYAmt': u'8776750'}</td>\n",
       "      <td>NaN</td>\n",
       "      <td>8776750</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "          FYE  donor_restrictions  temp_rest_assets  \\\n",
       "4768   FY2012            1.029663            261414   \n",
       "3425   FY2014            1.162178            124675   \n",
       "25435  FY2014            1.426897            312031   \n",
       "1853   FY2014            1.013958            685532   \n",
       "1872   FY2014            1.001405           2144621   \n",
       "1938   FY2014            1.001405           2144621   \n",
       "\n",
       "                          TemporarilyRstrNetAssetsGrp  \\\n",
       "4768                                              NaN   \n",
       "3425     {u'BOYAmt': u'271888', u'EOYAmt': u'124675'}   \n",
       "25435     {u'BOYAmt': u'94893', u'EOYAmt': u'312031'}   \n",
       "1853     {u'BOYAmt': u'499406', u'EOYAmt': u'685532'}   \n",
       "1872   {u'BOYAmt': u'1579409', u'EOYAmt': u'2144621'}   \n",
       "1938   {u'BOYAmt': u'1579409', u'EOYAmt': u'2144621'}   \n",
       "\n",
       "               TemporarilyRestrictedNetAssets  perm_rest_assets  \\\n",
       "4768   {u'BOY': u'589365', u'EOY': u'261414'}                 0   \n",
       "3425                                      NaN                 0   \n",
       "25435                                     NaN                 0   \n",
       "1853                                      NaN                 0   \n",
       "1872                                      NaN           6644457   \n",
       "1938                                      NaN           6644457   \n",
       "\n",
       "                          PermanentlyRstrNetAssetsGrp  \\\n",
       "4768                                              NaN   \n",
       "3425                                              NaN   \n",
       "25435                                             NaN   \n",
       "1853                                              NaN   \n",
       "1872   {u'BOYAmt': u'6501148', u'EOYAmt': u'6644457'}   \n",
       "1938   {u'BOYAmt': u'6501148', u'EOYAmt': u'6644457'}   \n",
       "\n",
       "      PermanentlyRestrictedNetAssets  net_assets  \\\n",
       "4768                             NaN      253883   \n",
       "3425                             NaN      107277   \n",
       "25435                            NaN      218678   \n",
       "1853                             NaN      676095   \n",
       "1872                             NaN     8776750   \n",
       "1938                             NaN     8776750   \n",
       "\n",
       "      NetAssetsOrFundBalancesEOYAmt NetAssetsOrFundBalancesEOY  \\\n",
       "4768                            NaN                     253883   \n",
       "3425                         107277                        NaN   \n",
       "25435                        218678                        NaN   \n",
       "1853                         676095                        NaN   \n",
       "1872                        8776750                        NaN   \n",
       "1938                        8776750                        NaN   \n",
       "\n",
       "                         TotalNetAssetsFundBalanceGrp  \\\n",
       "4768                                              NaN   \n",
       "3425     {u'BOYAmt': u'440094', u'EOYAmt': u'107277'}   \n",
       "25435    {u'BOYAmt': u'608360', u'EOYAmt': u'218678'}   \n",
       "1853     {u'BOYAmt': u'426629', u'EOYAmt': u'676095'}   \n",
       "1872   {u'BOYAmt': u'8480604', u'EOYAmt': u'8776750'}   \n",
       "1938   {u'BOYAmt': u'8480604', u'EOYAmt': u'8776750'}   \n",
       "\n",
       "                   TotalNetAssetsFundBalances NetAssetsOrFundBalancesEOYAmt  \\\n",
       "4768   {u'BOY': u'610579', u'EOY': u'253883'}                           NaN   \n",
       "3425                                      NaN                        107277   \n",
       "25435                                     NaN                        218678   \n",
       "1853                                      NaN                        676095   \n",
       "1872                                      NaN                       8776750   \n",
       "1938                                      NaN                       8776750   \n",
       "\n",
       "      NetAssetsOrFundBalancesEOY  \n",
       "4768                      253883  \n",
       "3425                         NaN  \n",
       "25435                        NaN  \n",
       "1853                         NaN  \n",
       "1872                         NaN  \n",
       "1938                         NaN  "
      ]
     },
     "execution_count": 224,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "print len(df_all[df_all['donor_restrictions']>.99])\n",
    "print len(df_all[df_all['donor_restrictions']>1])\n",
    "df_all[df_all['donor_restrictions']>1][['FYE', 'donor_restrictions', 'temp_rest_assets', 'TemporarilyRstrNetAssetsGrp', 'TemporarilyRestrictedNetAssets',\n",
    "        'perm_rest_assets', 'PermanentlyRstrNetAssetsGrp', 'PermanentlyRestrictedNetAssets',\n",
    "        'net_assets', 'NetAssetsOrFundBalancesEOYAmt', 'NetAssetsOrFundBalancesEOY',\n",
    "                      #'UnrestrictedNetAssetsGrp', 'UnrestrictedNetAssets', \n",
    "              'TotalNetAssetsFundBalanceGrp', 'TotalNetAssetsFundBalances', \n",
    "              #'NetAssetsOrFundBalancesBOYAmt', 'NetAssetsOrFundBalancesBOY', \n",
    "              'NetAssetsOrFundBalancesEOYAmt', 'NetAssetsOrFundBalancesEOY']][-6:]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "##### Fees for Services (proxy for audit)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 228,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "3723 \n",
      "\n",
      "25717 \n",
      "\n"
     ]
    }
   ],
   "source": [
    "df_all['FeesForServicesAccountingGrp_v2'] = np.where(df_all['FeesForServicesAccountingGrp'].notnull(),\n",
    "                                                    df_all['FeesForServicesAccountingGrp'],\n",
    "                                                    df_all['FeesForServicesAccounting']) \n",
    "print len(df_all[df_all['FeesForServicesAccountingGrp_v2'].isnull()]), '\\n'\n",
    "print len(df_all[df_all['FeesForServicesAccountingGrp_v2'].notnull()]), '\\n'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 229,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "('Index: ', 26647, 'Fees:', u'21614') \n",
      "\n",
      "# of minutes:  23.574959151 \n",
      "\n",
      "\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "25717"
      ]
     },
     "execution_count": 229,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "for index, row in df_all[df_all['FeesForServicesAccountingGrp_v2'].notnull()][:].iterrows():\n",
    "    #print type(row['FeesForServicesAccountingGrp_v2']), row['FeesForServicesAccountingGrp_v2']\n",
    "    if 'Total' in row['FeesForServicesAccountingGrp_v2']:\n",
    "        #print 'THERE IS \"Total\"', row['FeesForServicesAccountingGrp_v2']['Total'] \n",
    "        df_all.ix[index, 'FeesForServicesAccounting_TotalAmt'] = row['FeesForServicesAccountingGrp_v2']['Total'] \n",
    "    if 'TotalAmt' in row['FeesForServicesAccountingGrp_v2']:\n",
    "        #print 'THERE IS \"TotalAmt\"', row['FeesForServicesAccountingGrp_v2']['TotalAmt']      \n",
    "        df_all.ix[index, 'FeesForServicesAccounting_TotalAmt'] = row['FeesForServicesAccountingGrp_v2']['TotalAmt'] \n",
    "        \n",
    "    clear_output()\n",
    "    print ('Index: ', index, 'Fees:', df_all.ix[index, 'FeesForServicesAccounting_TotalAmt']), '\\n'\n",
    "    sys.stdout.flush()        \n",
    "        \n",
    "elapsed = timeit.default_timer() - start_time\n",
    "print '# of minutes: ', elapsed/60, '\\n', '\\n'         \n",
    "df_all['FeesForServicesAccounting_TotalAmt'].value_counts().sum()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 230,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "df_all['FeesForServicesAccounting_TotalAmt'] = df_all['FeesForServicesAccounting_TotalAmt'].astype('float')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 231,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "1    24439\n",
       "0     5001\n",
       "Name: FeesForServicesAccounting_binary, dtype: int64"
      ]
     },
     "execution_count": 231,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_all['FeesForServicesAccounting_binary'] = np.where(df_all['FeesForServicesAccounting_TotalAmt']>0, 1,0)\n",
    "df_all['FeesForServicesAccounting_binary'].value_counts()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 232,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "# of rows where fees less than 0: 4 \n",
      "\n",
      "# of rows where fees equal zero: 1274 \n",
      "\n",
      "# of rows where fees greater than zero: 24439 \n",
      "\n",
      "# of rows missing values: 3723 \n",
      "\n"
     ]
    }
   ],
   "source": [
    "print '# of rows where fees less than 0:', len(df_all[df_all['FeesForServicesAccounting_TotalAmt']<0]), '\\n'\n",
    "print '# of rows where fees equal zero:', len(df_all[df_all['FeesForServicesAccounting_TotalAmt']==0]), '\\n'\n",
    "print '# of rows where fees greater than zero:', len(df_all[df_all['FeesForServicesAccounting_TotalAmt']>0]), '\\n'\n",
    "print '# of rows missing values:', len(df_all[df_all['FeesForServicesAccounting_TotalAmt'].isnull()]), '\\n'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 233,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "# of rows that should have value of zero: 5001\n"
     ]
    }
   ],
   "source": [
    "print '# of rows that should have value of zero:', 4+1274+3723"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 236,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>FeesForServicesAccounting_TotalAmt</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>25894</th>\n",
       "      <td>-2500.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>24536</th>\n",
       "      <td>-67439.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1466</th>\n",
       "      <td>-48284.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>12588</th>\n",
       "      <td>-44250.0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "       FeesForServicesAccounting_TotalAmt\n",
       "25894                             -2500.0\n",
       "24536                            -67439.0\n",
       "1466                             -48284.0\n",
       "12588                            -44250.0"
      ]
     },
     "execution_count": 236,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_all[df_all['FeesForServicesAccounting_TotalAmt']<0][:][['FeesForServicesAccounting_TotalAmt']]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 237,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "1    24439\n",
       "0     5001\n",
       "Name: FeesForServicesAccounting_binary, dtype: int64"
      ]
     },
     "execution_count": 237,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_all['FeesForServicesAccounting_binary'].value_counts()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 254,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "['EIN', 'FYE', 'VotingMembersGoverningBodyCnt', 'NbrVotingMembersGoverningBody', 'VotingMembersIndependentCnt', 'NbrIndependentVotingMembers', 'DelegationOfMgmtDutiesInd', 'DelegationOfManagementDuties', 'Form990ProvidedToGvrnBodyInd', 'Form990ProvidedToGoverningBody', 'AuditCommitteeInd', 'AuditCommittee', 'perm_rest_assets', 'temp_rest_assets', 'net_assets', 'donor_restrictions', 'TaxExemptBondsInd', 'TaxExemptBonds', 'NetUnrelatedBusTxblIncmAmt', 'NetUnrelatedBusinessTxblIncome', 'FSAuditedInd', 'FSAudited', 'FederalGrantAuditPerformedInd', 'FederalGrantAuditPerformed', 'FederalGrantAuditRequiredInd', 'FederalGrantAuditRequired', 'IndependentAuditFinclStmtInd', 'IndependentAuditFinancialStmt', 'FeesForServicesAccounting_binary']\n"
     ]
    }
   ],
   "source": [
    "cols = cols + ['FeesForServicesAccounting_binary']\n",
    "print cols"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 287,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "1\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>EIN</th>\n",
       "      <th>FYE</th>\n",
       "      <th>VotingMembersGoverningBodyCnt</th>\n",
       "      <th>NbrVotingMembersGoverningBody</th>\n",
       "      <th>VotingMembersIndependentCnt</th>\n",
       "      <th>NbrIndependentVotingMembers</th>\n",
       "      <th>DelegationOfMgmtDutiesInd</th>\n",
       "      <th>DelegationOfManagementDuties</th>\n",
       "      <th>Form990ProvidedToGvrnBodyInd</th>\n",
       "      <th>Form990ProvidedToGoverningBody</th>\n",
       "      <th>AuditCommitteeInd</th>\n",
       "      <th>AuditCommittee</th>\n",
       "      <th>perm_rest_assets</th>\n",
       "      <th>temp_rest_assets</th>\n",
       "      <th>net_assets</th>\n",
       "      <th>donor_restrictions</th>\n",
       "      <th>TaxExemptBondsInd</th>\n",
       "      <th>TaxExemptBonds</th>\n",
       "      <th>NetUnrelatedBusTxblIncmAmt</th>\n",
       "      <th>NetUnrelatedBusinessTxblIncome</th>\n",
       "      <th>FSAuditedInd</th>\n",
       "      <th>FSAudited</th>\n",
       "      <th>FederalGrantAuditPerformedInd</th>\n",
       "      <th>FederalGrantAuditPerformed</th>\n",
       "      <th>FederalGrantAuditRequiredInd</th>\n",
       "      <th>FederalGrantAuditRequired</th>\n",
       "      <th>IndependentAuditFinclStmtInd</th>\n",
       "      <th>IndependentAuditFinancialStmt</th>\n",
       "      <th>FeesForServicesAccounting_binary</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>27377</th>\n",
       "      <td>840516736</td>\n",
       "      <td>FY2014</td>\n",
       "      <td>10</td>\n",
       "      <td>NaN</td>\n",
       "      <td>9</td>\n",
       "      <td>NaN</td>\n",
       "      <td>false</td>\n",
       "      <td>NaN</td>\n",
       "      <td>true</td>\n",
       "      <td>NaN</td>\n",
       "      <td>true</td>\n",
       "      <td>NaN</td>\n",
       "      <td>260654</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>inf</td>\n",
       "      <td>false</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>true</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>false</td>\n",
       "      <td>NaN</td>\n",
       "      <td>{u'#text': u'true', u'@referenceDocumentId': u'00000004', u'@referenceDocumentName': u'IRS990ScheduleD'}</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "             EIN     FYE VotingMembersGoverningBodyCnt  \\\n",
       "27377  840516736  FY2014                            10   \n",
       "\n",
       "      NbrVotingMembersGoverningBody VotingMembersIndependentCnt  \\\n",
       "27377                           NaN                           9   \n",
       "\n",
       "      NbrIndependentVotingMembers DelegationOfMgmtDutiesInd  \\\n",
       "27377                         NaN                     false   \n",
       "\n",
       "      DelegationOfManagementDuties Form990ProvidedToGvrnBodyInd  \\\n",
       "27377                          NaN                         true   \n",
       "\n",
       "      Form990ProvidedToGoverningBody AuditCommitteeInd AuditCommittee  \\\n",
       "27377                            NaN              true            NaN   \n",
       "\n",
       "       perm_rest_assets  temp_rest_assets  net_assets  donor_restrictions  \\\n",
       "27377            260654                 0           0                 inf   \n",
       "\n",
       "      TaxExemptBondsInd TaxExemptBonds NetUnrelatedBusTxblIncmAmt  \\\n",
       "27377             false            NaN                        NaN   \n",
       "\n",
       "      NetUnrelatedBusinessTxblIncome FSAuditedInd FSAudited  \\\n",
       "27377                            NaN         true       NaN   \n",
       "\n",
       "      FederalGrantAuditPerformedInd FederalGrantAuditPerformed  \\\n",
       "27377                           NaN                        NaN   \n",
       "\n",
       "      FederalGrantAuditRequiredInd FederalGrantAuditRequired  \\\n",
       "27377                        false                       NaN   \n",
       "\n",
       "                                                                                   IndependentAuditFinclStmtInd  \\\n",
       "27377  {u'#text': u'true', u'@referenceDocumentId': u'00000004', u'@referenceDocumentName': u'IRS990ScheduleD'}   \n",
       "\n",
       "      IndependentAuditFinancialStmt  FeesForServicesAccounting_binary  \n",
       "27377                           NaN                                 0  "
      ]
     },
     "execution_count": 287,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "print len(df_all[df_all['donor_restrictions']>100])\n",
    "df_all[df_all['donor_restrictions']>100][cols]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "##### Family Business Relations"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 443,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "['EIN', 'FYE', 'VotingMembersGoverningBodyCnt', 'NbrVotingMembersGoverningBody', 'VotingMembersIndependentCnt', 'NbrIndependentVotingMembers', 'DelegationOfMgmtDutiesInd', 'DelegationOfManagementDuties', 'Form990ProvidedToGvrnBodyInd', 'Form990ProvidedToGoverningBody', 'AuditCommitteeInd', 'AuditCommittee', 'perm_rest_assets', 'temp_rest_assets', 'net_assets', 'donor_restrictions', 'TaxExemptBondsInd', 'TaxExemptBonds', 'NetUnrelatedBusTxblIncmAmt', 'NetUnrelatedBusinessTxblIncome', 'FSAuditedInd', 'FSAudited', 'FederalGrantAuditPerformedInd', 'FederalGrantAuditPerformed', 'FederalGrantAuditRequiredInd', 'FederalGrantAuditRequired', 'IndependentAuditFinclStmtInd', 'IndependentAuditFinancialStmt', 'FeesForServicesAccounting_binary', 'FamilyOrBusinessRlnInd', 'FamilyOrBusinessRelationship']\n"
     ]
    }
   ],
   "source": [
    "cols = cols + ['FamilyOrBusinessRlnInd', 'FamilyOrBusinessRelationship']\n",
    "print cols"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "##### Tax Exempt Bonds, Take II"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 444,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "['EIN', 'FYE', 'VotingMembersGoverningBodyCnt', 'NbrVotingMembersGoverningBody', 'VotingMembersIndependentCnt', 'NbrIndependentVotingMembers', 'DelegationOfMgmtDutiesInd', 'DelegationOfManagementDuties', 'Form990ProvidedToGvrnBodyInd', 'Form990ProvidedToGoverningBody', 'AuditCommitteeInd', 'AuditCommittee', 'perm_rest_assets', 'temp_rest_assets', 'net_assets', 'donor_restrictions', 'TaxExemptBondsInd', 'TaxExemptBonds', 'NetUnrelatedBusTxblIncmAmt', 'NetUnrelatedBusinessTxblIncome', 'FSAuditedInd', 'FSAudited', 'FederalGrantAuditPerformedInd', 'FederalGrantAuditPerformed', 'FederalGrantAuditRequiredInd', 'FederalGrantAuditRequired', 'IndependentAuditFinclStmtInd', 'IndependentAuditFinancialStmt', 'FeesForServicesAccounting_binary', 'FamilyOrBusinessRlnInd', 'FamilyOrBusinessRelationship', 'TaxExemptBondLiabilitiesGrp', 'TaxExemptBondLiabilities']\n"
     ]
    }
   ],
   "source": [
    "cols = cols + ['TaxExemptBondLiabilitiesGrp', 'TaxExemptBondLiabilities']\n",
    "print cols"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "##### Government Grants"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 445,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "['EIN', 'FYE', 'VotingMembersGoverningBodyCnt', 'NbrVotingMembersGoverningBody', 'VotingMembersIndependentCnt', 'NbrIndependentVotingMembers', 'DelegationOfMgmtDutiesInd', 'DelegationOfManagementDuties', 'Form990ProvidedToGvrnBodyInd', 'Form990ProvidedToGoverningBody', 'AuditCommitteeInd', 'AuditCommittee', 'perm_rest_assets', 'temp_rest_assets', 'net_assets', 'donor_restrictions', 'TaxExemptBondsInd', 'TaxExemptBonds', 'NetUnrelatedBusTxblIncmAmt', 'NetUnrelatedBusinessTxblIncome', 'FSAuditedInd', 'FSAudited', 'FederalGrantAuditPerformedInd', 'FederalGrantAuditPerformed', 'FederalGrantAuditRequiredInd', 'FederalGrantAuditRequired', 'IndependentAuditFinclStmtInd', 'IndependentAuditFinancialStmt', 'FeesForServicesAccounting_binary', 'FamilyOrBusinessRlnInd', 'FamilyOrBusinessRelationship', 'TaxExemptBondLiabilitiesGrp', 'TaxExemptBondLiabilities', 'GovernmentGrantsAmt', 'GovernmentGrants']\n"
     ]
    }
   ],
   "source": [
    "cols = cols + ['GovernmentGrantsAmt', 'GovernmentGrants']\n",
    "print cols"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "##### Election of Board Members"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 446,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "['EIN', 'FYE', 'VotingMembersGoverningBodyCnt', 'NbrVotingMembersGoverningBody', 'VotingMembersIndependentCnt', 'NbrIndependentVotingMembers', 'DelegationOfMgmtDutiesInd', 'DelegationOfManagementDuties', 'Form990ProvidedToGvrnBodyInd', 'Form990ProvidedToGoverningBody', 'AuditCommitteeInd', 'AuditCommittee', 'perm_rest_assets', 'temp_rest_assets', 'net_assets', 'donor_restrictions', 'TaxExemptBondsInd', 'TaxExemptBonds', 'NetUnrelatedBusTxblIncmAmt', 'NetUnrelatedBusinessTxblIncome', 'FSAuditedInd', 'FSAudited', 'FederalGrantAuditPerformedInd', 'FederalGrantAuditPerformed', 'FederalGrantAuditRequiredInd', 'FederalGrantAuditRequired', 'IndependentAuditFinclStmtInd', 'IndependentAuditFinancialStmt', 'FeesForServicesAccounting_binary', 'FamilyOrBusinessRlnInd', 'FamilyOrBusinessRelationship', 'TaxExemptBondLiabilitiesGrp', 'TaxExemptBondLiabilities', 'GovernmentGrantsAmt', 'GovernmentGrants', 'ElectionOfBoardMembersInd', 'ElectionOfBoardMembers']\n"
     ]
    }
   ],
   "source": [
    "cols = cols + ['ElectionOfBoardMembersInd', 'ElectionOfBoardMembers']\n",
    "print cols"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "##### Zero Fundraising"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 551,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "['EIN', 'FYE', 'VotingMembersGoverningBodyCnt', 'NbrVotingMembersGoverningBody', 'VotingMembersIndependentCnt', 'NbrIndependentVotingMembers', 'DelegationOfMgmtDutiesInd', 'DelegationOfManagementDuties', 'Form990ProvidedToGvrnBodyInd', 'Form990ProvidedToGoverningBody', 'AuditCommitteeInd', 'AuditCommittee', 'perm_rest_assets', 'temp_rest_assets', 'net_assets', 'donor_restrictions', 'TaxExemptBondsInd', 'TaxExemptBonds', 'NetUnrelatedBusTxblIncmAmt', 'NetUnrelatedBusinessTxblIncome', 'FSAuditedInd', 'FSAudited', 'FederalGrantAuditPerformedInd', 'FederalGrantAuditPerformed', 'FederalGrantAuditRequiredInd', 'FederalGrantAuditRequired', 'IndependentAuditFinclStmtInd', 'IndependentAuditFinancialStmt', 'FeesForServicesAccounting_binary', 'FamilyOrBusinessRlnInd', 'FamilyOrBusinessRelationship', 'TaxExemptBondLiabilitiesGrp', 'TaxExemptBondLiabilities', 'GovernmentGrantsAmt', 'GovernmentGrants', 'ElectionOfBoardMembersInd', 'ElectionOfBoardMembers', 'CYTotalFundraisingExpenseAmt', 'TotalFundrsngExpCurrentYear']\n"
     ]
    }
   ],
   "source": [
    "cols = cols + ['CYTotalFundraisingExpenseAmt', 'TotalFundrsngExpCurrentYear']\n",
    "print cols"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "##### Save DF"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 255,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "29440"
      ]
     },
     "execution_count": 255,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(df_all)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 256,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "df_all.to_pickle('e-file data for 7,133 of 8,304 2016 EINs (n=24940).pkl')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Collapse Dataset -- take last"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 447,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>EIN</th>\n",
       "      <th>FYE</th>\n",
       "      <th>VotingMembersGoverningBodyCnt</th>\n",
       "      <th>NbrVotingMembersGoverningBody</th>\n",
       "      <th>VotingMembersIndependentCnt</th>\n",
       "      <th>NbrIndependentVotingMembers</th>\n",
       "      <th>DelegationOfMgmtDutiesInd</th>\n",
       "      <th>DelegationOfManagementDuties</th>\n",
       "      <th>Form990ProvidedToGvrnBodyInd</th>\n",
       "      <th>Form990ProvidedToGoverningBody</th>\n",
       "      <th>AuditCommitteeInd</th>\n",
       "      <th>AuditCommittee</th>\n",
       "      <th>perm_rest_assets</th>\n",
       "      <th>temp_rest_assets</th>\n",
       "      <th>net_assets</th>\n",
       "      <th>donor_restrictions</th>\n",
       "      <th>TaxExemptBondsInd</th>\n",
       "      <th>TaxExemptBonds</th>\n",
       "      <th>NetUnrelatedBusTxblIncmAmt</th>\n",
       "      <th>NetUnrelatedBusinessTxblIncome</th>\n",
       "      <th>FSAuditedInd</th>\n",
       "      <th>FSAudited</th>\n",
       "      <th>FederalGrantAuditPerformedInd</th>\n",
       "      <th>FederalGrantAuditPerformed</th>\n",
       "      <th>FederalGrantAuditRequiredInd</th>\n",
       "      <th>FederalGrantAuditRequired</th>\n",
       "      <th>IndependentAuditFinclStmtInd</th>\n",
       "      <th>IndependentAuditFinancialStmt</th>\n",
       "      <th>FeesForServicesAccounting_binary</th>\n",
       "      <th>FamilyOrBusinessRlnInd</th>\n",
       "      <th>FamilyOrBusinessRelationship</th>\n",
       "      <th>TaxExemptBondLiabilitiesGrp</th>\n",
       "      <th>TaxExemptBondLiabilities</th>\n",
       "      <th>GovernmentGrantsAmt</th>\n",
       "      <th>GovernmentGrants</th>\n",
       "      <th>ElectionOfBoardMembersInd</th>\n",
       "      <th>ElectionOfBoardMembers</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>18819</th>\n",
       "      <td>010202467</td>\n",
       "      <td>FY2014</td>\n",
       "      <td>22</td>\n",
       "      <td>NaN</td>\n",
       "      <td>21</td>\n",
       "      <td>NaN</td>\n",
       "      <td>1</td>\n",
       "      <td>NaN</td>\n",
       "      <td>1</td>\n",
       "      <td>NaN</td>\n",
       "      <td>1</td>\n",
       "      <td>NaN</td>\n",
       "      <td>4433997</td>\n",
       "      <td>7351259</td>\n",
       "      <td>23690097</td>\n",
       "      <td>0.50</td>\n",
       "      <td>{u'#text': u'1', u'@referenceDocumentId': u'RetDoc1053100001'}</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>1</td>\n",
       "      <td>NaN</td>\n",
       "      <td>1</td>\n",
       "      <td>NaN</td>\n",
       "      <td>1</td>\n",
       "      <td>NaN</td>\n",
       "      <td>{u'#text': u'1', u'@referenceDocumentId': u'RetDoc1040000001'}</td>\n",
       "      <td>NaN</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>7932282</td>\n",
       "      <td>NaN</td>\n",
       "      <td>1</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7965</th>\n",
       "      <td>010202467</td>\n",
       "      <td>FY2013</td>\n",
       "      <td>21</td>\n",
       "      <td>NaN</td>\n",
       "      <td>21</td>\n",
       "      <td>NaN</td>\n",
       "      <td>1</td>\n",
       "      <td>NaN</td>\n",
       "      <td>1</td>\n",
       "      <td>NaN</td>\n",
       "      <td>1</td>\n",
       "      <td>NaN</td>\n",
       "      <td>4193257</td>\n",
       "      <td>5880227</td>\n",
       "      <td>21657900</td>\n",
       "      <td>0.47</td>\n",
       "      <td>{u'#text': u'1', u'@referenceDocumentId': u'RetDoc1053100001'}</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>1</td>\n",
       "      <td>NaN</td>\n",
       "      <td>1</td>\n",
       "      <td>NaN</td>\n",
       "      <td>1</td>\n",
       "      <td>NaN</td>\n",
       "      <td>{u'#text': u'1', u'@referenceDocumentId': u'RetDoc1040000001'}</td>\n",
       "      <td>NaN</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>6258147</td>\n",
       "      <td>NaN</td>\n",
       "      <td>1</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8989</th>\n",
       "      <td>010202467</td>\n",
       "      <td>FY2012</td>\n",
       "      <td>NaN</td>\n",
       "      <td>20</td>\n",
       "      <td>NaN</td>\n",
       "      <td>20</td>\n",
       "      <td>NaN</td>\n",
       "      <td>1</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>1</td>\n",
       "      <td>4153627</td>\n",
       "      <td>5773597</td>\n",
       "      <td>22130919</td>\n",
       "      <td>0.45</td>\n",
       "      <td>NaN</td>\n",
       "      <td>{u'#text': u'1', u'@referenceDocumentId': u'RetDoc1053100001'}</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>1</td>\n",
       "      <td>NaN</td>\n",
       "      <td>1</td>\n",
       "      <td>NaN</td>\n",
       "      <td>1</td>\n",
       "      <td>NaN</td>\n",
       "      <td>{u'#text': u'1', u'@referenceDocumentId': u'RetDoc1040000001'}</td>\n",
       "      <td>1</td>\n",
       "      <td>NaN</td>\n",
       "      <td>1</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>7295185</td>\n",
       "      <td>NaN</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>22904</th>\n",
       "      <td>010202467</td>\n",
       "      <td>FY2011</td>\n",
       "      <td>NaN</td>\n",
       "      <td>23</td>\n",
       "      <td>NaN</td>\n",
       "      <td>23</td>\n",
       "      <td>NaN</td>\n",
       "      <td>1</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>1</td>\n",
       "      <td>3096087</td>\n",
       "      <td>6554891</td>\n",
       "      <td>20486698</td>\n",
       "      <td>0.47</td>\n",
       "      <td>NaN</td>\n",
       "      <td>{u'#text': u'1', u'@referenceDocumentId': u'RetDoc1053100001'}</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>1</td>\n",
       "      <td>NaN</td>\n",
       "      <td>1</td>\n",
       "      <td>NaN</td>\n",
       "      <td>1</td>\n",
       "      <td>NaN</td>\n",
       "      <td>{u'#text': u'1', u'@referenceDocumentId': u'RetDoc1040000001'}</td>\n",
       "      <td>1</td>\n",
       "      <td>NaN</td>\n",
       "      <td>1</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>9819406</td>\n",
       "      <td>NaN</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>19649</th>\n",
       "      <td>010202467</td>\n",
       "      <td>FY2010</td>\n",
       "      <td>NaN</td>\n",
       "      <td>24</td>\n",
       "      <td>NaN</td>\n",
       "      <td>24</td>\n",
       "      <td>NaN</td>\n",
       "      <td>1</td>\n",
       "      <td>NaN</td>\n",
       "      <td>1</td>\n",
       "      <td>NaN</td>\n",
       "      <td>1</td>\n",
       "      <td>2873816</td>\n",
       "      <td>7110362</td>\n",
       "      <td>17935900</td>\n",
       "      <td>0.56</td>\n",
       "      <td>NaN</td>\n",
       "      <td>{u'#text': u'1', u'@referenceDocumentId': u'RetDoc1053100001'}</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>1</td>\n",
       "      <td>NaN</td>\n",
       "      <td>1</td>\n",
       "      <td>NaN</td>\n",
       "      <td>1</td>\n",
       "      <td>NaN</td>\n",
       "      <td>{u'#text': u'1', u'@referenceDocumentId': u'RetDoc1039900001'}</td>\n",
       "      <td>1</td>\n",
       "      <td>NaN</td>\n",
       "      <td>1</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>7006788</td>\n",
       "      <td>NaN</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "             EIN     FYE VotingMembersGoverningBodyCnt  \\\n",
       "18819  010202467  FY2014                            22   \n",
       "7965   010202467  FY2013                            21   \n",
       "8989   010202467  FY2012                           NaN   \n",
       "22904  010202467  FY2011                           NaN   \n",
       "19649  010202467  FY2010                           NaN   \n",
       "\n",
       "      NbrVotingMembersGoverningBody VotingMembersIndependentCnt  \\\n",
       "18819                           NaN                          21   \n",
       "7965                            NaN                          21   \n",
       "8989                             20                         NaN   \n",
       "22904                            23                         NaN   \n",
       "19649                            24                         NaN   \n",
       "\n",
       "      NbrIndependentVotingMembers DelegationOfMgmtDutiesInd  \\\n",
       "18819                         NaN                         1   \n",
       "7965                          NaN                         1   \n",
       "8989                           20                       NaN   \n",
       "22904                          23                       NaN   \n",
       "19649                          24                       NaN   \n",
       "\n",
       "      DelegationOfManagementDuties Form990ProvidedToGvrnBodyInd  \\\n",
       "18819                          NaN                            1   \n",
       "7965                           NaN                            1   \n",
       "8989                             1                          NaN   \n",
       "22904                            1                          NaN   \n",
       "19649                            1                          NaN   \n",
       "\n",
       "      Form990ProvidedToGoverningBody AuditCommitteeInd AuditCommittee  \\\n",
       "18819                            NaN                 1            NaN   \n",
       "7965                             NaN                 1            NaN   \n",
       "8989                               0               NaN              1   \n",
       "22904                              0               NaN              1   \n",
       "19649                              1               NaN              1   \n",
       "\n",
       "       perm_rest_assets  temp_rest_assets  net_assets  donor_restrictions  \\\n",
       "18819           4433997           7351259    23690097                0.50   \n",
       "7965            4193257           5880227    21657900                0.47   \n",
       "8989            4153627           5773597    22130919                0.45   \n",
       "22904           3096087           6554891    20486698                0.47   \n",
       "19649           2873816           7110362    17935900                0.56   \n",
       "\n",
       "                                                    TaxExemptBondsInd  \\\n",
       "18819  {u'#text': u'1', u'@referenceDocumentId': u'RetDoc1053100001'}   \n",
       "7965   {u'#text': u'1', u'@referenceDocumentId': u'RetDoc1053100001'}   \n",
       "8989                                                              NaN   \n",
       "22904                                                             NaN   \n",
       "19649                                                             NaN   \n",
       "\n",
       "                                                       TaxExemptBonds  \\\n",
       "18819                                                             NaN   \n",
       "7965                                                              NaN   \n",
       "8989   {u'#text': u'1', u'@referenceDocumentId': u'RetDoc1053100001'}   \n",
       "22904  {u'#text': u'1', u'@referenceDocumentId': u'RetDoc1053100001'}   \n",
       "19649  {u'#text': u'1', u'@referenceDocumentId': u'RetDoc1053100001'}   \n",
       "\n",
       "      NetUnrelatedBusTxblIncmAmt NetUnrelatedBusinessTxblIncome FSAuditedInd  \\\n",
       "18819                          0                            NaN            1   \n",
       "7965                           0                            NaN            1   \n",
       "8989                         NaN                              0          NaN   \n",
       "22904                        NaN                              0          NaN   \n",
       "19649                        NaN                              0          NaN   \n",
       "\n",
       "      FSAudited FederalGrantAuditPerformedInd FederalGrantAuditPerformed  \\\n",
       "18819       NaN                             1                        NaN   \n",
       "7965        NaN                             1                        NaN   \n",
       "8989          1                           NaN                          1   \n",
       "22904         1                           NaN                          1   \n",
       "19649         1                           NaN                          1   \n",
       "\n",
       "      FederalGrantAuditRequiredInd FederalGrantAuditRequired  \\\n",
       "18819                            1                       NaN   \n",
       "7965                             1                       NaN   \n",
       "8989                           NaN                         1   \n",
       "22904                          NaN                         1   \n",
       "19649                          NaN                         1   \n",
       "\n",
       "                                         IndependentAuditFinclStmtInd  \\\n",
       "18819  {u'#text': u'1', u'@referenceDocumentId': u'RetDoc1040000001'}   \n",
       "7965   {u'#text': u'1', u'@referenceDocumentId': u'RetDoc1040000001'}   \n",
       "8989                                                              NaN   \n",
       "22904                                                             NaN   \n",
       "19649                                                             NaN   \n",
       "\n",
       "                                        IndependentAuditFinancialStmt  \\\n",
       "18819                                                             NaN   \n",
       "7965                                                              NaN   \n",
       "8989   {u'#text': u'1', u'@referenceDocumentId': u'RetDoc1040000001'}   \n",
       "22904  {u'#text': u'1', u'@referenceDocumentId': u'RetDoc1040000001'}   \n",
       "19649  {u'#text': u'1', u'@referenceDocumentId': u'RetDoc1039900001'}   \n",
       "\n",
       "       FeesForServicesAccounting_binary FamilyOrBusinessRlnInd  \\\n",
       "18819                                 1                      0   \n",
       "7965                                  1                      1   \n",
       "8989                                  1                    NaN   \n",
       "22904                                 1                    NaN   \n",
       "19649                                 1                    NaN   \n",
       "\n",
       "      FamilyOrBusinessRelationship TaxExemptBondLiabilitiesGrp  \\\n",
       "18819                          NaN                         NaN   \n",
       "7965                           NaN                         NaN   \n",
       "8989                             1                         NaN   \n",
       "22904                            1                         NaN   \n",
       "19649                            1                         NaN   \n",
       "\n",
       "      TaxExemptBondLiabilities GovernmentGrantsAmt GovernmentGrants  \\\n",
       "18819                      NaN             7932282              NaN   \n",
       "7965                       NaN             6258147              NaN   \n",
       "8989                       NaN                 NaN          7295185   \n",
       "22904                      NaN                 NaN          9819406   \n",
       "19649                      NaN                 NaN          7006788   \n",
       "\n",
       "      ElectionOfBoardMembersInd ElectionOfBoardMembers  \n",
       "18819                         1                    NaN  \n",
       "7965                          1                    NaN  \n",
       "8989                        NaN                      1  \n",
       "22904                       NaN                      1  \n",
       "19649                       NaN                      1  "
      ]
     },
     "execution_count": 447,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_all = df_all.sort_values(by=['EIN', 'FYE'], ascending=[1,0])\n",
    "df_all[:5][cols]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 259,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>EIN</th>\n",
       "      <th>FYE</th>\n",
       "      <th>VotingMembersGoverningBodyCnt</th>\n",
       "      <th>NbrVotingMembersGoverningBody</th>\n",
       "      <th>VotingMembersIndependentCnt</th>\n",
       "      <th>NbrIndependentVotingMembers</th>\n",
       "      <th>DelegationOfMgmtDutiesInd</th>\n",
       "      <th>DelegationOfManagementDuties</th>\n",
       "      <th>Form990ProvidedToGvrnBodyInd</th>\n",
       "      <th>Form990ProvidedToGoverningBody</th>\n",
       "      <th>AuditCommitteeInd</th>\n",
       "      <th>AuditCommittee</th>\n",
       "      <th>perm_rest_assets</th>\n",
       "      <th>temp_rest_assets</th>\n",
       "      <th>net_assets</th>\n",
       "      <th>donor_restrictions</th>\n",
       "      <th>TaxExemptBondsInd</th>\n",
       "      <th>TaxExemptBonds</th>\n",
       "      <th>NetUnrelatedBusTxblIncmAmt</th>\n",
       "      <th>NetUnrelatedBusinessTxblIncome</th>\n",
       "      <th>FSAuditedInd</th>\n",
       "      <th>FSAudited</th>\n",
       "      <th>FederalGrantAuditPerformedInd</th>\n",
       "      <th>FederalGrantAuditPerformed</th>\n",
       "      <th>FederalGrantAuditRequiredInd</th>\n",
       "      <th>FederalGrantAuditRequired</th>\n",
       "      <th>IndependentAuditFinclStmtInd</th>\n",
       "      <th>IndependentAuditFinancialStmt</th>\n",
       "      <th>FeesForServicesAccounting_binary</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>23577</th>\n",
       "      <td>990350803</td>\n",
       "      <td>FY2012</td>\n",
       "      <td>NaN</td>\n",
       "      <td>11</td>\n",
       "      <td>NaN</td>\n",
       "      <td>11</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>1</td>\n",
       "      <td>NaN</td>\n",
       "      <td>1</td>\n",
       "      <td>100000</td>\n",
       "      <td>467396</td>\n",
       "      <td>2441674</td>\n",
       "      <td>0.232380</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>1</td>\n",
       "      <td>NaN</td>\n",
       "      <td>1</td>\n",
       "      <td>NaN</td>\n",
       "      <td>1</td>\n",
       "      <td>NaN</td>\n",
       "      <td>{u'#text': u'1', u'@referenceDocumentId': u'RetDoc1040000001'}</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>16294</th>\n",
       "      <td>990350803</td>\n",
       "      <td>FY2011</td>\n",
       "      <td>NaN</td>\n",
       "      <td>11</td>\n",
       "      <td>NaN</td>\n",
       "      <td>11</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>1</td>\n",
       "      <td>NaN</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>171880</td>\n",
       "      <td>1953372</td>\n",
       "      <td>0.087991</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>1</td>\n",
       "      <td>NaN</td>\n",
       "      <td>1</td>\n",
       "      <td>NaN</td>\n",
       "      <td>1</td>\n",
       "      <td>NaN</td>\n",
       "      <td>{u'#text': u'1', u'@referenceDocumentId': u'RetDoc1040000001'}</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>26647</th>\n",
       "      <td>990350803</td>\n",
       "      <td>FY2010</td>\n",
       "      <td>NaN</td>\n",
       "      <td>7</td>\n",
       "      <td>NaN</td>\n",
       "      <td>7</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>1</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0</td>\n",
       "      <td>96956</td>\n",
       "      <td>1720152</td>\n",
       "      <td>0.056365</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>1</td>\n",
       "      <td>NaN</td>\n",
       "      <td>{u'#text': u'0', u'@referenceDocumentId': u'RetDoc1040000001'}</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "             EIN     FYE VotingMembersGoverningBodyCnt  \\\n",
       "23577  990350803  FY2012                           NaN   \n",
       "16294  990350803  FY2011                           NaN   \n",
       "26647  990350803  FY2010                           NaN   \n",
       "\n",
       "      NbrVotingMembersGoverningBody VotingMembersIndependentCnt  \\\n",
       "23577                            11                         NaN   \n",
       "16294                            11                         NaN   \n",
       "26647                             7                         NaN   \n",
       "\n",
       "      NbrIndependentVotingMembers DelegationOfMgmtDutiesInd  \\\n",
       "23577                          11                       NaN   \n",
       "16294                          11                       NaN   \n",
       "26647                           7                       NaN   \n",
       "\n",
       "      DelegationOfManagementDuties Form990ProvidedToGvrnBodyInd  \\\n",
       "23577                            0                          NaN   \n",
       "16294                            0                          NaN   \n",
       "26647                            0                          NaN   \n",
       "\n",
       "      Form990ProvidedToGoverningBody AuditCommitteeInd AuditCommittee  \\\n",
       "23577                              1               NaN              1   \n",
       "16294                              1               NaN              1   \n",
       "26647                              1               NaN            NaN   \n",
       "\n",
       "       perm_rest_assets  temp_rest_assets  net_assets  donor_restrictions  \\\n",
       "23577            100000            467396     2441674            0.232380   \n",
       "16294                 0            171880     1953372            0.087991   \n",
       "26647                 0             96956     1720152            0.056365   \n",
       "\n",
       "      TaxExemptBondsInd TaxExemptBonds NetUnrelatedBusTxblIncmAmt  \\\n",
       "23577               NaN              0                        NaN   \n",
       "16294               NaN              0                        NaN   \n",
       "26647               NaN              0                        NaN   \n",
       "\n",
       "      NetUnrelatedBusinessTxblIncome FSAuditedInd FSAudited  \\\n",
       "23577                              0          NaN         1   \n",
       "16294                              0          NaN         1   \n",
       "26647                              0          NaN         0   \n",
       "\n",
       "      FederalGrantAuditPerformedInd FederalGrantAuditPerformed  \\\n",
       "23577                           NaN                          1   \n",
       "16294                           NaN                          1   \n",
       "26647                           NaN                          0   \n",
       "\n",
       "      FederalGrantAuditRequiredInd FederalGrantAuditRequired  \\\n",
       "23577                          NaN                         1   \n",
       "16294                          NaN                         1   \n",
       "26647                          NaN                         1   \n",
       "\n",
       "      IndependentAuditFinclStmtInd  \\\n",
       "23577                          NaN   \n",
       "16294                          NaN   \n",
       "26647                          NaN   \n",
       "\n",
       "                                        IndependentAuditFinancialStmt  \\\n",
       "23577  {u'#text': u'1', u'@referenceDocumentId': u'RetDoc1040000001'}   \n",
       "16294  {u'#text': u'1', u'@referenceDocumentId': u'RetDoc1040000001'}   \n",
       "26647  {u'#text': u'0', u'@referenceDocumentId': u'RetDoc1040000001'}   \n",
       "\n",
       "       FeesForServicesAccounting_binary  \n",
       "23577                                 1  \n",
       "16294                                 1  \n",
       "26647                                 1  "
      ]
     },
     "execution_count": 259,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_all[-3:][cols]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 552,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "7133\n",
      "7133\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>EIN</th>\n",
       "      <th>FYE</th>\n",
       "      <th>VotingMembersGoverningBodyCnt</th>\n",
       "      <th>NbrVotingMembersGoverningBody</th>\n",
       "      <th>VotingMembersIndependentCnt</th>\n",
       "      <th>NbrIndependentVotingMembers</th>\n",
       "      <th>DelegationOfMgmtDutiesInd</th>\n",
       "      <th>DelegationOfManagementDuties</th>\n",
       "      <th>Form990ProvidedToGvrnBodyInd</th>\n",
       "      <th>Form990ProvidedToGoverningBody</th>\n",
       "      <th>AuditCommitteeInd</th>\n",
       "      <th>AuditCommittee</th>\n",
       "      <th>perm_rest_assets</th>\n",
       "      <th>temp_rest_assets</th>\n",
       "      <th>net_assets</th>\n",
       "      <th>donor_restrictions</th>\n",
       "      <th>TaxExemptBondsInd</th>\n",
       "      <th>TaxExemptBonds</th>\n",
       "      <th>NetUnrelatedBusTxblIncmAmt</th>\n",
       "      <th>NetUnrelatedBusinessTxblIncome</th>\n",
       "      <th>FSAuditedInd</th>\n",
       "      <th>FSAudited</th>\n",
       "      <th>FederalGrantAuditPerformedInd</th>\n",
       "      <th>FederalGrantAuditPerformed</th>\n",
       "      <th>FederalGrantAuditRequiredInd</th>\n",
       "      <th>FederalGrantAuditRequired</th>\n",
       "      <th>IndependentAuditFinclStmtInd</th>\n",
       "      <th>IndependentAuditFinancialStmt</th>\n",
       "      <th>FeesForServicesAccounting_binary</th>\n",
       "      <th>FamilyOrBusinessRlnInd</th>\n",
       "      <th>FamilyOrBusinessRelationship</th>\n",
       "      <th>TaxExemptBondLiabilitiesGrp</th>\n",
       "      <th>TaxExemptBondLiabilities</th>\n",
       "      <th>GovernmentGrantsAmt</th>\n",
       "      <th>GovernmentGrants</th>\n",
       "      <th>ElectionOfBoardMembersInd</th>\n",
       "      <th>ElectionOfBoardMembers</th>\n",
       "      <th>CYTotalFundraisingExpenseAmt</th>\n",
       "      <th>TotalFundrsngExpCurrentYear</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>010202467</td>\n",
       "      <td>FY2014</td>\n",
       "      <td>22</td>\n",
       "      <td>20</td>\n",
       "      <td>21</td>\n",
       "      <td>20</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>4433997</td>\n",
       "      <td>7351259</td>\n",
       "      <td>23690097</td>\n",
       "      <td>0.50</td>\n",
       "      <td>{u'#text': u'1', u'@referenceDocumentId': u'RetDoc1053100001'}</td>\n",
       "      <td>{u'#text': u'1', u'@referenceDocumentId': u'RetDoc1053100001'}</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>{u'#text': u'1', u'@referenceDocumentId': u'RetDoc1040000001'}</td>\n",
       "      <td>{u'#text': u'1', u'@referenceDocumentId': u'RetDoc1040000001'}</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>7932282</td>\n",
       "      <td>7295185</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>554608</td>\n",
       "      <td>358835</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>010211478</td>\n",
       "      <td>FY2015</td>\n",
       "      <td>18</td>\n",
       "      <td>18</td>\n",
       "      <td>18</td>\n",
       "      <td>18</td>\n",
       "      <td>false</td>\n",
       "      <td>false</td>\n",
       "      <td>true</td>\n",
       "      <td>true</td>\n",
       "      <td>true</td>\n",
       "      <td>true</td>\n",
       "      <td>2717000</td>\n",
       "      <td>908556</td>\n",
       "      <td>4804612</td>\n",
       "      <td>0.75</td>\n",
       "      <td>false</td>\n",
       "      <td>false</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>true</td>\n",
       "      <td>true</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>false</td>\n",
       "      <td>false</td>\n",
       "      <td>{u'#text': u'true', u'@referenceDocumentId': u'RetDoc3'}</td>\n",
       "      <td>{u'#text': u'true', u'@referenceDocumentId': u'RetDoc3'}</td>\n",
       "      <td>0</td>\n",
       "      <td>false</td>\n",
       "      <td>false</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>42442</td>\n",
       "      <td>129755</td>\n",
       "      <td>false</td>\n",
       "      <td>false</td>\n",
       "      <td>415743</td>\n",
       "      <td>363139</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>010211513</td>\n",
       "      <td>FY2014</td>\n",
       "      <td>23</td>\n",
       "      <td>22</td>\n",
       "      <td>21</td>\n",
       "      <td>20</td>\n",
       "      <td>false</td>\n",
       "      <td>false</td>\n",
       "      <td>true</td>\n",
       "      <td>true</td>\n",
       "      <td>true</td>\n",
       "      <td>true</td>\n",
       "      <td>15329657</td>\n",
       "      <td>38570318</td>\n",
       "      <td>492985953</td>\n",
       "      <td>0.11</td>\n",
       "      <td>{u'#text': u'true', u'@referenceDocumentId': u'IRS990ScheduleK'}</td>\n",
       "      <td>{u'#text': u'true', u'@referenceDocumentId': u'IRS990ScheduleK'}</td>\n",
       "      <td>-57641</td>\n",
       "      <td>-12313</td>\n",
       "      <td>true</td>\n",
       "      <td>true</td>\n",
       "      <td>true</td>\n",
       "      <td>true</td>\n",
       "      <td>true</td>\n",
       "      <td>true</td>\n",
       "      <td>{u'#text': u'true', u'@referenceDocumentId': u'IRS990ScheduleD'}</td>\n",
       "      <td>{u'#text': u'true', u'@referenceDocumentId': u'IRS990ScheduleD'}</td>\n",
       "      <td>1</td>\n",
       "      <td>false</td>\n",
       "      <td>true</td>\n",
       "      <td>{u'BOYAmt': u'106014868', u'EOYAmt': u'104647760'}</td>\n",
       "      <td>{u'BOY': u'77905017', u'EOY': u'109371484'}</td>\n",
       "      <td>77323736</td>\n",
       "      <td>57810755</td>\n",
       "      <td>false</td>\n",
       "      <td>false</td>\n",
       "      <td>3474538</td>\n",
       "      <td>2402953</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>010211530</td>\n",
       "      <td>FY2014</td>\n",
       "      <td>28</td>\n",
       "      <td>29</td>\n",
       "      <td>28</td>\n",
       "      <td>29</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>477857</td>\n",
       "      <td>1161177</td>\n",
       "      <td>12763602</td>\n",
       "      <td>0.13</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>{u'#text': u'1', u'@referenceDocumentId': u'RetDoc1040000001'}</td>\n",
       "      <td>{u'#text': u'1', u'@referenceDocumentId': u'RetDoc1040000001'}</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>237183</td>\n",
       "      <td>318383</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>181290</td>\n",
       "      <td>255685</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>010211543</td>\n",
       "      <td>FY2014</td>\n",
       "      <td>35</td>\n",
       "      <td>34</td>\n",
       "      <td>35</td>\n",
       "      <td>34</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>779327</td>\n",
       "      <td>4259694</td>\n",
       "      <td>16702601</td>\n",
       "      <td>0.30</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>{u'#text': u'1', u'@referenceDocumentId': u'RetDoc1040000001'}</td>\n",
       "      <td>{u'#text': u'1', u'@referenceDocumentId': u'RetDoc1040000001'}</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>94663</td>\n",
       "      <td>113548</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>150504</td>\n",
       "      <td>103085</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "         EIN     FYE VotingMembersGoverningBodyCnt  \\\n",
       "0  010202467  FY2014                            22   \n",
       "1  010211478  FY2015                            18   \n",
       "2  010211513  FY2014                            23   \n",
       "3  010211530  FY2014                            28   \n",
       "4  010211543  FY2014                            35   \n",
       "\n",
       "  NbrVotingMembersGoverningBody VotingMembersIndependentCnt  \\\n",
       "0                            20                          21   \n",
       "1                            18                          18   \n",
       "2                            22                          21   \n",
       "3                            29                          28   \n",
       "4                            34                          35   \n",
       "\n",
       "  NbrIndependentVotingMembers DelegationOfMgmtDutiesInd  \\\n",
       "0                          20                         1   \n",
       "1                          18                     false   \n",
       "2                          20                     false   \n",
       "3                          29                         0   \n",
       "4                          34                         0   \n",
       "\n",
       "  DelegationOfManagementDuties Form990ProvidedToGvrnBodyInd  \\\n",
       "0                            1                            1   \n",
       "1                        false                         true   \n",
       "2                        false                         true   \n",
       "3                            0                            0   \n",
       "4                            0                            1   \n",
       "\n",
       "  Form990ProvidedToGoverningBody AuditCommitteeInd AuditCommittee  \\\n",
       "0                              0                 1              1   \n",
       "1                           true              true           true   \n",
       "2                           true              true           true   \n",
       "3                              0                 1              1   \n",
       "4                              1                 1              1   \n",
       "\n",
       "   perm_rest_assets  temp_rest_assets  net_assets  donor_restrictions  \\\n",
       "0           4433997           7351259    23690097                0.50   \n",
       "1           2717000            908556     4804612                0.75   \n",
       "2          15329657          38570318   492985953                0.11   \n",
       "3            477857           1161177    12763602                0.13   \n",
       "4            779327           4259694    16702601                0.30   \n",
       "\n",
       "                                                  TaxExemptBondsInd  \\\n",
       "0    {u'#text': u'1', u'@referenceDocumentId': u'RetDoc1053100001'}   \n",
       "1                                                             false   \n",
       "2  {u'#text': u'true', u'@referenceDocumentId': u'IRS990ScheduleK'}   \n",
       "3                                                                 0   \n",
       "4                                                                 0   \n",
       "\n",
       "                                                     TaxExemptBonds  \\\n",
       "0    {u'#text': u'1', u'@referenceDocumentId': u'RetDoc1053100001'}   \n",
       "1                                                             false   \n",
       "2  {u'#text': u'true', u'@referenceDocumentId': u'IRS990ScheduleK'}   \n",
       "3                                                                 0   \n",
       "4                                                                 0   \n",
       "\n",
       "  NetUnrelatedBusTxblIncmAmt NetUnrelatedBusinessTxblIncome FSAuditedInd  \\\n",
       "0                          0                              0            1   \n",
       "1                        NaN                            NaN         true   \n",
       "2                     -57641                         -12313         true   \n",
       "3                          0                              0            1   \n",
       "4                          0                              0            1   \n",
       "\n",
       "  FSAudited FederalGrantAuditPerformedInd FederalGrantAuditPerformed  \\\n",
       "0         1                             1                          1   \n",
       "1      true                           NaN                        NaN   \n",
       "2      true                          true                       true   \n",
       "3         1                           NaN                        NaN   \n",
       "4         1                           NaN                        NaN   \n",
       "\n",
       "  FederalGrantAuditRequiredInd FederalGrantAuditRequired  \\\n",
       "0                            1                         1   \n",
       "1                        false                     false   \n",
       "2                         true                      true   \n",
       "3                            0                         0   \n",
       "4                            0                         0   \n",
       "\n",
       "                                       IndependentAuditFinclStmtInd  \\\n",
       "0    {u'#text': u'1', u'@referenceDocumentId': u'RetDoc1040000001'}   \n",
       "1          {u'#text': u'true', u'@referenceDocumentId': u'RetDoc3'}   \n",
       "2  {u'#text': u'true', u'@referenceDocumentId': u'IRS990ScheduleD'}   \n",
       "3    {u'#text': u'1', u'@referenceDocumentId': u'RetDoc1040000001'}   \n",
       "4    {u'#text': u'1', u'@referenceDocumentId': u'RetDoc1040000001'}   \n",
       "\n",
       "                                      IndependentAuditFinancialStmt  \\\n",
       "0    {u'#text': u'1', u'@referenceDocumentId': u'RetDoc1040000001'}   \n",
       "1          {u'#text': u'true', u'@referenceDocumentId': u'RetDoc3'}   \n",
       "2  {u'#text': u'true', u'@referenceDocumentId': u'IRS990ScheduleD'}   \n",
       "3    {u'#text': u'1', u'@referenceDocumentId': u'RetDoc1040000001'}   \n",
       "4    {u'#text': u'1', u'@referenceDocumentId': u'RetDoc1040000001'}   \n",
       "\n",
       "   FeesForServicesAccounting_binary FamilyOrBusinessRlnInd  \\\n",
       "0                                 1                      0   \n",
       "1                                 0                  false   \n",
       "2                                 1                  false   \n",
       "3                                 0                      0   \n",
       "4                                 1                      0   \n",
       "\n",
       "  FamilyOrBusinessRelationship  \\\n",
       "0                            1   \n",
       "1                        false   \n",
       "2                         true   \n",
       "3                            0   \n",
       "4                            0   \n",
       "\n",
       "                          TaxExemptBondLiabilitiesGrp  \\\n",
       "0                                                 NaN   \n",
       "1                                                 NaN   \n",
       "2  {u'BOYAmt': u'106014868', u'EOYAmt': u'104647760'}   \n",
       "3                                                 NaN   \n",
       "4                                                 NaN   \n",
       "\n",
       "                      TaxExemptBondLiabilities GovernmentGrantsAmt  \\\n",
       "0                                          NaN             7932282   \n",
       "1                                          NaN               42442   \n",
       "2  {u'BOY': u'77905017', u'EOY': u'109371484'}            77323736   \n",
       "3                                          NaN              237183   \n",
       "4                                          NaN               94663   \n",
       "\n",
       "  GovernmentGrants ElectionOfBoardMembersInd ElectionOfBoardMembers  \\\n",
       "0          7295185                         1                      1   \n",
       "1           129755                     false                  false   \n",
       "2         57810755                     false                  false   \n",
       "3           318383                         1                      1   \n",
       "4           113548                         0                      0   \n",
       "\n",
       "  CYTotalFundraisingExpenseAmt TotalFundrsngExpCurrentYear  \n",
       "0                       554608                      358835  \n",
       "1                       415743                      363139  \n",
       "2                      3474538                     2402953  \n",
       "3                       181290                      255685  \n",
       "4                       150504                      103085  "
      ]
     },
     "execution_count": 552,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "print len(df_all[cols].groupby('EIN').agg('first'))\n",
    "governance = df_all[cols].groupby('EIN').agg('first')\n",
    "print len(governance)\n",
    "governance = governance.reset_index()\n",
    "governance[:5]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 449,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "FY2014    5225\n",
       "FY2015    1578\n",
       "FY2013     197\n",
       "FY2012      55\n",
       "FY2011      41\n",
       "FY2010      37\n",
       "Name: FYE, dtype: int64"
      ]
     },
     "execution_count": 449,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "governance['FYE'].value_counts()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Create combined variables"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 262,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "['EIN', 'FYE', 'VotingMembersGoverningBodyCnt', 'NbrVotingMembersGoverningBody', 'VotingMembersIndependentCnt', 'NbrIndependentVotingMembers', 'DelegationOfMgmtDutiesInd', 'DelegationOfManagementDuties', 'Form990ProvidedToGvrnBodyInd', 'Form990ProvidedToGoverningBody', 'AuditCommitteeInd', 'AuditCommittee', 'perm_rest_assets', 'temp_rest_assets', 'net_assets', 'donor_restrictions', 'TaxExemptBondsInd', 'TaxExemptBonds', 'NetUnrelatedBusTxblIncmAmt', 'NetUnrelatedBusinessTxblIncome', 'FSAuditedInd', 'FSAudited', 'FederalGrantAuditPerformedInd', 'FederalGrantAuditPerformed', 'FederalGrantAuditRequiredInd', 'FederalGrantAuditRequired', 'IndependentAuditFinclStmtInd', 'IndependentAuditFinancialStmt', 'FeesForServicesAccounting_binary']\n"
     ]
    }
   ],
   "source": [
    "print cols"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "##### Independent Directors"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 450,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "count   7133.00\n",
       "mean      21.45\n",
       "std       34.58\n",
       "min        0.00\n",
       "25%       11.00\n",
       "50%       17.00\n",
       "75%       26.00\n",
       "max     2500.00\n",
       "Name: independent_directors_num, dtype: float64"
      ]
     },
     "execution_count": 450,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "governance['independent_directors_num'] = governance['VotingMembersIndependentCnt']\n",
    "governance['independent_directors_num'] = np.where(governance['independent_directors_num'].isnull(),\n",
    "                                          governance['NbrIndependentVotingMembers'], governance['independent_directors_num'])\n",
    "print len(governance[governance['independent_directors_num'].isnull()])\n",
    "#print governance['independent_directors_num'].value_counts()\n",
    "governance['independent_directors_num'] = governance['independent_directors_num'].astype('int')\n",
    "governance['independent_directors_num'].describe()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 451,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>EIN</th>\n",
       "      <th>FYE</th>\n",
       "      <th>VotingMembersGoverningBodyCnt</th>\n",
       "      <th>NbrVotingMembersGoverningBody</th>\n",
       "      <th>VotingMembersIndependentCnt</th>\n",
       "      <th>NbrIndependentVotingMembers</th>\n",
       "      <th>DelegationOfMgmtDutiesInd</th>\n",
       "      <th>DelegationOfManagementDuties</th>\n",
       "      <th>Form990ProvidedToGvrnBodyInd</th>\n",
       "      <th>Form990ProvidedToGoverningBody</th>\n",
       "      <th>AuditCommitteeInd</th>\n",
       "      <th>AuditCommittee</th>\n",
       "      <th>perm_rest_assets</th>\n",
       "      <th>temp_rest_assets</th>\n",
       "      <th>net_assets</th>\n",
       "      <th>donor_restrictions</th>\n",
       "      <th>TaxExemptBondsInd</th>\n",
       "      <th>TaxExemptBonds</th>\n",
       "      <th>NetUnrelatedBusTxblIncmAmt</th>\n",
       "      <th>NetUnrelatedBusinessTxblIncome</th>\n",
       "      <th>FSAuditedInd</th>\n",
       "      <th>FSAudited</th>\n",
       "      <th>FederalGrantAuditPerformedInd</th>\n",
       "      <th>FederalGrantAuditPerformed</th>\n",
       "      <th>FederalGrantAuditRequiredInd</th>\n",
       "      <th>FederalGrantAuditRequired</th>\n",
       "      <th>IndependentAuditFinclStmtInd</th>\n",
       "      <th>IndependentAuditFinancialStmt</th>\n",
       "      <th>FeesForServicesAccounting_binary</th>\n",
       "      <th>FamilyOrBusinessRlnInd</th>\n",
       "      <th>FamilyOrBusinessRelationship</th>\n",
       "      <th>TaxExemptBondLiabilitiesGrp</th>\n",
       "      <th>TaxExemptBondLiabilities</th>\n",
       "      <th>GovernmentGrantsAmt</th>\n",
       "      <th>GovernmentGrants</th>\n",
       "      <th>ElectionOfBoardMembersInd</th>\n",
       "      <th>ElectionOfBoardMembers</th>\n",
       "      <th>independent_directors_num</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>35</th>\n",
       "      <td>010679337</td>\n",
       "      <td>FY2013</td>\n",
       "      <td>NaN</td>\n",
       "      <td>11</td>\n",
       "      <td>NaN</td>\n",
       "      <td>10</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>1</td>\n",
       "      <td>NaN</td>\n",
       "      <td>1</td>\n",
       "      <td>111972495</td>\n",
       "      <td>584038945</td>\n",
       "      <td>1137256059</td>\n",
       "      <td>0.61</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>30871</td>\n",
       "      <td>NaN</td>\n",
       "      <td>1</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>{u'#text': u'0', u'@referenceDocumentId': u'RetDoc1039900001'}</td>\n",
       "      <td>1</td>\n",
       "      <td>NaN</td>\n",
       "      <td>1</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>237833</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0</td>\n",
       "      <td>10</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>65</th>\n",
       "      <td>026015642</td>\n",
       "      <td>FY2012</td>\n",
       "      <td>NaN</td>\n",
       "      <td>24</td>\n",
       "      <td>NaN</td>\n",
       "      <td>24</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>1</td>\n",
       "      <td>NaN</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>243648</td>\n",
       "      <td>1047304</td>\n",
       "      <td>0.23</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>-34607</td>\n",
       "      <td>NaN</td>\n",
       "      <td>1</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>{u'#text': u'1', u'@referenceDocumentId': u'RetDoc1040000001'}</td>\n",
       "      <td>0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>1</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0</td>\n",
       "      <td>24</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "          EIN     FYE VotingMembersGoverningBodyCnt  \\\n",
       "35  010679337  FY2013                           NaN   \n",
       "65  026015642  FY2012                           NaN   \n",
       "\n",
       "   NbrVotingMembersGoverningBody VotingMembersIndependentCnt  \\\n",
       "35                            11                         NaN   \n",
       "65                            24                         NaN   \n",
       "\n",
       "   NbrIndependentVotingMembers DelegationOfMgmtDutiesInd  \\\n",
       "35                          10                       NaN   \n",
       "65                          24                       NaN   \n",
       "\n",
       "   DelegationOfManagementDuties Form990ProvidedToGvrnBodyInd  \\\n",
       "35                            0                          NaN   \n",
       "65                            0                          NaN   \n",
       "\n",
       "   Form990ProvidedToGoverningBody AuditCommitteeInd AuditCommittee  \\\n",
       "35                              1               NaN              1   \n",
       "65                              1               NaN              1   \n",
       "\n",
       "    perm_rest_assets  temp_rest_assets  net_assets  donor_restrictions  \\\n",
       "35         111972495         584038945  1137256059                0.61   \n",
       "65                 0            243648     1047304                0.23   \n",
       "\n",
       "   TaxExemptBondsInd TaxExemptBonds NetUnrelatedBusTxblIncmAmt  \\\n",
       "35               NaN              0                        NaN   \n",
       "65               NaN              0                        NaN   \n",
       "\n",
       "   NetUnrelatedBusinessTxblIncome FSAuditedInd FSAudited  \\\n",
       "35                          30871          NaN         1   \n",
       "65                         -34607          NaN         1   \n",
       "\n",
       "   FederalGrantAuditPerformedInd FederalGrantAuditPerformed  \\\n",
       "35                           NaN                        NaN   \n",
       "65                           NaN                        NaN   \n",
       "\n",
       "   FederalGrantAuditRequiredInd FederalGrantAuditRequired  \\\n",
       "35                          NaN                         0   \n",
       "65                          NaN                         0   \n",
       "\n",
       "   IndependentAuditFinclStmtInd  \\\n",
       "35                          NaN   \n",
       "65                          NaN   \n",
       "\n",
       "                                     IndependentAuditFinancialStmt  \\\n",
       "35  {u'#text': u'0', u'@referenceDocumentId': u'RetDoc1039900001'}   \n",
       "65  {u'#text': u'1', u'@referenceDocumentId': u'RetDoc1040000001'}   \n",
       "\n",
       "    FeesForServicesAccounting_binary FamilyOrBusinessRlnInd  \\\n",
       "35                                 1                    NaN   \n",
       "65                                 0                    NaN   \n",
       "\n",
       "   FamilyOrBusinessRelationship TaxExemptBondLiabilitiesGrp  \\\n",
       "35                            1                         NaN   \n",
       "65                            1                         NaN   \n",
       "\n",
       "   TaxExemptBondLiabilities GovernmentGrantsAmt GovernmentGrants  \\\n",
       "35                      NaN                 NaN           237833   \n",
       "65                      NaN                 NaN              NaN   \n",
       "\n",
       "   ElectionOfBoardMembersInd ElectionOfBoardMembers  independent_directors_num  \n",
       "35                       NaN                      0                         10  \n",
       "65                       NaN                      0                         24  "
      ]
     },
     "execution_count": 451,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "governance[governance['VotingMembersIndependentCnt'].isnull()][:2]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "##### Voting Directors"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 452,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0\n"
     ]
    }
   ],
   "source": [
    "governance['voting_directors'] = governance['VotingMembersGoverningBodyCnt']\n",
    "governance['voting_directors'] = np.where(governance['voting_directors'].isnull(),\n",
    "                                          governance['NbrVotingMembersGoverningBody'], governance['voting_directors'])\n",
    "print len(governance[governance['voting_directors'].isnull()])\n",
    "#print governance['voting_directors'].value_counts()\n",
    "governance['voting_directors'] = governance['voting_directors'].astype('int')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "##### Percent Independent Directors"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 457,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "count   7129.00\n",
       "mean       0.97\n",
       "std        1.09\n",
       "min        0.00\n",
       "25%        0.96\n",
       "50%        1.00\n",
       "75%        1.00\n",
       "max       92.59\n",
       "Name: independent_directors_pct, dtype: float64"
      ]
     },
     "execution_count": 457,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "governance['independent_directors_pct'] = governance['independent_directors_num']/governance['voting_directors']\n",
    "governance['independent_directors_pct'].describe()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 456,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>EIN</th>\n",
       "      <th>FYE</th>\n",
       "      <th>VotingMembersGoverningBodyCnt</th>\n",
       "      <th>NbrVotingMembersGoverningBody</th>\n",
       "      <th>VotingMembersIndependentCnt</th>\n",
       "      <th>NbrIndependentVotingMembers</th>\n",
       "      <th>DelegationOfMgmtDutiesInd</th>\n",
       "      <th>DelegationOfManagementDuties</th>\n",
       "      <th>Form990ProvidedToGvrnBodyInd</th>\n",
       "      <th>Form990ProvidedToGoverningBody</th>\n",
       "      <th>AuditCommitteeInd</th>\n",
       "      <th>AuditCommittee</th>\n",
       "      <th>perm_rest_assets</th>\n",
       "      <th>temp_rest_assets</th>\n",
       "      <th>net_assets</th>\n",
       "      <th>donor_restrictions</th>\n",
       "      <th>TaxExemptBondsInd</th>\n",
       "      <th>TaxExemptBonds</th>\n",
       "      <th>NetUnrelatedBusTxblIncmAmt</th>\n",
       "      <th>NetUnrelatedBusinessTxblIncome</th>\n",
       "      <th>FSAuditedInd</th>\n",
       "      <th>FSAudited</th>\n",
       "      <th>FederalGrantAuditPerformedInd</th>\n",
       "      <th>FederalGrantAuditPerformed</th>\n",
       "      <th>FederalGrantAuditRequiredInd</th>\n",
       "      <th>FederalGrantAuditRequired</th>\n",
       "      <th>IndependentAuditFinclStmtInd</th>\n",
       "      <th>IndependentAuditFinancialStmt</th>\n",
       "      <th>FeesForServicesAccounting_binary</th>\n",
       "      <th>FamilyOrBusinessRlnInd</th>\n",
       "      <th>FamilyOrBusinessRelationship</th>\n",
       "      <th>TaxExemptBondLiabilitiesGrp</th>\n",
       "      <th>TaxExemptBondLiabilities</th>\n",
       "      <th>GovernmentGrantsAmt</th>\n",
       "      <th>GovernmentGrants</th>\n",
       "      <th>ElectionOfBoardMembersInd</th>\n",
       "      <th>ElectionOfBoardMembers</th>\n",
       "      <th>independent_directors_num</th>\n",
       "      <th>voting_directors</th>\n",
       "      <th>independent_directors_pct</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "Empty DataFrame\n",
       "Columns: [EIN, FYE, VotingMembersGoverningBodyCnt, NbrVotingMembersGoverningBody, VotingMembersIndependentCnt, NbrIndependentVotingMembers, DelegationOfMgmtDutiesInd, DelegationOfManagementDuties, Form990ProvidedToGvrnBodyInd, Form990ProvidedToGoverningBody, AuditCommitteeInd, AuditCommittee, perm_rest_assets, temp_rest_assets, net_assets, donor_restrictions, TaxExemptBondsInd, TaxExemptBonds, NetUnrelatedBusTxblIncmAmt, NetUnrelatedBusinessTxblIncome, FSAuditedInd, FSAudited, FederalGrantAuditPerformedInd, FederalGrantAuditPerformed, FederalGrantAuditRequiredInd, FederalGrantAuditRequired, IndependentAuditFinclStmtInd, IndependentAuditFinancialStmt, FeesForServicesAccounting_binary, FamilyOrBusinessRlnInd, FamilyOrBusinessRelationship, TaxExemptBondLiabilitiesGrp, TaxExemptBondLiabilities, GovernmentGrantsAmt, GovernmentGrants, ElectionOfBoardMembersInd, ElectionOfBoardMembers, independent_directors_num, voting_directors, independent_directors_pct]\n",
       "Index: []"
      ]
     },
     "execution_count": 456,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "governance[governance['independent_directors_pct'].isnull()]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 458,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "count   7133.00\n",
       "mean       0.97\n",
       "std        1.09\n",
       "min        0.00\n",
       "25%        0.96\n",
       "50%        1.00\n",
       "75%        1.00\n",
       "max       92.59\n",
       "Name: independent_directors_pct, dtype: float64"
      ]
     },
     "execution_count": 458,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "governance['independent_directors_pct'] = np.where(governance['independent_directors_pct'].isnull(), 0, \n",
    "                                                   governance['independent_directors_pct'])\n",
    "governance['independent_directors_pct'].describe()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "<br>Second version -- none over 100%"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 461,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "11\n",
      "0\n",
      "count   7133.00\n",
      "mean       0.95\n",
      "std        0.13\n",
      "min        0.00\n",
      "25%        0.96\n",
      "50%        1.00\n",
      "75%        1.00\n",
      "max        1.00\n",
      "Name: independent_directors_pct_v2, dtype: float64\n"
     ]
    }
   ],
   "source": [
    "print len(governance[governance['independent_directors_pct']>1])\n",
    "governance['independent_directors_pct_v2'] = np.where(governance['independent_directors_pct']>1, 1, \n",
    "                                               governance['independent_directors_pct'])\n",
    "print len(governance[governance['independent_directors_pct_v2']>1])\n",
    "print governance['independent_directors_pct_v2'].describe()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "##### Outsourced Management"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 463,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0\n",
      "0        4365\n",
      "false    2636\n",
      "1          92\n",
      "true       40\n",
      "Name: outsourced_mgt, dtype: int64\n"
     ]
    }
   ],
   "source": [
    "governance['outsourced_mgt'] = governance['DelegationOfMgmtDutiesInd']\n",
    "governance['outsourced_mgt'] = np.where(governance['outsourced_mgt'].isnull(),\n",
    "                                          governance['DelegationOfManagementDuties'], governance['outsourced_mgt'])\n",
    "print len(governance[governance['outsourced_mgt'].isnull()])\n",
    "print governance['outsourced_mgt'].value_counts()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 464,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0    4365\n",
      "0    2636\n",
      "1      92\n",
      "1      40\n",
      "Name: outsourced_mgt, dtype: int64\n"
     ]
    }
   ],
   "source": [
    "governance['outsourced_mgt'] = np.where(governance['outsourced_mgt']=='true', 1, governance['outsourced_mgt'])\n",
    "governance['outsourced_mgt'] = np.where(governance['outsourced_mgt']=='false', 0, governance['outsourced_mgt'])\n",
    "print governance['outsourced_mgt'].value_counts()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 465,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0    7001\n",
      "1     132\n",
      "Name: outsourced_mgt, dtype: int64\n"
     ]
    }
   ],
   "source": [
    "governance['outsourced_mgt']=governance['outsourced_mgt'].astype('int')\n",
    "print governance['outsourced_mgt'].value_counts()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "##### IRS 990 Review"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 466,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0\n",
      "1        4088\n",
      "true     2352\n",
      "0         369\n",
      "false     324\n",
      "Name: 990_review, dtype: int64\n"
     ]
    }
   ],
   "source": [
    "governance['990_review'] = governance['Form990ProvidedToGvrnBodyInd']\n",
    "governance['990_review'] = np.where(governance['990_review'].isnull(),\n",
    "                                          governance['Form990ProvidedToGoverningBody'], governance['990_review'])\n",
    "print len(governance[governance['990_review'].isnull()])\n",
    "print governance['990_review'].value_counts()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 467,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "1    6440\n",
      "0     693\n",
      "Name: 990_review, dtype: int64\n"
     ]
    }
   ],
   "source": [
    "governance['990_review'] = np.where(governance['990_review']=='true', 1, governance['990_review'])\n",
    "governance['990_review'] = np.where(governance['990_review']=='false', 0, governance['990_review'])\n",
    "governance['990_review'] = governance['990_review'].astype('int')\n",
    "print governance['990_review'].value_counts()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "##### Audit Committee\n",
    "I'M ASSIGNING A VALUE OF '0' TO THE 138 CASES MISSING VALUES"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 468,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "138\n",
      "1        4205\n",
      "true     2454\n",
      "0         189\n",
      "false     147\n",
      "Name: audit_committee, dtype: int64\n"
     ]
    }
   ],
   "source": [
    "governance['audit_committee'] = governance['AuditCommitteeInd']\n",
    "governance['audit_committee'] = np.where(governance['audit_committee'].isnull(),\n",
    "                                          governance['AuditCommittee'], governance['audit_committee'])\n",
    "print len(governance[governance['audit_committee'].isnull()])\n",
    "print governance['audit_committee'].value_counts()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 469,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "1    6659\n",
      "0     474\n",
      "Name: audit_committee, dtype: int64\n"
     ]
    }
   ],
   "source": [
    "governance['audit_committee'] = np.where(governance['audit_committee']=='true', 1, governance['audit_committee'])\n",
    "governance['audit_committee'] = np.where(governance['audit_committee']=='false', 0, governance['audit_committee'])\n",
    "governance['audit_committee'] = np.where(governance['audit_committee'].isnull(), 0, governance['audit_committee'])\n",
    "governance['audit_committee']=governance['audit_committee'].astype('int')\n",
    "print governance['audit_committee'].value_counts()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 470,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>count</th>\n",
       "      <th>mean</th>\n",
       "      <th>std</th>\n",
       "      <th>min</th>\n",
       "      <th>25%</th>\n",
       "      <th>50%</th>\n",
       "      <th>75%</th>\n",
       "      <th>max</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>perm_rest_assets</th>\n",
       "      <td>7133.00</td>\n",
       "      <td>5398758.08</td>\n",
       "      <td>34888704.49</td>\n",
       "      <td>0.00</td>\n",
       "      <td>0.00</td>\n",
       "      <td>0.00</td>\n",
       "      <td>1033359.00</td>\n",
       "      <td>1097020543.00</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>temp_rest_assets</th>\n",
       "      <td>7133.00</td>\n",
       "      <td>7347199.39</td>\n",
       "      <td>40974295.28</td>\n",
       "      <td>-1417569.00</td>\n",
       "      <td>119999.00</td>\n",
       "      <td>660803.00</td>\n",
       "      <td>2879306.00</td>\n",
       "      <td>1473750677.00</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>net_assets</th>\n",
       "      <td>7133.00</td>\n",
       "      <td>31409548.61</td>\n",
       "      <td>175347462.23</td>\n",
       "      <td>-139954787.00</td>\n",
       "      <td>1766750.00</td>\n",
       "      <td>5157328.00</td>\n",
       "      <td>15828642.00</td>\n",
       "      <td>7982613662.00</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>donor_restrictions</th>\n",
       "      <td>7133.00</td>\n",
       "      <td>inf</td>\n",
       "      <td>nan</td>\n",
       "      <td>-156.44</td>\n",
       "      <td>0.06</td>\n",
       "      <td>0.24</td>\n",
       "      <td>0.55</td>\n",
       "      <td>inf</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>FeesForServicesAccounting_binary</th>\n",
       "      <td>7133.00</td>\n",
       "      <td>0.83</td>\n",
       "      <td>0.38</td>\n",
       "      <td>0.00</td>\n",
       "      <td>1.00</td>\n",
       "      <td>1.00</td>\n",
       "      <td>1.00</td>\n",
       "      <td>1.00</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>independent_directors_num</th>\n",
       "      <td>7133.00</td>\n",
       "      <td>21.45</td>\n",
       "      <td>34.58</td>\n",
       "      <td>0.00</td>\n",
       "      <td>11.00</td>\n",
       "      <td>17.00</td>\n",
       "      <td>26.00</td>\n",
       "      <td>2500.00</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>voting_directors</th>\n",
       "      <td>7133.00</td>\n",
       "      <td>21.70</td>\n",
       "      <td>18.27</td>\n",
       "      <td>0.00</td>\n",
       "      <td>12.00</td>\n",
       "      <td>18.00</td>\n",
       "      <td>26.00</td>\n",
       "      <td>434.00</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>independent_directors_pct</th>\n",
       "      <td>7133.00</td>\n",
       "      <td>0.97</td>\n",
       "      <td>1.09</td>\n",
       "      <td>0.00</td>\n",
       "      <td>0.96</td>\n",
       "      <td>1.00</td>\n",
       "      <td>1.00</td>\n",
       "      <td>92.59</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>independent_directors_pct_v2</th>\n",
       "      <td>7133.00</td>\n",
       "      <td>0.95</td>\n",
       "      <td>0.13</td>\n",
       "      <td>0.00</td>\n",
       "      <td>0.96</td>\n",
       "      <td>1.00</td>\n",
       "      <td>1.00</td>\n",
       "      <td>1.00</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>outsourced_mgt</th>\n",
       "      <td>7133.00</td>\n",
       "      <td>0.02</td>\n",
       "      <td>0.13</td>\n",
       "      <td>0.00</td>\n",
       "      <td>0.00</td>\n",
       "      <td>0.00</td>\n",
       "      <td>0.00</td>\n",
       "      <td>1.00</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>990_review</th>\n",
       "      <td>7133.00</td>\n",
       "      <td>0.90</td>\n",
       "      <td>0.30</td>\n",
       "      <td>0.00</td>\n",
       "      <td>1.00</td>\n",
       "      <td>1.00</td>\n",
       "      <td>1.00</td>\n",
       "      <td>1.00</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>audit_committee</th>\n",
       "      <td>7133.00</td>\n",
       "      <td>0.93</td>\n",
       "      <td>0.25</td>\n",
       "      <td>0.00</td>\n",
       "      <td>1.00</td>\n",
       "      <td>1.00</td>\n",
       "      <td>1.00</td>\n",
       "      <td>1.00</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                                   count        mean          std  \\\n",
       "perm_rest_assets                 7133.00  5398758.08  34888704.49   \n",
       "temp_rest_assets                 7133.00  7347199.39  40974295.28   \n",
       "net_assets                       7133.00 31409548.61 175347462.23   \n",
       "donor_restrictions               7133.00         inf          nan   \n",
       "FeesForServicesAccounting_binary 7133.00        0.83         0.38   \n",
       "independent_directors_num        7133.00       21.45        34.58   \n",
       "voting_directors                 7133.00       21.70        18.27   \n",
       "independent_directors_pct        7133.00        0.97         1.09   \n",
       "independent_directors_pct_v2     7133.00        0.95         0.13   \n",
       "outsourced_mgt                   7133.00        0.02         0.13   \n",
       "990_review                       7133.00        0.90         0.30   \n",
       "audit_committee                  7133.00        0.93         0.25   \n",
       "\n",
       "                                           min        25%        50%  \\\n",
       "perm_rest_assets                          0.00       0.00       0.00   \n",
       "temp_rest_assets                   -1417569.00  119999.00  660803.00   \n",
       "net_assets                       -139954787.00 1766750.00 5157328.00   \n",
       "donor_restrictions                     -156.44       0.06       0.24   \n",
       "FeesForServicesAccounting_binary          0.00       1.00       1.00   \n",
       "independent_directors_num                 0.00      11.00      17.00   \n",
       "voting_directors                          0.00      12.00      18.00   \n",
       "independent_directors_pct                 0.00       0.96       1.00   \n",
       "independent_directors_pct_v2              0.00       0.96       1.00   \n",
       "outsourced_mgt                            0.00       0.00       0.00   \n",
       "990_review                                0.00       1.00       1.00   \n",
       "audit_committee                           0.00       1.00       1.00   \n",
       "\n",
       "                                         75%           max  \n",
       "perm_rest_assets                  1033359.00 1097020543.00  \n",
       "temp_rest_assets                  2879306.00 1473750677.00  \n",
       "net_assets                       15828642.00 7982613662.00  \n",
       "donor_restrictions                      0.55           inf  \n",
       "FeesForServicesAccounting_binary        1.00          1.00  \n",
       "independent_directors_num              26.00       2500.00  \n",
       "voting_directors                       26.00        434.00  \n",
       "independent_directors_pct               1.00         92.59  \n",
       "independent_directors_pct_v2            1.00          1.00  \n",
       "outsourced_mgt                          0.00          1.00  \n",
       "990_review                              1.00          1.00  \n",
       "audit_committee                         1.00          1.00  "
      ]
     },
     "execution_count": 470,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "governance.describe().T"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Donor Restrictions\n",
    "- NOTE THAT I'M NOT DOING IT THIS WAY ANY MORE -- NOW IT IS DONE ABOVE IN *df_all* BEFORE COLLAPSING THE DATA\n",
    "- ALSO NOTE THAT THE 990 DATA MUST HAVE SOME ERRORS BECAUSE THERE ARE A FEW CASES WITH VERY HIGH VALUES ON *DONOR_RESTRICTIONS*"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 471,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "-156.442553191\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "inf"
      ]
     },
     "execution_count": 471,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "#print governance['donor_restrictions'].min()\n",
    "#governance['donor_restrictions'].max()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 473,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "1\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>EIN</th>\n",
       "      <th>FYE</th>\n",
       "      <th>VotingMembersGoverningBodyCnt</th>\n",
       "      <th>NbrVotingMembersGoverningBody</th>\n",
       "      <th>VotingMembersIndependentCnt</th>\n",
       "      <th>NbrIndependentVotingMembers</th>\n",
       "      <th>DelegationOfMgmtDutiesInd</th>\n",
       "      <th>DelegationOfManagementDuties</th>\n",
       "      <th>Form990ProvidedToGvrnBodyInd</th>\n",
       "      <th>Form990ProvidedToGoverningBody</th>\n",
       "      <th>AuditCommitteeInd</th>\n",
       "      <th>AuditCommittee</th>\n",
       "      <th>perm_rest_assets</th>\n",
       "      <th>temp_rest_assets</th>\n",
       "      <th>net_assets</th>\n",
       "      <th>donor_restrictions</th>\n",
       "      <th>TaxExemptBondsInd</th>\n",
       "      <th>TaxExemptBonds</th>\n",
       "      <th>NetUnrelatedBusTxblIncmAmt</th>\n",
       "      <th>NetUnrelatedBusinessTxblIncome</th>\n",
       "      <th>FSAuditedInd</th>\n",
       "      <th>FSAudited</th>\n",
       "      <th>FederalGrantAuditPerformedInd</th>\n",
       "      <th>FederalGrantAuditPerformed</th>\n",
       "      <th>FederalGrantAuditRequiredInd</th>\n",
       "      <th>FederalGrantAuditRequired</th>\n",
       "      <th>IndependentAuditFinclStmtInd</th>\n",
       "      <th>IndependentAuditFinancialStmt</th>\n",
       "      <th>FeesForServicesAccounting_binary</th>\n",
       "      <th>FamilyOrBusinessRlnInd</th>\n",
       "      <th>FamilyOrBusinessRelationship</th>\n",
       "      <th>TaxExemptBondLiabilitiesGrp</th>\n",
       "      <th>TaxExemptBondLiabilities</th>\n",
       "      <th>GovernmentGrantsAmt</th>\n",
       "      <th>GovernmentGrants</th>\n",
       "      <th>ElectionOfBoardMembersInd</th>\n",
       "      <th>ElectionOfBoardMembers</th>\n",
       "      <th>independent_directors_num</th>\n",
       "      <th>voting_directors</th>\n",
       "      <th>independent_directors_pct</th>\n",
       "      <th>independent_directors_pct_v2</th>\n",
       "      <th>outsourced_mgt</th>\n",
       "      <th>990_review</th>\n",
       "      <th>audit_committee</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>5939</th>\n",
       "      <td>840516736</td>\n",
       "      <td>FY2014</td>\n",
       "      <td>10</td>\n",
       "      <td>10</td>\n",
       "      <td>9</td>\n",
       "      <td>9</td>\n",
       "      <td>false</td>\n",
       "      <td>false</td>\n",
       "      <td>true</td>\n",
       "      <td>true</td>\n",
       "      <td>true</td>\n",
       "      <td>true</td>\n",
       "      <td>260654</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>inf</td>\n",
       "      <td>false</td>\n",
       "      <td>false</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>true</td>\n",
       "      <td>true</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>false</td>\n",
       "      <td>false</td>\n",
       "      <td>{u'#text': u'true', u'@referenceDocumentId': u'00000004', u'@referenceDocumentName': u'IRS990ScheduleD'}</td>\n",
       "      <td>{u'#text': u'true', u'@referenceDocumentId': u'00000004', u'@referenceDocumentName': u'IRS990ScheduleD'}</td>\n",
       "      <td>0</td>\n",
       "      <td>true</td>\n",
       "      <td>false</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>5644</td>\n",
       "      <td>false</td>\n",
       "      <td>false</td>\n",
       "      <td>9</td>\n",
       "      <td>10</td>\n",
       "      <td>0.90</td>\n",
       "      <td>0.90</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "            EIN     FYE VotingMembersGoverningBodyCnt  \\\n",
       "5939  840516736  FY2014                            10   \n",
       "\n",
       "     NbrVotingMembersGoverningBody VotingMembersIndependentCnt  \\\n",
       "5939                            10                           9   \n",
       "\n",
       "     NbrIndependentVotingMembers DelegationOfMgmtDutiesInd  \\\n",
       "5939                           9                     false   \n",
       "\n",
       "     DelegationOfManagementDuties Form990ProvidedToGvrnBodyInd  \\\n",
       "5939                        false                         true   \n",
       "\n",
       "     Form990ProvidedToGoverningBody AuditCommitteeInd AuditCommittee  \\\n",
       "5939                           true              true           true   \n",
       "\n",
       "      perm_rest_assets  temp_rest_assets  net_assets  donor_restrictions  \\\n",
       "5939            260654                 0           0                 inf   \n",
       "\n",
       "     TaxExemptBondsInd TaxExemptBonds NetUnrelatedBusTxblIncmAmt  \\\n",
       "5939             false          false                        NaN   \n",
       "\n",
       "     NetUnrelatedBusinessTxblIncome FSAuditedInd FSAudited  \\\n",
       "5939                            NaN         true      true   \n",
       "\n",
       "     FederalGrantAuditPerformedInd FederalGrantAuditPerformed  \\\n",
       "5939                           NaN                        NaN   \n",
       "\n",
       "     FederalGrantAuditRequiredInd FederalGrantAuditRequired  \\\n",
       "5939                        false                     false   \n",
       "\n",
       "                                                                                  IndependentAuditFinclStmtInd  \\\n",
       "5939  {u'#text': u'true', u'@referenceDocumentId': u'00000004', u'@referenceDocumentName': u'IRS990ScheduleD'}   \n",
       "\n",
       "                                                                                 IndependentAuditFinancialStmt  \\\n",
       "5939  {u'#text': u'true', u'@referenceDocumentId': u'00000004', u'@referenceDocumentName': u'IRS990ScheduleD'}   \n",
       "\n",
       "      FeesForServicesAccounting_binary FamilyOrBusinessRlnInd  \\\n",
       "5939                                 0                   true   \n",
       "\n",
       "     FamilyOrBusinessRelationship TaxExemptBondLiabilitiesGrp  \\\n",
       "5939                        false                         NaN   \n",
       "\n",
       "     TaxExemptBondLiabilities GovernmentGrantsAmt GovernmentGrants  \\\n",
       "5939                      NaN                 NaN             5644   \n",
       "\n",
       "     ElectionOfBoardMembersInd ElectionOfBoardMembers  \\\n",
       "5939                     false                  false   \n",
       "\n",
       "      independent_directors_num  voting_directors  independent_directors_pct  \\\n",
       "5939                          9                10                       0.90   \n",
       "\n",
       "      independent_directors_pct_v2  outsourced_mgt  990_review  \\\n",
       "5939                          0.90               0           1   \n",
       "\n",
       "      audit_committee  \n",
       "5939                1  "
      ]
     },
     "execution_count": 473,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "print len(governance[governance['donor_restrictions']>100])\n",
    "governance[governance['donor_restrictions']>100]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 472,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "6\n"
     ]
    }
   ],
   "source": [
    "print len(governance[governance['donor_restrictions']>10])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 290,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>EIN</th>\n",
       "      <th>FYE</th>\n",
       "      <th>VotingMembersGoverningBodyCnt</th>\n",
       "      <th>NbrVotingMembersGoverningBody</th>\n",
       "      <th>VotingMembersIndependentCnt</th>\n",
       "      <th>NbrIndependentVotingMembers</th>\n",
       "      <th>DelegationOfMgmtDutiesInd</th>\n",
       "      <th>DelegationOfManagementDuties</th>\n",
       "      <th>Form990ProvidedToGvrnBodyInd</th>\n",
       "      <th>Form990ProvidedToGoverningBody</th>\n",
       "      <th>AuditCommitteeInd</th>\n",
       "      <th>AuditCommittee</th>\n",
       "      <th>perm_rest_assets</th>\n",
       "      <th>temp_rest_assets</th>\n",
       "      <th>net_assets</th>\n",
       "      <th>donor_restrictions</th>\n",
       "      <th>TaxExemptBondsInd</th>\n",
       "      <th>TaxExemptBonds</th>\n",
       "      <th>NetUnrelatedBusTxblIncmAmt</th>\n",
       "      <th>NetUnrelatedBusinessTxblIncome</th>\n",
       "      <th>FSAuditedInd</th>\n",
       "      <th>FSAudited</th>\n",
       "      <th>FederalGrantAuditPerformedInd</th>\n",
       "      <th>FederalGrantAuditPerformed</th>\n",
       "      <th>FederalGrantAuditRequiredInd</th>\n",
       "      <th>FederalGrantAuditRequired</th>\n",
       "      <th>IndependentAuditFinclStmtInd</th>\n",
       "      <th>IndependentAuditFinancialStmt</th>\n",
       "      <th>FeesForServicesAccounting_binary</th>\n",
       "      <th>independent_directors_num</th>\n",
       "      <th>voting_directors</th>\n",
       "      <th>independent_directors_pct</th>\n",
       "      <th>outsourced_mgt</th>\n",
       "      <th>990_review</th>\n",
       "      <th>audit_committee</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>435</th>\n",
       "      <td>061343149</td>\n",
       "      <td>FY2015</td>\n",
       "      <td>30</td>\n",
       "      <td>24</td>\n",
       "      <td>28</td>\n",
       "      <td>22</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>38420</td>\n",
       "      <td>478</td>\n",
       "      <td>80.376569</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>{u'#text': u'1', u'@referenceDocumentId': u'RetDoc1040000001'}</td>\n",
       "      <td>{u'#text': u'1', u'@referenceDocumentId': u'RetDoc1040000001'}</td>\n",
       "      <td>1</td>\n",
       "      <td>28</td>\n",
       "      <td>30</td>\n",
       "      <td>0.933333</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3907</th>\n",
       "      <td>521591381</td>\n",
       "      <td>FY2014</td>\n",
       "      <td>13</td>\n",
       "      <td>14</td>\n",
       "      <td>12</td>\n",
       "      <td>13</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>1893106</td>\n",
       "      <td>137848</td>\n",
       "      <td>13.733286</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>-1009</td>\n",
       "      <td>880</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>{u'#text': u'1', u'@referenceDocumentId': u'RetDoc1040000001'}</td>\n",
       "      <td>{u'#text': u'1', u'@referenceDocumentId': u'RetDoc1040000001'}</td>\n",
       "      <td>1</td>\n",
       "      <td>12</td>\n",
       "      <td>13</td>\n",
       "      <td>0.923077</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4438</th>\n",
       "      <td>570761297</td>\n",
       "      <td>FY2015</td>\n",
       "      <td>24</td>\n",
       "      <td>23</td>\n",
       "      <td>21</td>\n",
       "      <td>23</td>\n",
       "      <td>false</td>\n",
       "      <td>false</td>\n",
       "      <td>false</td>\n",
       "      <td>true</td>\n",
       "      <td>true</td>\n",
       "      <td>true</td>\n",
       "      <td>224809</td>\n",
       "      <td>122436</td>\n",
       "      <td>21241</td>\n",
       "      <td>16.347865</td>\n",
       "      <td>false</td>\n",
       "      <td>false</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>true</td>\n",
       "      <td>false</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>false</td>\n",
       "      <td>false</td>\n",
       "      <td>{u'#text': u'true', u'@referenceDocumentId': u'RetDoc4'}</td>\n",
       "      <td>false</td>\n",
       "      <td>1</td>\n",
       "      <td>21</td>\n",
       "      <td>24</td>\n",
       "      <td>0.875000</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5358</th>\n",
       "      <td>730568096</td>\n",
       "      <td>FY2014</td>\n",
       "      <td>11</td>\n",
       "      <td>11</td>\n",
       "      <td>11</td>\n",
       "      <td>11</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>521821</td>\n",
       "      <td>1082185</td>\n",
       "      <td>63200</td>\n",
       "      <td>25.379842</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>NaN</td>\n",
       "      <td>false</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>{u'#text': u'0', u'@referenceDocumentId': u'RetDoc1040000001'}</td>\n",
       "      <td>{u'#text': u'0', u'@referenceDocumentId': u'RetDoc1040000001'}</td>\n",
       "      <td>1</td>\n",
       "      <td>11</td>\n",
       "      <td>11</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7045</th>\n",
       "      <td>954291515</td>\n",
       "      <td>FY2014</td>\n",
       "      <td>6</td>\n",
       "      <td>9</td>\n",
       "      <td>4</td>\n",
       "      <td>7</td>\n",
       "      <td>0</td>\n",
       "      <td>true</td>\n",
       "      <td>1</td>\n",
       "      <td>true</td>\n",
       "      <td>1</td>\n",
       "      <td>false</td>\n",
       "      <td>0</td>\n",
       "      <td>5844708</td>\n",
       "      <td>503140</td>\n",
       "      <td>11.616465</td>\n",
       "      <td>0</td>\n",
       "      <td>false</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>true</td>\n",
       "      <td>1</td>\n",
       "      <td>true</td>\n",
       "      <td>1</td>\n",
       "      <td>true</td>\n",
       "      <td>{u'#text': u'0', u'@referenceDocumentId': u'RetDoc1040000001'}</td>\n",
       "      <td>{u'#text': u'true', u'@referenceDocumentId': u'IRS990ScheduleD'}</td>\n",
       "      <td>0</td>\n",
       "      <td>4</td>\n",
       "      <td>6</td>\n",
       "      <td>0.666667</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "            EIN     FYE VotingMembersGoverningBodyCnt  \\\n",
       "435   061343149  FY2015                            30   \n",
       "3907  521591381  FY2014                            13   \n",
       "4438  570761297  FY2015                            24   \n",
       "5358  730568096  FY2014                            11   \n",
       "7045  954291515  FY2014                             6   \n",
       "\n",
       "     NbrVotingMembersGoverningBody VotingMembersIndependentCnt  \\\n",
       "435                             24                          28   \n",
       "3907                            14                          12   \n",
       "4438                            23                          21   \n",
       "5358                            11                          11   \n",
       "7045                             9                           4   \n",
       "\n",
       "     NbrIndependentVotingMembers DelegationOfMgmtDutiesInd  \\\n",
       "435                           22                         0   \n",
       "3907                          13                         0   \n",
       "4438                          23                     false   \n",
       "5358                          11                         0   \n",
       "7045                           7                         0   \n",
       "\n",
       "     DelegationOfManagementDuties Form990ProvidedToGvrnBodyInd  \\\n",
       "435                             0                            1   \n",
       "3907                            0                            1   \n",
       "4438                        false                        false   \n",
       "5358                            0                            1   \n",
       "7045                         true                            1   \n",
       "\n",
       "     Form990ProvidedToGoverningBody AuditCommitteeInd AuditCommittee  \\\n",
       "435                               1                 1              1   \n",
       "3907                              1                 1              1   \n",
       "4438                           true              true           true   \n",
       "5358                              1                 1              1   \n",
       "7045                           true                 1          false   \n",
       "\n",
       "      perm_rest_assets  temp_rest_assets  net_assets  donor_restrictions  \\\n",
       "435                  0             38420         478           80.376569   \n",
       "3907                 0           1893106      137848           13.733286   \n",
       "4438            224809            122436       21241           16.347865   \n",
       "5358            521821           1082185       63200           25.379842   \n",
       "7045                 0           5844708      503140           11.616465   \n",
       "\n",
       "     TaxExemptBondsInd TaxExemptBonds NetUnrelatedBusTxblIncmAmt  \\\n",
       "435                  0              0                          0   \n",
       "3907                 0              0                      -1009   \n",
       "4438             false          false                          0   \n",
       "5358                 0              0                          0   \n",
       "7045                 0          false                          0   \n",
       "\n",
       "     NetUnrelatedBusinessTxblIncome FSAuditedInd FSAudited  \\\n",
       "435                               0            1         1   \n",
       "3907                            880            1         1   \n",
       "4438                              0         true     false   \n",
       "5358                              0            1         1   \n",
       "7045                              0            1      true   \n",
       "\n",
       "     FederalGrantAuditPerformedInd FederalGrantAuditPerformed  \\\n",
       "435                            NaN                        NaN   \n",
       "3907                           NaN                        NaN   \n",
       "4438                           NaN                        NaN   \n",
       "5358                           NaN                      false   \n",
       "7045                             1                       true   \n",
       "\n",
       "     FederalGrantAuditRequiredInd FederalGrantAuditRequired  \\\n",
       "435                             0                         0   \n",
       "3907                            0                         0   \n",
       "4438                        false                     false   \n",
       "5358                            0                         0   \n",
       "7045                            1                      true   \n",
       "\n",
       "                                        IndependentAuditFinclStmtInd  \\\n",
       "435   {u'#text': u'1', u'@referenceDocumentId': u'RetDoc1040000001'}   \n",
       "3907  {u'#text': u'1', u'@referenceDocumentId': u'RetDoc1040000001'}   \n",
       "4438        {u'#text': u'true', u'@referenceDocumentId': u'RetDoc4'}   \n",
       "5358  {u'#text': u'0', u'@referenceDocumentId': u'RetDoc1040000001'}   \n",
       "7045  {u'#text': u'0', u'@referenceDocumentId': u'RetDoc1040000001'}   \n",
       "\n",
       "                                         IndependentAuditFinancialStmt  \\\n",
       "435     {u'#text': u'1', u'@referenceDocumentId': u'RetDoc1040000001'}   \n",
       "3907    {u'#text': u'1', u'@referenceDocumentId': u'RetDoc1040000001'}   \n",
       "4438                                                             false   \n",
       "5358    {u'#text': u'0', u'@referenceDocumentId': u'RetDoc1040000001'}   \n",
       "7045  {u'#text': u'true', u'@referenceDocumentId': u'IRS990ScheduleD'}   \n",
       "\n",
       "      FeesForServicesAccounting_binary  independent_directors_num  \\\n",
       "435                                  1                         28   \n",
       "3907                                 1                         12   \n",
       "4438                                 1                         21   \n",
       "5358                                 1                         11   \n",
       "7045                                 0                          4   \n",
       "\n",
       "      voting_directors  independent_directors_pct  outsourced_mgt  990_review  \\\n",
       "435                 30                   0.933333               0           1   \n",
       "3907                13                   0.923077               0           1   \n",
       "4438                24                   0.875000               0           0   \n",
       "5358                11                   1.000000               0           1   \n",
       "7045                 6                   0.666667               0           1   \n",
       "\n",
       "      audit_committee  \n",
       "435                 1  \n",
       "3907                1  \n",
       "4438                1  \n",
       "5358                1  \n",
       "7045                1  "
      ]
     },
     "execution_count": 290,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "governance[governance['donor_restrictions']>10]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 474,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "count   7133.00\n",
       "mean       0.32\n",
       "std        2.27\n",
       "min     -156.44\n",
       "25%        0.06\n",
       "50%        0.24\n",
       "75%        0.55\n",
       "max       80.38\n",
       "Name: donor_restrictions, dtype: float64"
      ]
     },
     "execution_count": 474,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "governance['donor_restrictions'] = np.where(governance['donor_restrictions']>100, 0, governance['donor_restrictions'])\n",
    "governance['donor_restrictions'].describe()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "##### Donor Restrictions (binary) - as in Harris et al."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 476,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>EIN</th>\n",
       "      <th>FYE</th>\n",
       "      <th>VotingMembersGoverningBodyCnt</th>\n",
       "      <th>NbrVotingMembersGoverningBody</th>\n",
       "      <th>VotingMembersIndependentCnt</th>\n",
       "      <th>NbrIndependentVotingMembers</th>\n",
       "      <th>DelegationOfMgmtDutiesInd</th>\n",
       "      <th>DelegationOfManagementDuties</th>\n",
       "      <th>Form990ProvidedToGvrnBodyInd</th>\n",
       "      <th>Form990ProvidedToGoverningBody</th>\n",
       "      <th>AuditCommitteeInd</th>\n",
       "      <th>AuditCommittee</th>\n",
       "      <th>perm_rest_assets</th>\n",
       "      <th>temp_rest_assets</th>\n",
       "      <th>net_assets</th>\n",
       "      <th>donor_restrictions</th>\n",
       "      <th>TaxExemptBondsInd</th>\n",
       "      <th>TaxExemptBonds</th>\n",
       "      <th>NetUnrelatedBusTxblIncmAmt</th>\n",
       "      <th>NetUnrelatedBusinessTxblIncome</th>\n",
       "      <th>FSAuditedInd</th>\n",
       "      <th>FSAudited</th>\n",
       "      <th>FederalGrantAuditPerformedInd</th>\n",
       "      <th>FederalGrantAuditPerformed</th>\n",
       "      <th>FederalGrantAuditRequiredInd</th>\n",
       "      <th>FederalGrantAuditRequired</th>\n",
       "      <th>IndependentAuditFinclStmtInd</th>\n",
       "      <th>IndependentAuditFinancialStmt</th>\n",
       "      <th>FeesForServicesAccounting_binary</th>\n",
       "      <th>FamilyOrBusinessRlnInd</th>\n",
       "      <th>FamilyOrBusinessRelationship</th>\n",
       "      <th>TaxExemptBondLiabilitiesGrp</th>\n",
       "      <th>TaxExemptBondLiabilities</th>\n",
       "      <th>GovernmentGrantsAmt</th>\n",
       "      <th>GovernmentGrants</th>\n",
       "      <th>ElectionOfBoardMembersInd</th>\n",
       "      <th>ElectionOfBoardMembers</th>\n",
       "      <th>independent_directors_num</th>\n",
       "      <th>voting_directors</th>\n",
       "      <th>independent_directors_pct</th>\n",
       "      <th>independent_directors_pct_v2</th>\n",
       "      <th>outsourced_mgt</th>\n",
       "      <th>990_review</th>\n",
       "      <th>audit_committee</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>010202467</td>\n",
       "      <td>FY2014</td>\n",
       "      <td>22</td>\n",
       "      <td>20</td>\n",
       "      <td>21</td>\n",
       "      <td>20</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>4433997</td>\n",
       "      <td>7351259</td>\n",
       "      <td>23690097</td>\n",
       "      <td>0.50</td>\n",
       "      <td>{u'#text': u'1', u'@referenceDocumentId': u'RetDoc1053100001'}</td>\n",
       "      <td>{u'#text': u'1', u'@referenceDocumentId': u'RetDoc1053100001'}</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>{u'#text': u'1', u'@referenceDocumentId': u'RetDoc1040000001'}</td>\n",
       "      <td>{u'#text': u'1', u'@referenceDocumentId': u'RetDoc1040000001'}</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>7932282</td>\n",
       "      <td>7295185</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>21</td>\n",
       "      <td>22</td>\n",
       "      <td>0.95</td>\n",
       "      <td>0.95</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "         EIN     FYE VotingMembersGoverningBodyCnt  \\\n",
       "0  010202467  FY2014                            22   \n",
       "\n",
       "  NbrVotingMembersGoverningBody VotingMembersIndependentCnt  \\\n",
       "0                            20                          21   \n",
       "\n",
       "  NbrIndependentVotingMembers DelegationOfMgmtDutiesInd  \\\n",
       "0                          20                         1   \n",
       "\n",
       "  DelegationOfManagementDuties Form990ProvidedToGvrnBodyInd  \\\n",
       "0                            1                            1   \n",
       "\n",
       "  Form990ProvidedToGoverningBody AuditCommitteeInd AuditCommittee  \\\n",
       "0                              0                 1              1   \n",
       "\n",
       "   perm_rest_assets  temp_rest_assets  net_assets  donor_restrictions  \\\n",
       "0           4433997           7351259    23690097                0.50   \n",
       "\n",
       "                                                TaxExemptBondsInd  \\\n",
       "0  {u'#text': u'1', u'@referenceDocumentId': u'RetDoc1053100001'}   \n",
       "\n",
       "                                                   TaxExemptBonds  \\\n",
       "0  {u'#text': u'1', u'@referenceDocumentId': u'RetDoc1053100001'}   \n",
       "\n",
       "  NetUnrelatedBusTxblIncmAmt NetUnrelatedBusinessTxblIncome FSAuditedInd  \\\n",
       "0                          0                              0            1   \n",
       "\n",
       "  FSAudited FederalGrantAuditPerformedInd FederalGrantAuditPerformed  \\\n",
       "0         1                             1                          1   \n",
       "\n",
       "  FederalGrantAuditRequiredInd FederalGrantAuditRequired  \\\n",
       "0                            1                         1   \n",
       "\n",
       "                                     IndependentAuditFinclStmtInd  \\\n",
       "0  {u'#text': u'1', u'@referenceDocumentId': u'RetDoc1040000001'}   \n",
       "\n",
       "                                    IndependentAuditFinancialStmt  \\\n",
       "0  {u'#text': u'1', u'@referenceDocumentId': u'RetDoc1040000001'}   \n",
       "\n",
       "   FeesForServicesAccounting_binary FamilyOrBusinessRlnInd  \\\n",
       "0                                 1                      0   \n",
       "\n",
       "  FamilyOrBusinessRelationship TaxExemptBondLiabilitiesGrp  \\\n",
       "0                            1                         NaN   \n",
       "\n",
       "  TaxExemptBondLiabilities GovernmentGrantsAmt GovernmentGrants  \\\n",
       "0                      NaN             7932282          7295185   \n",
       "\n",
       "  ElectionOfBoardMembersInd ElectionOfBoardMembers  independent_directors_num  \\\n",
       "0                         1                      1                         21   \n",
       "\n",
       "   voting_directors  independent_directors_pct  independent_directors_pct_v2  \\\n",
       "0                22                       0.95                          0.95   \n",
       "\n",
       "   outsourced_mgt  990_review  audit_committee  \n",
       "0               1           1                1  "
      ]
     },
     "execution_count": 476,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "governance[:1]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 479,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "1    6414\n",
       "0     719\n",
       "Name: restricted_donations, dtype: int64"
      ]
     },
     "execution_count": 479,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "governance['restricted_donations'] = np.where(((governance['perm_rest_assets']>1)|\n",
    "                                               (governance['temp_rest_assets']>1)), 1,0)\n",
    "governance['restricted_donations'].value_counts()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 491,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>perm_rest_assets</th>\n",
       "      <th>temp_rest_assets</th>\n",
       "      <th>restricted_donations</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>20</th>\n",
       "      <td>23608300</td>\n",
       "      <td>3240612</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>21</th>\n",
       "      <td>0</td>\n",
       "      <td>8161109</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>22</th>\n",
       "      <td>153032</td>\n",
       "      <td>1981180</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>23</th>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>24</th>\n",
       "      <td>11057335</td>\n",
       "      <td>14212432</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "    perm_rest_assets  temp_rest_assets  restricted_donations\n",
       "20          23608300           3240612                     1\n",
       "21                 0           8161109                     1\n",
       "22            153032           1981180                     1\n",
       "23                 0                 0                     0\n",
       "24          11057335          14212432                     1"
      ]
     },
     "execution_count": 491,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "governance[['perm_rest_assets', 'temp_rest_assets', 'restricted_donations']][20:25]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "##### Donor Restrictions -- Part (a) - Temporarily Restricted Assets"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 123,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "842\n"
     ]
    }
   ],
   "source": [
    "'''\n",
    "print len(governance[governance['TemporarilyRstrNetAssetsGrp'].isnull()])\n",
    "#, 'TemporarilyRestrictedNetAssets', \n",
    "#'PermanentlyRstrNetAssetsGrp', 'PermanentlyRestrictedNetAssets', \n",
    "#'NetAssetsOrFundBalancesEOYAmt', 'NetAssetsOrFundBalancesEOY',\n",
    "'''"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 291,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "#governance[267:270]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 140,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "('Index: ', 7132, 'Temp. Restr Assets:', u'52310') \n",
      "\n",
      "# of minutes:  0.214020017783 \n",
      "\n",
      "\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "7133"
      ]
     },
     "execution_count": 140,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "'''\n",
    "from IPython.display import display, clear_output   ##### FOR USE WITH STDOUT (DYNAMIC, SINGLE-LINE PRINTING)\n",
    "\n",
    "import timeit\n",
    "start_time = timeit.default_timer()\n",
    "\n",
    "for index, row in governance[:].iterrows():\n",
    "    if pd.notnull(row['TemporarilyRstrNetAssetsGrp']):\n",
    "        #print 'not null!'\n",
    "        if 'EOYAmt' in row['TemporarilyRstrNetAssetsGrp']:\n",
    "            temp_rest = row['TemporarilyRstrNetAssetsGrp']['EOYAmt']\n",
    "        elif 'BOYAmt' in row['TemporarilyRstrNetAssetsGrp']:\n",
    "            temp_rest = row['TemporarilyRstrNetAssetsGrp']['BOYAmt']            \n",
    "    elif pd.notnull(row['TemporarilyRestrictedNetAssets']):\n",
    "        print 'going to second'\n",
    "        if 'EOY' in row['TemporarilyRestrictedNetAssets']:\n",
    "            temp_rest = row['TemporarilyRestrictedNetAssets']['EOY']\n",
    "        elif 'BOY' in row['TemporarilyRestrictedNetAssets']:\n",
    "            temp_rest = row['TemporarilyRestrictedNetAssets']['BOY']\n",
    "            \n",
    "    governance.ix[index, 'temp_rest_assets'] = temp_rest\n",
    "        \n",
    "    clear_output()\n",
    "    print ('Index: ', index, 'Temp. Restr Assets:', temp_rest), '\\n'\n",
    "    sys.stdout.flush()        \n",
    "        \n",
    "        \n",
    "elapsed = timeit.default_timer() - start_time\n",
    "print '# of minutes: ', elapsed/60, '\\n', '\\n'         \n",
    "print governance['temp_rest_assets'].value_counts().sum()\n",
    "'''"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 146,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "7133\n",
      "0\n",
      "count     7133\n",
      "unique    6282\n",
      "top          0\n",
      "freq       195\n",
      "Name: temp_rest_assets, dtype: object\n",
      "count    7.133000e+03\n",
      "mean     7.934227e+06\n",
      "std      4.203098e+07\n",
      "min     -1.417569e+06\n",
      "25%      2.050260e+05\n",
      "50%      8.326920e+05\n",
      "75%      3.255075e+06\n",
      "max      1.473751e+09\n",
      "Name: temp_rest_assets, dtype: float64\n"
     ]
    }
   ],
   "source": [
    "'''\n",
    "print len(governance[governance['temp_rest_assets'].notnull()])\n",
    "print len(governance[governance['temp_rest_assets'].isnull()])\n",
    "print governance['temp_rest_assets'].describe()\n",
    "governance['temp_rest_assets'] = governance['temp_rest_assets'].astype('int')\n",
    "print governance['temp_rest_assets'].describe()\n",
    "'''"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "##### Donor Restrictions -- Part (b) - Permanently Restricted Assets"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 292,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "#governance[:2]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 149,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "3520\n",
      "3613\n",
      "3699\n",
      "3434\n"
     ]
    }
   ],
   "source": [
    "'''\n",
    "print len(governance[governance['PermanentlyRestrictedNetAssets'].notnull()])\n",
    "print len(governance[governance['PermanentlyRestrictedNetAssets'].isnull()])\n",
    "print len(governance[governance['PermanentlyRstrNetAssetsGrp'].notnull()])\n",
    "print len(governance[governance['PermanentlyRstrNetAssetsGrp'].isnull()])\n",
    "'''"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 151,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "('Index: ', 7132, 'Perm. Restr Assets:', u'284363') \n",
      "\n",
      "# of minutes:  0.194717868169 \n",
      "\n",
      "\n",
      "7133\n"
     ]
    }
   ],
   "source": [
    "'''\n",
    "from IPython.display import display, clear_output   ##### FOR USE WITH STDOUT (DYNAMIC, SINGLE-LINE PRINTING)\n",
    "\n",
    "import timeit\n",
    "start_time = timeit.default_timer()\n",
    "\n",
    "for index, row in governance[:].iterrows():\n",
    "    if pd.notnull(row['PermanentlyRstrNetAssetsGrp']):\n",
    "        #print 'not null!'\n",
    "        if 'EOYAmt' in row['PermanentlyRstrNetAssetsGrp']:\n",
    "            perm_rest = row['PermanentlyRstrNetAssetsGrp']['EOYAmt']\n",
    "        elif 'BOYAmt' in row['PermanentlyRstrNetAssetsGrp']:\n",
    "            perm_rest = row['PermanentlyRstrNetAssetsGrp']['BOYAmt']            \n",
    "    elif pd.notnull(row['PermanentlyRestrictedNetAssets']):\n",
    "        print 'going to second variable'\n",
    "        if 'EOY' in row['PermanentlyRestrictedNetAssets']:\n",
    "            perm_rest = row['PermanentlyRestrictedNetAssets']['EOY']\n",
    "        elif 'BOY' in row['PermanentlyRestrictedNetAssets']:\n",
    "            perm_rest = row['PermanentlyRestrictedNetAssets']['BOY']\n",
    "            \n",
    "    governance.ix[index, 'perm_rest_assets'] = perm_rest\n",
    "        \n",
    "    clear_output()\n",
    "    print ('Index: ', index, 'Perm. Restr Assets:', perm_rest), '\\n'\n",
    "    sys.stdout.flush()        \n",
    "        \n",
    "        \n",
    "elapsed = timeit.default_timer() - start_time\n",
    "print '# of minutes: ', elapsed/60, '\\n', '\\n'         \n",
    "print governance['perm_rest_assets'].value_counts().sum()\n",
    "'''"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 153,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "7133\n",
      "0\n",
      "count     7133\n",
      "unique    3215\n",
      "top          0\n",
      "freq       887\n",
      "Name: perm_rest_assets, dtype: object\n",
      "count    7.133000e+03\n",
      "mean     7.831133e+06\n",
      "std      3.922188e+07\n",
      "min      0.000000e+00\n",
      "25%      6.430000e+04\n",
      "50%      5.732270e+05\n",
      "75%      3.064642e+06\n",
      "max      1.097021e+09\n",
      "Name: perm_rest_assets, dtype: float64\n"
     ]
    }
   ],
   "source": [
    "'''\n",
    "print len(governance[governance['perm_rest_assets'].notnull()])\n",
    "print len(governance[governance['perm_rest_assets'].isnull()])\n",
    "print governance['perm_rest_assets'].describe(), '\\n'\n",
    "governance['perm_rest_assets'] = governance['perm_rest_assets'].astype('int')\n",
    "print governance['perm_rest_assets'].describe()\n",
    "'''"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "##### Donor Restrictions -- Part (c) - Total Assets"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 157,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "count     7133\n",
      "unique    7120\n",
      "top          0\n",
      "freq        14\n",
      "Name: net_assets, dtype: object \n",
      "\n",
      "count    7.133000e+03\n",
      "mean     3.140955e+07\n",
      "std      1.753475e+08\n",
      "min     -1.399548e+08\n",
      "25%      1.766750e+06\n",
      "50%      5.157328e+06\n",
      "75%      1.582864e+07\n",
      "max      7.982614e+09\n",
      "Name: net_assets, dtype: float64\n"
     ]
    }
   ],
   "source": [
    "'''\n",
    "governance['net_assets'] = governance['NetAssetsOrFundBalancesEOYAmt']\n",
    "governance['net_assets'] = np.where(governance['net_assets'].isnull(),\n",
    "                                          governance['NetAssetsOrFundBalancesEOY'], governance['net_assets'])\n",
    "print governance['net_assets'].describe(), '\\n'\n",
    "governance['net_assets'] = governance['net_assets'].astype('int')\n",
    "print governance['net_assets'].describe()\n",
    "'''"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "##### Donor Restrictions -- Part (d) - Calculations"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 159,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "count    7.131000e+03\n",
       "mean              inf\n",
       "std               NaN\n",
       "min     -4.987176e+05\n",
       "25%      1.567595e-01\n",
       "50%      4.216044e-01\n",
       "75%      8.305730e-01\n",
       "max               inf\n",
       "Name: donor_restrictions, dtype: float64"
      ]
     },
     "execution_count": 159,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "'''\n",
    "governance['donor_restrictions'] = (governance['temp_rest_assets']+governance['perm_rest_assets'])/governance['net_assets']\n",
    "governance['donor_restrictions'].describe()\n",
    "'''"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 162,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "count    7.133000e+03\n",
       "mean              inf\n",
       "std               NaN\n",
       "min     -4.987176e+05\n",
       "25%      1.567123e-01\n",
       "50%      4.216012e-01\n",
       "75%      8.303938e-01\n",
       "max               inf\n",
       "Name: donor_restrictions, dtype: float64"
      ]
     },
     "execution_count": 162,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "'''\n",
    "governance['donor_restrictions'] = np.where(governance['donor_restrictions'].isnull(), 0, governance['donor_restrictions'])\n",
    "governance['donor_restrictions'].describe()\n",
    "'''"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 169,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>FYE</th>\n",
       "      <th>TemporarilyRstrNetAssetsGrp</th>\n",
       "      <th>TemporarilyRestrictedNetAssets</th>\n",
       "      <th>PermanentlyRstrNetAssetsGrp</th>\n",
       "      <th>PermanentlyRestrictedNetAssets</th>\n",
       "      <th>UnrestrictedNetAssetsGrp</th>\n",
       "      <th>UnrestrictedNetAssets</th>\n",
       "      <th>TotalNetAssetsFundBalanceGrp</th>\n",
       "      <th>TotalNetAssetsFundBalances</th>\n",
       "      <th>NetAssetsOrFundBalancesBOYAmt</th>\n",
       "      <th>NetAssetsOrFundBalancesBOY</th>\n",
       "      <th>NetAssetsOrFundBalancesEOYAmt</th>\n",
       "      <th>NetAssetsOrFundBalancesEOY</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>19137</th>\n",
       "      <td>FY2014</td>\n",
       "      <td>{u'BOYAmt': u'763419', u'EOYAmt': u'718875'}</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>{u'BOYAmt': u'1114442', u'EOYAmt': u'1112268'}</td>\n",
       "      <td>NaN</td>\n",
       "      <td>{u'BOYAmt': u'1877861', u'EOYAmt': u'1831143'}</td>\n",
       "      <td>NaN</td>\n",
       "      <td>1877861</td>\n",
       "      <td>NaN</td>\n",
       "      <td>1831143</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4153</th>\n",
       "      <td>FY2013</td>\n",
       "      <td>{u'BOYAmt': u'568056', u'EOYAmt': u'763419'}</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>{u'BOYAmt': u'969792', u'EOYAmt': u'1114442'}</td>\n",
       "      <td>NaN</td>\n",
       "      <td>{u'BOYAmt': u'1537848', u'EOYAmt': u'1877861'}</td>\n",
       "      <td>NaN</td>\n",
       "      <td>1537848</td>\n",
       "      <td>NaN</td>\n",
       "      <td>1877861</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4696</th>\n",
       "      <td>FY2012</td>\n",
       "      <td>NaN</td>\n",
       "      <td>{u'BOY': u'489517', u'EOY': u'568056'}</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>{u'BOY': u'1227268', u'EOY': u'969792'}</td>\n",
       "      <td>NaN</td>\n",
       "      <td>{u'BOY': u'1716785', u'EOY': u'1537848'}</td>\n",
       "      <td>NaN</td>\n",
       "      <td>1716785</td>\n",
       "      <td>NaN</td>\n",
       "      <td>1537848</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>22671</th>\n",
       "      <td>FY2011</td>\n",
       "      <td>NaN</td>\n",
       "      <td>{u'BOY': u'472102', u'EOY': u'489517'}</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>{u'BOY': u'1451414', u'EOY': u'1227268'}</td>\n",
       "      <td>NaN</td>\n",
       "      <td>{u'BOY': u'1923516', u'EOY': u'1716785'}</td>\n",
       "      <td>NaN</td>\n",
       "      <td>1923516</td>\n",
       "      <td>NaN</td>\n",
       "      <td>1716785</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "          FYE                   TemporarilyRstrNetAssetsGrp  \\\n",
       "19137  FY2014  {u'BOYAmt': u'763419', u'EOYAmt': u'718875'}   \n",
       "4153   FY2013  {u'BOYAmt': u'568056', u'EOYAmt': u'763419'}   \n",
       "4696   FY2012                                           NaN   \n",
       "22671  FY2011                                           NaN   \n",
       "\n",
       "               TemporarilyRestrictedNetAssets PermanentlyRstrNetAssetsGrp  \\\n",
       "19137                                     NaN                         NaN   \n",
       "4153                                      NaN                         NaN   \n",
       "4696   {u'BOY': u'489517', u'EOY': u'568056'}                         NaN   \n",
       "22671  {u'BOY': u'472102', u'EOY': u'489517'}                         NaN   \n",
       "\n",
       "      PermanentlyRestrictedNetAssets  \\\n",
       "19137                            NaN   \n",
       "4153                             NaN   \n",
       "4696                             NaN   \n",
       "22671                            NaN   \n",
       "\n",
       "                             UnrestrictedNetAssetsGrp  \\\n",
       "19137  {u'BOYAmt': u'1114442', u'EOYAmt': u'1112268'}   \n",
       "4153    {u'BOYAmt': u'969792', u'EOYAmt': u'1114442'}   \n",
       "4696                                              NaN   \n",
       "22671                                             NaN   \n",
       "\n",
       "                          UnrestrictedNetAssets  \\\n",
       "19137                                       NaN   \n",
       "4153                                        NaN   \n",
       "4696    {u'BOY': u'1227268', u'EOY': u'969792'}   \n",
       "22671  {u'BOY': u'1451414', u'EOY': u'1227268'}   \n",
       "\n",
       "                         TotalNetAssetsFundBalanceGrp  \\\n",
       "19137  {u'BOYAmt': u'1877861', u'EOYAmt': u'1831143'}   \n",
       "4153   {u'BOYAmt': u'1537848', u'EOYAmt': u'1877861'}   \n",
       "4696                                              NaN   \n",
       "22671                                             NaN   \n",
       "\n",
       "                     TotalNetAssetsFundBalances NetAssetsOrFundBalancesBOYAmt  \\\n",
       "19137                                       NaN                       1877861   \n",
       "4153                                        NaN                       1537848   \n",
       "4696   {u'BOY': u'1716785', u'EOY': u'1537848'}                           NaN   \n",
       "22671  {u'BOY': u'1923516', u'EOY': u'1716785'}                           NaN   \n",
       "\n",
       "      NetAssetsOrFundBalancesBOY NetAssetsOrFundBalancesEOYAmt  \\\n",
       "19137                        NaN                       1831143   \n",
       "4153                         NaN                       1877861   \n",
       "4696                     1716785                           NaN   \n",
       "22671                    1923516                           NaN   \n",
       "\n",
       "      NetAssetsOrFundBalancesEOY  \n",
       "19137                        NaN  \n",
       "4153                         NaN  \n",
       "4696                     1537848  \n",
       "22671                    1716785  "
      ]
     },
     "execution_count": 169,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "'''\n",
    "df_all[df_all['EIN']=='010287624'][['FYE', 'TemporarilyRstrNetAssetsGrp', 'TemporarilyRestrictedNetAssets',\n",
    "              'PermanentlyRstrNetAssetsGrp', 'PermanentlyRestrictedNetAssets',\n",
    "              'UnrestrictedNetAssetsGrp', 'UnrestrictedNetAssets', \n",
    "              'TotalNetAssetsFundBalanceGrp', 'TotalNetAssetsFundBalances', \n",
    "              'NetAssetsOrFundBalancesBOYAmt', 'NetAssetsOrFundBalancesBOY', \n",
    "              'NetAssetsOrFundBalancesEOYAmt', 'NetAssetsOrFundBalancesEOY']]\n",
    "'''"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 171,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.39258266558100596"
      ]
     },
     "execution_count": 171,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "#(718875+0)/(718875+1112268)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 174,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0\n",
      "-1\n"
     ]
    }
   ],
   "source": [
    "#print (718875+1112268)-1831143\n",
    "#print 1831143 - (718875+1112268)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "collapsed": true
   },
   "source": [
    "##### Tax-Exempt Bond"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 492,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "('Index: ', 7132, 'Tax-exempt bond:', u'0') \n",
      "\n",
      "# of minutes:  0.197681299845 \n",
      "\n",
      "\n",
      "7133\n"
     ]
    }
   ],
   "source": [
    "from IPython.display import display, clear_output   ##### FOR USE WITH STDOUT (DYNAMIC, SINGLE-LINE PRINTING)\n",
    "\n",
    "import timeit\n",
    "start_time = timeit.default_timer()\n",
    "\n",
    "for index, row in governance[:].iterrows():\n",
    "    bond = 0\n",
    "    if pd.notnull(row['TaxExemptBondsInd']):\n",
    "        #print 'not null!'\n",
    "        if '#text' in row['TaxExemptBondsInd']:\n",
    "            bond = row['TaxExemptBondsInd']['#text']\n",
    "        else:\n",
    "            bond = row['TaxExemptBondsInd']           \n",
    "    elif pd.notnull(row['TaxExemptBonds']):\n",
    "        if '#text' in row['TaxExemptBonds']:\n",
    "            bond = row['TaxExemptBonds']['#text']\n",
    "        else:\n",
    "            bond = row['TaxExemptBonds']            \n",
    "            \n",
    "    governance.ix[index, 'tax_exempt_bond'] = bond\n",
    "        \n",
    "    clear_output()\n",
    "    print ('Index: ', index, 'Tax-exempt bond:', bond), '\\n'\n",
    "    sys.stdout.flush()        \n",
    "        \n",
    "        \n",
    "elapsed = timeit.default_timer() - start_time\n",
    "print '# of minutes: ', elapsed/60, '\\n', '\\n'         \n",
    "print governance['tax_exempt_bond'].value_counts().sum()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 493,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0        4251\n",
      "false    2580\n",
      "1         206\n",
      "true       96\n",
      "Name: tax_exempt_bond, dtype: int64 \n",
      "\n",
      "0    6831\n",
      "1     302\n",
      "Name: tax_exempt_bond, dtype: int64\n"
     ]
    }
   ],
   "source": [
    "print governance['tax_exempt_bond'].value_counts(), '\\n'\n",
    "governance['tax_exempt_bond'] = np.where(governance['tax_exempt_bond']=='true', 1, governance['tax_exempt_bond'])\n",
    "governance['tax_exempt_bond'] = np.where(governance['tax_exempt_bond']=='false', 0, governance['tax_exempt_bond'])\n",
    "governance['tax_exempt_bond'] = np.where(governance['tax_exempt_bond'].isnull(), 0, governance['tax_exempt_bond'])\n",
    "governance['tax_exempt_bond'] = governance['tax_exempt_bond'].astype('int')\n",
    "print governance['tax_exempt_bond'].value_counts()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "##### Tax-Exempt Bonds, v2 (Harris et al.)\n",
    "NO NEED TO DO THIS -- MY VARIABLE ALREADY TAPS IT"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 501,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>TaxExemptBondLiabilitiesGrp</th>\n",
       "      <th>TaxExemptBondLiabilities</th>\n",
       "      <th>tax_exempt_bond</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>{u'BOYAmt': u'106014868', u'EOYAmt': u'104647760'}</td>\n",
       "      <td>{u'BOY': u'77905017', u'EOY': u'109371484'}</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                          TaxExemptBondLiabilitiesGrp  \\\n",
       "2  {u'BOYAmt': u'106014868', u'EOYAmt': u'104647760'}   \n",
       "3                                                 NaN   \n",
       "\n",
       "                      TaxExemptBondLiabilities  tax_exempt_bond  \n",
       "2  {u'BOY': u'77905017', u'EOY': u'109371484'}                1  \n",
       "3                                          NaN                0  "
      ]
     },
     "execution_count": 501,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "governance[['TaxExemptBondLiabilitiesGrp', 'TaxExemptBondLiabilities', 'tax_exempt_bond']][2:4]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "collapsed": true
   },
   "source": [
    "##### Taxable Revenues\n",
    "AS IN YETMAN AND YETMAN'S *TAXABLE REVENUE* VARIABLE -- A REGULATORY OVERSIGHT VARIABLE, GIVEN THAT IT IS RELATED TO AUDIT LIKELIHOOD."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 502,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "1743\n",
      "count     5390\n",
      "unique     971\n",
      "top          0\n",
      "freq      4406\n",
      "Name: taxable_revenue, dtype: object \n",
      "\n",
      "count     7133\n",
      "unique     972\n",
      "top          0\n",
      "freq      4406\n",
      "Name: taxable_revenue, dtype: object \n",
      "\n",
      "count       7133.00\n",
      "mean       -7272.61\n",
      "std       156439.97\n",
      "min     -8633000.00\n",
      "25%            0.00\n",
      "50%            0.00\n",
      "75%            0.00\n",
      "max      2376271.00\n",
      "Name: taxable_revenue, dtype: float64 \n",
      "\n"
     ]
    }
   ],
   "source": [
    "governance['taxable_revenue'] = governance['NetUnrelatedBusTxblIncmAmt']\n",
    "governance['taxable_revenue'] = np.where(governance['taxable_revenue'].isnull(),\n",
    "                                          governance['NetUnrelatedBusinessTxblIncome'], governance['taxable_revenue'])\n",
    "print len(governance[governance['taxable_revenue'].isnull()])\n",
    "print governance['taxable_revenue'].describe(), '\\n'\n",
    "governance['taxable_revenue'] = np.where(governance['taxable_revenue'].isnull(), 0, governance['taxable_revenue'])\n",
    "print governance['taxable_revenue'].describe(), '\\n'\n",
    "governance['taxable_revenue'] = governance['taxable_revenue'].astype('int')\n",
    "print governance['taxable_revenue'].describe(), '\\n'"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "<br>CREATE TWO BINARY VERSION -- ONE FOR TAXABLE REVENUE GREATER THAN ZERO AND ONE FOR TAXABLE REVENUE OTHER THAN ZERO"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 503,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "361\n",
      "6149\n",
      "623\n",
      "0    6772\n",
      "1     361\n",
      "Name: taxable_revenue_binary, dtype: int64 \n",
      "\n",
      "0    6149\n",
      "1     984\n",
      "Name: taxable_revenue_neg_or_pos_rev_binary, dtype: int64\n"
     ]
    }
   ],
   "source": [
    "print len(governance[governance['taxable_revenue']>0])\n",
    "print len(governance[governance['taxable_revenue']==0])\n",
    "print len(governance[governance['taxable_revenue']<0])\n",
    "governance['taxable_revenue_binary'] = np.where(governance['taxable_revenue']>0, 1,0)\n",
    "print governance['taxable_revenue_binary'].value_counts(), '\\n'\n",
    "governance['taxable_revenue_neg_or_pos_rev_binary'] = np.where( ((governance['taxable_revenue']>0) |\n",
    "                                                                 (governance['taxable_revenue']<0)), 1,0)\n",
    "print governance['taxable_revenue_neg_or_pos_rev_binary'].value_counts()                                                               "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 504,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>EIN</th>\n",
       "      <th>FYE</th>\n",
       "      <th>VotingMembersGoverningBodyCnt</th>\n",
       "      <th>NbrVotingMembersGoverningBody</th>\n",
       "      <th>VotingMembersIndependentCnt</th>\n",
       "      <th>NbrIndependentVotingMembers</th>\n",
       "      <th>DelegationOfMgmtDutiesInd</th>\n",
       "      <th>DelegationOfManagementDuties</th>\n",
       "      <th>Form990ProvidedToGvrnBodyInd</th>\n",
       "      <th>Form990ProvidedToGoverningBody</th>\n",
       "      <th>AuditCommitteeInd</th>\n",
       "      <th>AuditCommittee</th>\n",
       "      <th>perm_rest_assets</th>\n",
       "      <th>temp_rest_assets</th>\n",
       "      <th>net_assets</th>\n",
       "      <th>donor_restrictions</th>\n",
       "      <th>TaxExemptBondsInd</th>\n",
       "      <th>TaxExemptBonds</th>\n",
       "      <th>NetUnrelatedBusTxblIncmAmt</th>\n",
       "      <th>NetUnrelatedBusinessTxblIncome</th>\n",
       "      <th>FSAuditedInd</th>\n",
       "      <th>FSAudited</th>\n",
       "      <th>FederalGrantAuditPerformedInd</th>\n",
       "      <th>FederalGrantAuditPerformed</th>\n",
       "      <th>FederalGrantAuditRequiredInd</th>\n",
       "      <th>FederalGrantAuditRequired</th>\n",
       "      <th>IndependentAuditFinclStmtInd</th>\n",
       "      <th>IndependentAuditFinancialStmt</th>\n",
       "      <th>FeesForServicesAccounting_binary</th>\n",
       "      <th>FamilyOrBusinessRlnInd</th>\n",
       "      <th>FamilyOrBusinessRelationship</th>\n",
       "      <th>TaxExemptBondLiabilitiesGrp</th>\n",
       "      <th>TaxExemptBondLiabilities</th>\n",
       "      <th>GovernmentGrantsAmt</th>\n",
       "      <th>GovernmentGrants</th>\n",
       "      <th>ElectionOfBoardMembersInd</th>\n",
       "      <th>ElectionOfBoardMembers</th>\n",
       "      <th>independent_directors_num</th>\n",
       "      <th>voting_directors</th>\n",
       "      <th>independent_directors_pct</th>\n",
       "      <th>independent_directors_pct_v2</th>\n",
       "      <th>outsourced_mgt</th>\n",
       "      <th>990_review</th>\n",
       "      <th>audit_committee</th>\n",
       "      <th>restricted_donations</th>\n",
       "      <th>tax_exempt_bond</th>\n",
       "      <th>taxable_revenue</th>\n",
       "      <th>taxable_revenue_binary</th>\n",
       "      <th>taxable_revenue_neg_or_pos_rev_binary</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>010202467</td>\n",
       "      <td>FY2014</td>\n",
       "      <td>22</td>\n",
       "      <td>20</td>\n",
       "      <td>21</td>\n",
       "      <td>20</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>4433997</td>\n",
       "      <td>7351259</td>\n",
       "      <td>23690097</td>\n",
       "      <td>0.50</td>\n",
       "      <td>{u'#text': u'1', u'@referenceDocumentId': u'RetDoc1053100001'}</td>\n",
       "      <td>{u'#text': u'1', u'@referenceDocumentId': u'RetDoc1053100001'}</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>{u'#text': u'1', u'@referenceDocumentId': u'RetDoc1040000001'}</td>\n",
       "      <td>{u'#text': u'1', u'@referenceDocumentId': u'RetDoc1040000001'}</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>7932282</td>\n",
       "      <td>7295185</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>21</td>\n",
       "      <td>22</td>\n",
       "      <td>0.95</td>\n",
       "      <td>0.95</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "         EIN     FYE VotingMembersGoverningBodyCnt  \\\n",
       "0  010202467  FY2014                            22   \n",
       "\n",
       "  NbrVotingMembersGoverningBody VotingMembersIndependentCnt  \\\n",
       "0                            20                          21   \n",
       "\n",
       "  NbrIndependentVotingMembers DelegationOfMgmtDutiesInd  \\\n",
       "0                          20                         1   \n",
       "\n",
       "  DelegationOfManagementDuties Form990ProvidedToGvrnBodyInd  \\\n",
       "0                            1                            1   \n",
       "\n",
       "  Form990ProvidedToGoverningBody AuditCommitteeInd AuditCommittee  \\\n",
       "0                              0                 1              1   \n",
       "\n",
       "   perm_rest_assets  temp_rest_assets  net_assets  donor_restrictions  \\\n",
       "0           4433997           7351259    23690097                0.50   \n",
       "\n",
       "                                                TaxExemptBondsInd  \\\n",
       "0  {u'#text': u'1', u'@referenceDocumentId': u'RetDoc1053100001'}   \n",
       "\n",
       "                                                   TaxExemptBonds  \\\n",
       "0  {u'#text': u'1', u'@referenceDocumentId': u'RetDoc1053100001'}   \n",
       "\n",
       "  NetUnrelatedBusTxblIncmAmt NetUnrelatedBusinessTxblIncome FSAuditedInd  \\\n",
       "0                          0                              0            1   \n",
       "\n",
       "  FSAudited FederalGrantAuditPerformedInd FederalGrantAuditPerformed  \\\n",
       "0         1                             1                          1   \n",
       "\n",
       "  FederalGrantAuditRequiredInd FederalGrantAuditRequired  \\\n",
       "0                            1                         1   \n",
       "\n",
       "                                     IndependentAuditFinclStmtInd  \\\n",
       "0  {u'#text': u'1', u'@referenceDocumentId': u'RetDoc1040000001'}   \n",
       "\n",
       "                                    IndependentAuditFinancialStmt  \\\n",
       "0  {u'#text': u'1', u'@referenceDocumentId': u'RetDoc1040000001'}   \n",
       "\n",
       "   FeesForServicesAccounting_binary FamilyOrBusinessRlnInd  \\\n",
       "0                                 1                      0   \n",
       "\n",
       "  FamilyOrBusinessRelationship TaxExemptBondLiabilitiesGrp  \\\n",
       "0                            1                         NaN   \n",
       "\n",
       "  TaxExemptBondLiabilities GovernmentGrantsAmt GovernmentGrants  \\\n",
       "0                      NaN             7932282          7295185   \n",
       "\n",
       "  ElectionOfBoardMembersInd ElectionOfBoardMembers  independent_directors_num  \\\n",
       "0                         1                      1                         21   \n",
       "\n",
       "   voting_directors  independent_directors_pct  independent_directors_pct_v2  \\\n",
       "0                22                       0.95                          0.95   \n",
       "\n",
       "   outsourced_mgt  990_review  audit_committee  restricted_donations  \\\n",
       "0               1           1                1                     1   \n",
       "\n",
       "   tax_exempt_bond  taxable_revenue  taxable_revenue_binary  \\\n",
       "0                1                0                       0   \n",
       "\n",
       "   taxable_revenue_neg_or_pos_rev_binary  \n",
       "0                                      0  "
      ]
     },
     "execution_count": 504,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "governance[:1]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "##### Audited Financials"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 505,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0\n",
      "6914 \n",
      "\n",
      "7133 \n",
      "\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "1        4254\n",
       "true     2454\n",
       "false     222\n",
       "0         203\n",
       "Name: audited_financials, dtype: int64"
      ]
     },
     "execution_count": 505,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "governance['audited_financials'] = np.nan\n",
    "print len(governance[governance['audited_financials'].notnull()])\n",
    "governance['audited_financials'] = np.where(governance['FSAuditedInd'].notnull(), governance['FSAuditedInd'], \n",
    "                                      governance['audited_financials'] )\n",
    "print len(governance[governance['audited_financials'].notnull()]), '\\n'\n",
    "governance['audited_financials'] = np.where( ((governance['audited_financials'].isnull()) & \n",
    "                                      (governance['FSAudited'].notnull())), \n",
    "                         governance['FSAudited'], governance['audited_financials'] )\n",
    "print len(governance[governance['audited_financials'].notnull()]), '\\n'\n",
    "governance['audited_financials'].value_counts()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 506,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "7133\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "1    6708\n",
       "0     425\n",
       "Name: audited_financials, dtype: int64"
      ]
     },
     "execution_count": 506,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "governance['audited_financials'] = np.where( governance['audited_financials']=='true', 1, governance['audited_financials'] )\n",
    "governance['audited_financials'] = np.where( governance['audited_financials']=='1', 1, governance['audited_financials'] )\n",
    "governance['audited_financials'] = np.where( governance['audited_financials']=='false', 0, governance['audited_financials'] )\n",
    "governance['audited_financials'] = np.where( governance['audited_financials']=='0', 0, governance['audited_financials'] )\n",
    "print len(governance[governance['audited_financials'].notnull()])\n",
    "governance['audited_financials'].value_counts()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "##### Federal Grant Audit Performed"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 507,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "5974\n",
      "5061\n",
      "1        931\n",
      "false    646\n",
      "true     490\n",
      "0          5\n",
      "Name: federal_grant_audit_performed, dtype: int64\n"
     ]
    }
   ],
   "source": [
    "governance['federal_grant_audit_performed'] = governance['FederalGrantAuditPerformedInd']\n",
    "print len(governance[governance['federal_grant_audit_performed'].isnull()])\n",
    "governance['federal_grant_audit_performed'] = np.where(governance['federal_grant_audit_performed'].isnull(),\n",
    "                                          governance['FederalGrantAuditPerformed'], governance['federal_grant_audit_performed'])\n",
    "print len(governance[governance['federal_grant_audit_performed'].isnull()])\n",
    "print governance['federal_grant_audit_performed'].value_counts()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 508,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "2072\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "1    1421\n",
       "0     651\n",
       "Name: federal_grant_audit_performed, dtype: int64"
      ]
     },
     "execution_count": 508,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "governance['federal_grant_audit_performed'] = np.where( governance['federal_grant_audit_performed']=='true', 1, governance['federal_grant_audit_performed'] )\n",
    "governance['federal_grant_audit_performed'] = np.where( governance['federal_grant_audit_performed']=='1', 1, governance['federal_grant_audit_performed'] )\n",
    "governance['federal_grant_audit_performed'] = np.where( governance['federal_grant_audit_performed']=='false', 0, governance['federal_grant_audit_performed'] )\n",
    "governance['federal_grant_audit_performed'] = np.where( governance['federal_grant_audit_performed']=='0', 0, governance['federal_grant_audit_performed'] )\n",
    "print len(governance[governance['federal_grant_audit_performed'].notnull()])\n",
    "governance['federal_grant_audit_performed'].value_counts()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 509,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0    5712\n",
       "1    1421\n",
       "Name: federal_grant_audit_performed, dtype: int64"
      ]
     },
     "execution_count": 509,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "governance['federal_grant_audit_performed'] = np.where( governance['federal_grant_audit_performed'].isnull(), \n",
    "                                                       0, governance['federal_grant_audit_performed'])\n",
    "governance['federal_grant_audit_performed'].value_counts()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "##### Federal Grant Audit Required"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 510,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "288\n",
      "45 \n",
      "\n",
      "0        3746\n",
      "false    2214\n",
      "1         719\n",
      "true      409\n",
      "Name: federal_grant_audit_required, dtype: int64 \n",
      "\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "0        3746\n",
       "false    2214\n",
       "1         719\n",
       "true      409\n",
       "0          45\n",
       "Name: federal_grant_audit_required, dtype: int64"
      ]
     },
     "execution_count": 510,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "governance['federal_grant_audit_required'] = governance['FederalGrantAuditRequiredInd']\n",
    "print len(governance[governance['federal_grant_audit_required'].isnull()])\n",
    "governance['federal_grant_audit_required'] = np.where(governance['federal_grant_audit_required'].isnull(),\n",
    "                                          governance['FederalGrantAuditRequired'], governance['federal_grant_audit_required'])\n",
    "print len(governance[governance['federal_grant_audit_required'].isnull()]), '\\n'\n",
    "print governance['federal_grant_audit_required'].value_counts(), '\\n'\n",
    "governance['federal_grant_audit_required'] = np.where( governance['federal_grant_audit_required'].isnull(), \n",
    "                                                       0, governance['federal_grant_audit_required'])\n",
    "governance['federal_grant_audit_required'].value_counts()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 511,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "7133\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "0    6005\n",
       "1    1128\n",
       "Name: federal_grant_audit_required, dtype: int64"
      ]
     },
     "execution_count": 511,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "governance['federal_grant_audit_required'] = np.where( governance['federal_grant_audit_required']=='true', 1, governance['federal_grant_audit_required'] )\n",
    "governance['federal_grant_audit_required'] = np.where( governance['federal_grant_audit_required']=='1', 1, governance['federal_grant_audit_required'] )\n",
    "governance['federal_grant_audit_required'] = np.where( governance['federal_grant_audit_required']=='false', 0, governance['federal_grant_audit_required'] )\n",
    "governance['federal_grant_audit_required'] = np.where( governance['federal_grant_audit_required']=='0', 0, governance['federal_grant_audit_required'] )\n",
    "print len(governance[governance['federal_grant_audit_required'].notnull()])\n",
    "governance['federal_grant_audit_required'].value_counts()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "##### Independent Audited Financial Statements"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 512,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0\n"
     ]
    },
    {
     "ename": "TypeError",
     "evalue": "unhashable type: 'dict'",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[0;31mTypeError\u001b[0m                                 Traceback (most recent call last)",
      "\u001b[0;32m<ipython-input-512-199086f70e01>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m()\u001b[0m\n\u001b[1;32m      3\u001b[0m                                           governance['IndependentAuditFinancialStmt'], governance['independent_audited_fs'])\n\u001b[1;32m      4\u001b[0m \u001b[0;32mprint\u001b[0m \u001b[0mlen\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mgovernance\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0mgovernance\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;34m'independent_audited_fs'\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0misnull\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m----> 5\u001b[0;31m \u001b[0;32mprint\u001b[0m \u001b[0mgovernance\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;34m'independent_audited_fs'\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mvalue_counts\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m",
      "\u001b[0;32m//anaconda/lib/python2.7/site-packages/pandas/core/base.pyc\u001b[0m in \u001b[0;36mvalue_counts\u001b[0;34m(self, normalize, sort, ascending, bins, dropna)\u001b[0m\n\u001b[1;32m    949\u001b[0m         \u001b[0;32mfrom\u001b[0m \u001b[0mpandas\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mcore\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0malgorithms\u001b[0m \u001b[0;32mimport\u001b[0m \u001b[0mvalue_counts\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    950\u001b[0m         result = value_counts(self, sort=sort, ascending=ascending,\n\u001b[0;32m--> 951\u001b[0;31m                               normalize=normalize, bins=bins, dropna=dropna)\n\u001b[0m\u001b[1;32m    952\u001b[0m         \u001b[0;32mreturn\u001b[0m \u001b[0mresult\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    953\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m//anaconda/lib/python2.7/site-packages/pandas/core/algorithms.pyc\u001b[0m in \u001b[0;36mvalue_counts\u001b[0;34m(values, sort, ascending, normalize, bins, dropna)\u001b[0m\n\u001b[1;32m    376\u001b[0m     \u001b[0;32melse\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    377\u001b[0m         \u001b[0;31m# ndarray path. pass original to handle DatetimeTzBlock\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 378\u001b[0;31m         \u001b[0mkeys\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mcounts\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0m_value_counts_arraylike\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mvalues\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mdropna\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mdropna\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    379\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    380\u001b[0m         \u001b[0;32mfrom\u001b[0m \u001b[0mpandas\u001b[0m \u001b[0;32mimport\u001b[0m \u001b[0mIndex\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mSeries\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m//anaconda/lib/python2.7/site-packages/pandas/core/algorithms.pyc\u001b[0m in \u001b[0;36m_value_counts_arraylike\u001b[0;34m(values, dropna)\u001b[0m\n\u001b[1;32m    444\u001b[0m         \u001b[0mvalues\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0m_ensure_object\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mvalues\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    445\u001b[0m         \u001b[0mmask\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0misnull\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mvalues\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 446\u001b[0;31m         \u001b[0mkeys\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mcounts\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mhtable\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mvalue_count_object\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mvalues\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mmask\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    447\u001b[0m         \u001b[0;32mif\u001b[0m \u001b[0;32mnot\u001b[0m \u001b[0mdropna\u001b[0m \u001b[0;32mand\u001b[0m \u001b[0mmask\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0many\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    448\u001b[0m             \u001b[0mkeys\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mnp\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0minsert\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mkeys\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;36m0\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mnp\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mNaN\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32mpandas/hashtable.pyx\u001b[0m in \u001b[0;36mpandas.hashtable.value_count_object (pandas/hashtable.c:21313)\u001b[0;34m()\u001b[0m\n",
      "\u001b[0;32mpandas/hashtable.pyx\u001b[0m in \u001b[0;36mpandas.hashtable.value_count_object (pandas/hashtable.c:21048)\u001b[0;34m()\u001b[0m\n",
      "\u001b[0;31mTypeError\u001b[0m: unhashable type: 'dict'"
     ]
    }
   ],
   "source": [
    "governance['independent_audited_fs'] = governance['IndependentAuditFinclStmtInd']\n",
    "governance['independent_audited_fs'] = np.where(governance['independent_audited_fs'].isnull(),\n",
    "                                          governance['IndependentAuditFinancialStmt'], governance['independent_audited_fs'])\n",
    "print len(governance[governance['independent_audited_fs'].isnull()])\n",
    "print governance['independent_audited_fs'].value_counts()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 517,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0"
      ]
     },
     "execution_count": 517,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(governance[governance['independent_audited_fs'].isnull()])"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "<br>GIVEN ABOVE ERROR, I WILL HAVE TO EXTRACT THE RIGHT SUB-VARIABLE FROM THE COLUMN\n",
    "- NOTE: WHEN CHECKING VARIABLE TYPE DO NOT PUT 'DICT' IN QUOTES"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 518,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "('Index: ', 9, 'Independent audited F/S:', u'1') \n",
      "\n",
      "# of minutes:  0.000384934743245 \n",
      "\n",
      "\n"
     ]
    }
   ],
   "source": [
    "from IPython.display import display, clear_output   ##### FOR USE WITH STDOUT (DYNAMIC, SINGLE-LINE PRINTING)\n",
    "\n",
    "import timeit\n",
    "start_time = timeit.default_timer()\n",
    "\n",
    "for index, row in governance[:10].iterrows():\n",
    "    #bond = 0\n",
    "    if type(row['independent_audited_fs'])==dict and '#text' in row['independent_audited_fs']:\n",
    "        #print 'looking'\n",
    "        governance.ix[index, 'independent_audited_fs'] = row['independent_audited_fs']['#text']\n",
    "        \n",
    "    clear_output()\n",
    "    print ('Index: ', index, 'Independent audited F/S:', governance.ix[index, 'independent_audited_fs']), '\\n'\n",
    "    sys.stdout.flush()        \n",
    "        \n",
    "        \n",
    "elapsed = timeit.default_timer() - start_time\n",
    "print '# of minutes: ', elapsed/60, '\\n', '\\n'         "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 519,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>EIN</th>\n",
       "      <th>FYE</th>\n",
       "      <th>VotingMembersGoverningBodyCnt</th>\n",
       "      <th>NbrVotingMembersGoverningBody</th>\n",
       "      <th>VotingMembersIndependentCnt</th>\n",
       "      <th>NbrIndependentVotingMembers</th>\n",
       "      <th>DelegationOfMgmtDutiesInd</th>\n",
       "      <th>DelegationOfManagementDuties</th>\n",
       "      <th>Form990ProvidedToGvrnBodyInd</th>\n",
       "      <th>Form990ProvidedToGoverningBody</th>\n",
       "      <th>AuditCommitteeInd</th>\n",
       "      <th>AuditCommittee</th>\n",
       "      <th>perm_rest_assets</th>\n",
       "      <th>temp_rest_assets</th>\n",
       "      <th>net_assets</th>\n",
       "      <th>donor_restrictions</th>\n",
       "      <th>TaxExemptBondsInd</th>\n",
       "      <th>TaxExemptBonds</th>\n",
       "      <th>NetUnrelatedBusTxblIncmAmt</th>\n",
       "      <th>NetUnrelatedBusinessTxblIncome</th>\n",
       "      <th>FSAuditedInd</th>\n",
       "      <th>FSAudited</th>\n",
       "      <th>FederalGrantAuditPerformedInd</th>\n",
       "      <th>FederalGrantAuditPerformed</th>\n",
       "      <th>FederalGrantAuditRequiredInd</th>\n",
       "      <th>FederalGrantAuditRequired</th>\n",
       "      <th>IndependentAuditFinclStmtInd</th>\n",
       "      <th>IndependentAuditFinancialStmt</th>\n",
       "      <th>FeesForServicesAccounting_binary</th>\n",
       "      <th>FamilyOrBusinessRlnInd</th>\n",
       "      <th>FamilyOrBusinessRelationship</th>\n",
       "      <th>TaxExemptBondLiabilitiesGrp</th>\n",
       "      <th>TaxExemptBondLiabilities</th>\n",
       "      <th>GovernmentGrantsAmt</th>\n",
       "      <th>GovernmentGrants</th>\n",
       "      <th>ElectionOfBoardMembersInd</th>\n",
       "      <th>ElectionOfBoardMembers</th>\n",
       "      <th>independent_directors_num</th>\n",
       "      <th>voting_directors</th>\n",
       "      <th>independent_directors_pct</th>\n",
       "      <th>independent_directors_pct_v2</th>\n",
       "      <th>outsourced_mgt</th>\n",
       "      <th>990_review</th>\n",
       "      <th>audit_committee</th>\n",
       "      <th>restricted_donations</th>\n",
       "      <th>tax_exempt_bond</th>\n",
       "      <th>taxable_revenue</th>\n",
       "      <th>taxable_revenue_binary</th>\n",
       "      <th>taxable_revenue_neg_or_pos_rev_binary</th>\n",
       "      <th>audited_financials</th>\n",
       "      <th>federal_grant_audit_performed</th>\n",
       "      <th>federal_grant_audit_required</th>\n",
       "      <th>independent_audited_fs</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>010202467</td>\n",
       "      <td>FY2014</td>\n",
       "      <td>22</td>\n",
       "      <td>20</td>\n",
       "      <td>21</td>\n",
       "      <td>20</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>4433997</td>\n",
       "      <td>7351259</td>\n",
       "      <td>23690097</td>\n",
       "      <td>0.50</td>\n",
       "      <td>{u'#text': u'1', u'@referenceDocumentId': u'RetDoc1053100001'}</td>\n",
       "      <td>{u'#text': u'1', u'@referenceDocumentId': u'RetDoc1053100001'}</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>{u'#text': u'1', u'@referenceDocumentId': u'RetDoc1040000001'}</td>\n",
       "      <td>{u'#text': u'1', u'@referenceDocumentId': u'RetDoc1040000001'}</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>7932282</td>\n",
       "      <td>7295185</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>21</td>\n",
       "      <td>22</td>\n",
       "      <td>0.95</td>\n",
       "      <td>0.95</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "         EIN     FYE VotingMembersGoverningBodyCnt  \\\n",
       "0  010202467  FY2014                            22   \n",
       "\n",
       "  NbrVotingMembersGoverningBody VotingMembersIndependentCnt  \\\n",
       "0                            20                          21   \n",
       "\n",
       "  NbrIndependentVotingMembers DelegationOfMgmtDutiesInd  \\\n",
       "0                          20                         1   \n",
       "\n",
       "  DelegationOfManagementDuties Form990ProvidedToGvrnBodyInd  \\\n",
       "0                            1                            1   \n",
       "\n",
       "  Form990ProvidedToGoverningBody AuditCommitteeInd AuditCommittee  \\\n",
       "0                              0                 1              1   \n",
       "\n",
       "   perm_rest_assets  temp_rest_assets  net_assets  donor_restrictions  \\\n",
       "0           4433997           7351259    23690097                0.50   \n",
       "\n",
       "                                                TaxExemptBondsInd  \\\n",
       "0  {u'#text': u'1', u'@referenceDocumentId': u'RetDoc1053100001'}   \n",
       "\n",
       "                                                   TaxExemptBonds  \\\n",
       "0  {u'#text': u'1', u'@referenceDocumentId': u'RetDoc1053100001'}   \n",
       "\n",
       "  NetUnrelatedBusTxblIncmAmt NetUnrelatedBusinessTxblIncome FSAuditedInd  \\\n",
       "0                          0                              0            1   \n",
       "\n",
       "  FSAudited FederalGrantAuditPerformedInd FederalGrantAuditPerformed  \\\n",
       "0         1                             1                          1   \n",
       "\n",
       "  FederalGrantAuditRequiredInd FederalGrantAuditRequired  \\\n",
       "0                            1                         1   \n",
       "\n",
       "                                     IndependentAuditFinclStmtInd  \\\n",
       "0  {u'#text': u'1', u'@referenceDocumentId': u'RetDoc1040000001'}   \n",
       "\n",
       "                                    IndependentAuditFinancialStmt  \\\n",
       "0  {u'#text': u'1', u'@referenceDocumentId': u'RetDoc1040000001'}   \n",
       "\n",
       "   FeesForServicesAccounting_binary FamilyOrBusinessRlnInd  \\\n",
       "0                                 1                      0   \n",
       "\n",
       "  FamilyOrBusinessRelationship TaxExemptBondLiabilitiesGrp  \\\n",
       "0                            1                         NaN   \n",
       "\n",
       "  TaxExemptBondLiabilities GovernmentGrantsAmt GovernmentGrants  \\\n",
       "0                      NaN             7932282          7295185   \n",
       "\n",
       "  ElectionOfBoardMembersInd ElectionOfBoardMembers  independent_directors_num  \\\n",
       "0                         1                      1                         21   \n",
       "\n",
       "   voting_directors  independent_directors_pct  independent_directors_pct_v2  \\\n",
       "0                22                       0.95                          0.95   \n",
       "\n",
       "   outsourced_mgt  990_review  audit_committee  restricted_donations  \\\n",
       "0               1           1                1                     1   \n",
       "\n",
       "   tax_exempt_bond  taxable_revenue  taxable_revenue_binary  \\\n",
       "0                1                0                       0   \n",
       "\n",
       "   taxable_revenue_neg_or_pos_rev_binary audited_financials  \\\n",
       "0                                      0                  1   \n",
       "\n",
       "  federal_grant_audit_performed federal_grant_audit_required  \\\n",
       "0                             1                            1   \n",
       "\n",
       "  independent_audited_fs  \n",
       "0                      1  "
      ]
     },
     "execution_count": 519,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "governance[:1]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 520,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "1        3503\n",
       "true     2050\n",
       "0         954\n",
       "false     626\n",
       "Name: independent_audited_fs, dtype: int64"
      ]
     },
     "execution_count": 520,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "governance['independent_audited_fs'].value_counts()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 521,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "7133\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "1    5553\n",
       "0    1580\n",
       "Name: independent_audited_fs, dtype: int64"
      ]
     },
     "execution_count": 521,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "governance['independent_audited_fs'] = np.where( governance['independent_audited_fs']=='true', 1, governance['independent_audited_fs'] )\n",
    "governance['independent_audited_fs'] = np.where( governance['independent_audited_fs']=='1', 1, governance['independent_audited_fs'] )\n",
    "governance['independent_audited_fs'] = np.where( governance['independent_audited_fs']=='false', 0, governance['independent_audited_fs'] )\n",
    "governance['independent_audited_fs'] = np.where( governance['independent_audited_fs']=='0', 0, governance['independent_audited_fs'] )\n",
    "print len(governance[governance['independent_audited_fs'].notnull()])\n",
    "governance['independent_audited_fs'].value_counts()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "##### Family Business Relations"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 522,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0\n",
      "0        3461\n",
      "false    2092\n",
      "1         996\n",
      "true      584\n",
      "Name: no_relations, dtype: int64\n"
     ]
    }
   ],
   "source": [
    "governance['no_relations'] = governance['FamilyOrBusinessRlnInd']\n",
    "governance['no_relations'] = np.where(governance['no_relations'].isnull(),\n",
    "                                          governance['FamilyOrBusinessRelationship'], governance['no_relations'])\n",
    "print len(governance[governance['no_relations'].isnull()])\n",
    "print governance['no_relations'].value_counts()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 523,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "7133\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "0    5553\n",
       "1    1580\n",
       "Name: no_relations, dtype: int64"
      ]
     },
     "execution_count": 523,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "governance['no_relations'] = np.where( governance['no_relations']=='true', 1, governance['no_relations'] )\n",
    "governance['no_relations'] = np.where( governance['no_relations']=='1', 1, governance['no_relations'] )\n",
    "governance['no_relations'] = np.where( governance['no_relations']=='false', 0, governance['no_relations'] )\n",
    "governance['no_relations'] = np.where( governance['no_relations']=='0', 0, governance['no_relations'] )\n",
    "print len(governance[governance['no_relations'].notnull()])\n",
    "governance['no_relations'].value_counts()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "##### Election of Board Members"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 525,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0\n",
      "0        3843\n",
      "false    2342\n",
      "1         614\n",
      "true      334\n",
      "Name: elected_board, dtype: int64\n"
     ]
    }
   ],
   "source": [
    "governance['elected_board'] = governance['ElectionOfBoardMembersInd']\n",
    "governance['elected_board'] = np.where(governance['elected_board'].isnull(),\n",
    "                                          governance['ElectionOfBoardMembers'], governance['elected_board'])\n",
    "print len(governance[governance['elected_board'].isnull()])\n",
    "print governance['elected_board'].value_counts() "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 526,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "7133\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "0    6185\n",
       "1     948\n",
       "Name: elected_board, dtype: int64"
      ]
     },
     "execution_count": 526,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "governance['elected_board'] = np.where( governance['elected_board']=='true', 1, governance['elected_board'] )\n",
    "governance['elected_board'] = np.where( governance['elected_board']=='1', 1, governance['elected_board'] )\n",
    "governance['elected_board'] = np.where( governance['elected_board']=='false', 0, governance['elected_board'] )\n",
    "governance['elected_board'] = np.where( governance['elected_board']=='0', 0, governance['elected_board'] )\n",
    "print len(governance[governance['elected_board'].notnull()])\n",
    "governance['elected_board'].value_counts()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "##### Government Grant"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "governance[['GovernmentGrantsAmt', ]]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 537,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "3327\n",
      "count     3806\n",
      "unique    3559\n",
      "top          0\n",
      "freq        86\n",
      "Name: government_grant, dtype: object \n",
      "\n",
      "0\n",
      "count     7133\n",
      "unique    3560\n",
      "top          0\n",
      "freq      3327\n",
      "Name: government_grant, dtype: int64 \n",
      "\n",
      "0\n",
      "1    3806\n",
      "0    3327\n",
      "Name: government_grant, dtype: int64 \n",
      "\n"
     ]
    }
   ],
   "source": [
    "governance['government_grant'] = governance['GovernmentGrantsAmt']\n",
    "governance['government_grant'] = np.where(governance['government_grant'].isnull(),\n",
    "                                          governance['GovernmentGrants'], governance['government_grant'])\n",
    "print len(governance[governance['government_grant'].isnull()])\n",
    "print governance['government_grant'].describe(), '\\n'\n",
    "governance['government_grant'] = np.where(governance['government_grant'].isnull(), 0,governance['government_grant'] )\n",
    "print len(governance[governance['government_grant'].isnull()])\n",
    "print governance['government_grant'].describe(), '\\n'\n",
    "governance['government_grant'] = np.where(governance['government_grant']>0, 1,0)\n",
    "print len(governance[governance['government_grant'].isnull()])\n",
    "print governance['government_grant'].value_counts(), '\\n'"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "##### Zero FR Expenses"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 555,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0 \n",
      "\n",
      "6434 \n",
      "\n",
      "7133\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "0         86\n",
       "140183     2\n",
       "221765     2\n",
       "135672     2\n",
       "121804     2\n",
       "Name: zero_FR, dtype: int64"
      ]
     },
     "execution_count": 555,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "governance2['zero_FR'] = np.nan\n",
    "print len(governance2[governance2['zero_FR'].notnull()]), '\\n'\n",
    "governance2['zero_FR'] = np.where(governance2['TotalFundrsngExpCurrentYear'].notnull(), \n",
    "                                 governance2['TotalFundrsngExpCurrentYear'], \n",
    "                                   governance2['zero_FR'] )\n",
    "print len(governance2[governance2['zero_FR'].notnull()]), '\\n'\n",
    "governance2['zero_FR'] = np.where( ((governance2['zero_FR'].isnull()) & \n",
    "                                      (governance2['CYTotalFundraisingExpenseAmt'].notnull())), \n",
    "                         governance2['CYTotalFundraisingExpenseAmt'], governance2['zero_FR'] )\n",
    "print len(governance2[governance2['zero_FR'].notnull()])\n",
    "governance2['zero_FR'].value_counts()[:5]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 558,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0    7047\n",
       "1      86\n",
       "Name: zero_FR, dtype: int64"
      ]
     },
     "execution_count": 558,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "governance2['zero_FR'] = np.where(governance2['zero_FR']=='0', 1,0)\n",
    "governance2['zero_FR'].value_counts()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "##### Save DF"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 539,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "governance.to_pickle('governance variables including original columns (n=7,133).pkl')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "##### Create subset of columns in preparation for merge"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 540,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "['EIN', 'FYE', u'VotingMembersGoverningBodyCnt', u'NbrVotingMembersGoverningBody', u'VotingMembersIndependentCnt', u'NbrIndependentVotingMembers', u'DelegationOfMgmtDutiesInd', u'DelegationOfManagementDuties', u'Form990ProvidedToGvrnBodyInd', u'Form990ProvidedToGoverningBody', u'AuditCommitteeInd', u'AuditCommittee', 'perm_rest_assets', 'temp_rest_assets', 'net_assets', 'donor_restrictions', u'TaxExemptBondsInd', u'TaxExemptBonds', u'NetUnrelatedBusTxblIncmAmt', u'NetUnrelatedBusinessTxblIncome', u'FSAuditedInd', u'FSAudited', u'FederalGrantAuditPerformedInd', u'FederalGrantAuditPerformed', u'FederalGrantAuditRequiredInd', u'FederalGrantAuditRequired', u'IndependentAuditFinclStmtInd', u'IndependentAuditFinancialStmt', 'FeesForServicesAccounting_binary', u'FamilyOrBusinessRlnInd', u'FamilyOrBusinessRelationship', u'TaxExemptBondLiabilitiesGrp', u'TaxExemptBondLiabilities', u'GovernmentGrantsAmt', u'GovernmentGrants', u'ElectionOfBoardMembersInd', u'ElectionOfBoardMembers', 'independent_directors_num', 'voting_directors', 'independent_directors_pct', 'independent_directors_pct_v2', 'outsourced_mgt', '990_review', 'audit_committee', 'restricted_donations', 'tax_exempt_bond', 'taxable_revenue', 'taxable_revenue_binary', 'taxable_revenue_neg_or_pos_rev_binary', 'audited_financials', 'federal_grant_audit_performed', 'federal_grant_audit_required', 'independent_audited_fs', 'no_relations', 'elected_board', 'government_grant']\n"
     ]
    }
   ],
   "source": [
    "print governance.columns.tolist()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 542,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "governance_cols = ['EIN', 'FYE', \n",
    "                   'independent_directors_num', 'independent_directors_pct', 'independent_directors_pct_v2',\n",
    "                   'voting_directors',\n",
    "                   'outsourced_mgt', '990_review', 'audit_committee', \n",
    "                   'perm_rest_assets', 'temp_rest_assets', 'net_assets', 'donor_restrictions', \n",
    "                   'restricted_donations',\n",
    "                   'tax_exempt_bond', 'taxable_revenue', 'taxable_revenue_binary', \n",
    "                   'taxable_revenue_neg_or_pos_rev_binary', \n",
    "                   'audited_financials', 'independent_audited_fs', \n",
    "                   'federal_grant_audit_performed', 'federal_grant_audit_required', \n",
    "                   'FeesForServicesAccounting_binary',\n",
    "                   'no_relations', \n",
    "                   'elected_board', \n",
    "                   'government_grant',\n",
    "                    ]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Merge with Model 4 Data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 394,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Number of columns: 35\n",
      "Number of observations: 8304\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>donor_advisory</th>\n",
       "      <th>donor_advisory_2016</th>\n",
       "      <th>donor_advisory_2011_to_2016</th>\n",
       "      <th>org_id</th>\n",
       "      <th>EIN</th>\n",
       "      <th>FYE</th>\n",
       "      <th>Form 990 FYE</th>\n",
       "      <th>ratings_system</th>\n",
       "      <th>2011_data</th>\n",
       "      <th>2016_data</th>\n",
       "      <th>conflict_of_interest_policy_v2</th>\n",
       "      <th>records_retention_policy_v2</th>\n",
       "      <th>whistleblower_policy_v2</th>\n",
       "      <th>SOX_policies</th>\n",
       "      <th>SOX_policies_binary</th>\n",
       "      <th>SOX_policies_all_binary</th>\n",
       "      <th>program_efficiency</th>\n",
       "      <th>complexity</th>\n",
       "      <th>complexity_2011</th>\n",
       "      <th>age</th>\n",
       "      <th>total_revenue_logged</th>\n",
       "      <th>category</th>\n",
       "      <th>state</th>\n",
       "      <th>tot_rev</th>\n",
       "      <th>category_Animals</th>\n",
       "      <th>category_Arts, Culture, Humanities</th>\n",
       "      <th>category_Community Development</th>\n",
       "      <th>category_Education</th>\n",
       "      <th>category_Environment</th>\n",
       "      <th>category_Health</th>\n",
       "      <th>category_Human Services</th>\n",
       "      <th>category_Human and Civil Rights</th>\n",
       "      <th>category_International</th>\n",
       "      <th>category_Religion</th>\n",
       "      <th>category_Research and Public Policy</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>50709</th>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>5954</td>\n",
       "      <td>010202467</td>\n",
       "      <td>FY2014</td>\n",
       "      <td>2014-12</td>\n",
       "      <td>CN 2.1</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>3.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.794457</td>\n",
       "      <td>6.0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>62.0</td>\n",
       "      <td>16.377993</td>\n",
       "      <td>Research and Public Policy</td>\n",
       "      <td>ME</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "       donor_advisory  donor_advisory_2016  donor_advisory_2011_to_2016  \\\n",
       "50709             0.0                  0.0                          0.0   \n",
       "\n",
       "      org_id        EIN     FYE Form 990 FYE ratings_system  2011_data  \\\n",
       "50709   5954  010202467  FY2014      2014-12         CN 2.1        0.0   \n",
       "\n",
       "       2016_data  conflict_of_interest_policy_v2  records_retention_policy_v2  \\\n",
       "50709        1.0                             1.0                          1.0   \n",
       "\n",
       "       whistleblower_policy_v2  SOX_policies  SOX_policies_binary  \\\n",
       "50709                      1.0           3.0                  1.0   \n",
       "\n",
       "       SOX_policies_all_binary  program_efficiency  complexity  \\\n",
       "50709                      1.0            0.794457         6.0   \n",
       "\n",
       "       complexity_2011   age  total_revenue_logged  \\\n",
       "50709              NaN  62.0             16.377993   \n",
       "\n",
       "                         category state  tot_rev  category_Animals  \\\n",
       "50709  Research and Public Policy    ME      NaN               0.0   \n",
       "\n",
       "       category_Arts, Culture, Humanities  category_Community Development  \\\n",
       "50709                                 0.0                             0.0   \n",
       "\n",
       "       category_Education  category_Environment  category_Health  \\\n",
       "50709                 0.0                   0.0              0.0   \n",
       "\n",
       "       category_Human Services  category_Human and Civil Rights  \\\n",
       "50709                      0.0                              0.0   \n",
       "\n",
       "       category_International  category_Religion  \\\n",
       "50709                     0.0                0.0   \n",
       "\n",
       "       category_Research and Public Policy  \n",
       "50709                                  1.0  "
      ]
     },
     "execution_count": 394,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df = pd.read_pickle('2016 - Test 4 data.pkl')\n",
    "print \"Number of columns:\", len(df.columns)\n",
    "print \"Number of observations:\", len(df)\n",
    "df.head(1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 543,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Number of columns: 39\n",
      "Number of observations: 8238\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Col1</th>\n",
       "      <th>A2011_data</th>\n",
       "      <th>A2016_data</th>\n",
       "      <th>ein</th>\n",
       "      <th>fye</th>\n",
       "      <th>Form_990_FYE</th>\n",
       "      <th>SOX_policies</th>\n",
       "      <th>SOX_policies_all_binary</th>\n",
       "      <th>SOX_policies_binary</th>\n",
       "      <th>age</th>\n",
       "      <th>category</th>\n",
       "      <th>category_Animals</th>\n",
       "      <th>category_Arts__Culture__Humaniti</th>\n",
       "      <th>category_Community_Development</th>\n",
       "      <th>category_Education</th>\n",
       "      <th>category_Environment</th>\n",
       "      <th>category_Health</th>\n",
       "      <th>category_Human_Services</th>\n",
       "      <th>category_Human_and_Civil_Rights</th>\n",
       "      <th>category_International</th>\n",
       "      <th>category_Religion</th>\n",
       "      <th>category_Research_and_Public_Pol</th>\n",
       "      <th>complexity</th>\n",
       "      <th>complexity_2011</th>\n",
       "      <th>conflict_of_interest_policy_v2</th>\n",
       "      <th>donor_advisory</th>\n",
       "      <th>donor_advisory_2011_to_2016</th>\n",
       "      <th>donor_advisory_2016</th>\n",
       "      <th>org_id</th>\n",
       "      <th>program_efficiency</th>\n",
       "      <th>ratings_system</th>\n",
       "      <th>records_retention_policy_v2</th>\n",
       "      <th>state</th>\n",
       "      <th>tot_rev</th>\n",
       "      <th>total_revenue_logged</th>\n",
       "      <th>whistleblower_policy_v2</th>\n",
       "      <th>ncategory</th>\n",
       "      <th>revs</th>\n",
       "      <th>samused_</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>50709</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>010202467</td>\n",
       "      <td>FY2014</td>\n",
       "      <td>2014-12</td>\n",
       "      <td>3.00</td>\n",
       "      <td>1.00</td>\n",
       "      <td>1.00</td>\n",
       "      <td>62.00</td>\n",
       "      <td>Research and Public Policy</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>6.00</td>\n",
       "      <td>nan</td>\n",
       "      <td>1.00</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>5954</td>\n",
       "      <td>0.79</td>\n",
       "      <td>CN 2.1</td>\n",
       "      <td>1.00</td>\n",
       "      <td>ME</td>\n",
       "      <td>nan</td>\n",
       "      <td>16.38</td>\n",
       "      <td>1.00</td>\n",
       "      <td>Research and Public Policy</td>\n",
       "      <td>12967968.00</td>\n",
       "      <td>1.00</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "    Col1  A2011_data  A2016_data        ein     fye Form_990_FYE  \\\n",
       "0  50709           0           1  010202467  FY2014      2014-12   \n",
       "\n",
       "   SOX_policies  SOX_policies_all_binary  SOX_policies_binary   age  \\\n",
       "0          3.00                     1.00                 1.00 62.00   \n",
       "\n",
       "                     category  category_Animals  \\\n",
       "0  Research and Public Policy                 0   \n",
       "\n",
       "   category_Arts__Culture__Humaniti  category_Community_Development  \\\n",
       "0                                 0                               0   \n",
       "\n",
       "   category_Education  category_Environment  category_Health  \\\n",
       "0                   0                     0                0   \n",
       "\n",
       "   category_Human_Services  category_Human_and_Civil_Rights  \\\n",
       "0                        0                                0   \n",
       "\n",
       "   category_International  category_Religion  \\\n",
       "0                       0                  0   \n",
       "\n",
       "   category_Research_and_Public_Pol  complexity  complexity_2011  \\\n",
       "0                                 1        6.00              nan   \n",
       "\n",
       "   conflict_of_interest_policy_v2  donor_advisory  \\\n",
       "0                            1.00               0   \n",
       "\n",
       "   donor_advisory_2011_to_2016  donor_advisory_2016 org_id  \\\n",
       "0                            0                    0   5954   \n",
       "\n",
       "   program_efficiency ratings_system  records_retention_policy_v2 state  \\\n",
       "0                0.79         CN 2.1                         1.00    ME   \n",
       "\n",
       "   tot_rev  total_revenue_logged  whistleblower_policy_v2  \\\n",
       "0      nan                 16.38                     1.00   \n",
       "\n",
       "                    ncategory        revs  samused_  \n",
       "0  Research and Public Policy 12967968.00      1.00  "
      ]
     },
     "execution_count": 543,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_test4 = pd.read_stata('Test 4 data.dta')\n",
    "print \"Number of columns:\", len(df_test4.columns)\n",
    "print \"Number of observations:\", len(df_test4)\n",
    "df_test4.head(1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 544,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "set(['Form 990 FYE', 'category_Human Services', 'category_Community Development', 'FYE', '2016_data', 'category_Arts, Culture, Humanities', 'category_Research and Public Policy', 'category_Human and Civil Rights', '2011_data', 'EIN']) \n",
      "\n",
      "set(['Form_990_FYE', 'samused_', 'category_Research_and_Public_Pol', 'category_Human_Services', 'A2011_data', 'fye', 'category_Human_and_Civil_Rights', 'Col1', 'category_Community_Development', 'A2016_data', 'revs', 'ein', 'category_Arts__Culture__Humaniti', 'ncategory'])\n"
     ]
    }
   ],
   "source": [
    "print set(df.columns.tolist()) - set(df_test4.columns.tolist()), '\\n'\n",
    "print set(df_test4.columns.tolist()) - set(df.columns.tolist())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 405,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "35\n",
      "8304 \n",
      "\n",
      "56\n",
      "8304 \n",
      "\n",
      "both          7133\n",
      "left_only     1171\n",
      "right_only       0\n",
      "Name: _merge, dtype: int64\n"
     ]
    }
   ],
   "source": [
    "#print len(df.columns)\n",
    "#print len(pd.merge(df, governance[governance_cols], left_on='EIN', right_on='EIN', how='left', indicator=True)), '\\n'\n",
    "#merged = pd.merge(df, governance[governance_cols], left_on='EIN', right_on='EIN', how='left', indicator=True)\n",
    "#print len(merged.columns)\n",
    "#print len(merged), '\\n'\n",
    "#print merged['_merge'].value_counts()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 545,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "35\n",
      "8238 \n",
      "\n",
      "66\n",
      "8238 \n",
      "\n",
      "both          7133\n",
      "left_only     1105\n",
      "right_only       0\n",
      "Name: _merge, dtype: int64\n"
     ]
    }
   ],
   "source": [
    "print len(df.columns)\n",
    "print len(pd.merge(df_test4, governance[governance_cols], left_on='ein', right_on='EIN', how='left', indicator=True)), '\\n'\n",
    "merged = pd.merge(df_test4, governance[governance_cols], left_on='ein', right_on='EIN', how='left', indicator=True)\n",
    "print len(merged.columns)\n",
    "print len(merged), '\\n'\n",
    "print merged['_merge'].value_counts()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 560,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "#merged = merged.drop('_merge', 1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 562,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "65\n",
      "8238 \n",
      "\n",
      "68\n",
      "8238 \n",
      "\n",
      "both          7133\n",
      "left_only     1105\n",
      "right_only       0\n",
      "Name: _merge, dtype: int64\n"
     ]
    }
   ],
   "source": [
    "'''\n",
    "print len(merged.columns)\n",
    "print len(pd.merge(merged, governance2[['EIN', 'zero_FR']], left_on='ein', right_on='EIN', how='left', indicator=True)), '\\n'\n",
    "merged = pd.merge(merged, governance2[['EIN', 'zero_FR']], left_on='ein', right_on='EIN', how='left', indicator=True)\n",
    "print len(merged.columns)\n",
    "print len(merged), '\\n'\n",
    "print merged['_merge'].value_counts()\n",
    "'''"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 563,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Col1</th>\n",
       "      <th>A2011_data</th>\n",
       "      <th>A2016_data</th>\n",
       "      <th>ein</th>\n",
       "      <th>fye</th>\n",
       "      <th>Form_990_FYE</th>\n",
       "      <th>SOX_policies</th>\n",
       "      <th>SOX_policies_all_binary</th>\n",
       "      <th>SOX_policies_binary</th>\n",
       "      <th>age</th>\n",
       "      <th>category</th>\n",
       "      <th>category_Animals</th>\n",
       "      <th>category_Arts__Culture__Humaniti</th>\n",
       "      <th>category_Community_Development</th>\n",
       "      <th>category_Education</th>\n",
       "      <th>category_Environment</th>\n",
       "      <th>category_Health</th>\n",
       "      <th>category_Human_Services</th>\n",
       "      <th>category_Human_and_Civil_Rights</th>\n",
       "      <th>category_International</th>\n",
       "      <th>category_Religion</th>\n",
       "      <th>category_Research_and_Public_Pol</th>\n",
       "      <th>complexity</th>\n",
       "      <th>complexity_2011</th>\n",
       "      <th>conflict_of_interest_policy_v2</th>\n",
       "      <th>donor_advisory</th>\n",
       "      <th>donor_advisory_2011_to_2016</th>\n",
       "      <th>donor_advisory_2016</th>\n",
       "      <th>org_id</th>\n",
       "      <th>program_efficiency</th>\n",
       "      <th>ratings_system</th>\n",
       "      <th>records_retention_policy_v2</th>\n",
       "      <th>state</th>\n",
       "      <th>tot_rev</th>\n",
       "      <th>total_revenue_logged</th>\n",
       "      <th>whistleblower_policy_v2</th>\n",
       "      <th>ncategory</th>\n",
       "      <th>revs</th>\n",
       "      <th>samused_</th>\n",
       "      <th>EIN_x</th>\n",
       "      <th>FYE</th>\n",
       "      <th>independent_directors_num</th>\n",
       "      <th>independent_directors_pct</th>\n",
       "      <th>independent_directors_pct_v2</th>\n",
       "      <th>voting_directors</th>\n",
       "      <th>outsourced_mgt</th>\n",
       "      <th>990_review</th>\n",
       "      <th>audit_committee</th>\n",
       "      <th>perm_rest_assets</th>\n",
       "      <th>temp_rest_assets</th>\n",
       "      <th>net_assets</th>\n",
       "      <th>donor_restrictions</th>\n",
       "      <th>restricted_donations</th>\n",
       "      <th>tax_exempt_bond</th>\n",
       "      <th>taxable_revenue</th>\n",
       "      <th>taxable_revenue_binary</th>\n",
       "      <th>taxable_revenue_neg_or_pos_rev_binary</th>\n",
       "      <th>audited_financials</th>\n",
       "      <th>independent_audited_fs</th>\n",
       "      <th>federal_grant_audit_performed</th>\n",
       "      <th>federal_grant_audit_required</th>\n",
       "      <th>FeesForServicesAccounting_binary</th>\n",
       "      <th>no_relations</th>\n",
       "      <th>elected_board</th>\n",
       "      <th>government_grant</th>\n",
       "      <th>EIN_y</th>\n",
       "      <th>zero_FR</th>\n",
       "      <th>_merge</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>50709</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>010202467</td>\n",
       "      <td>FY2014</td>\n",
       "      <td>2014-12</td>\n",
       "      <td>3.00</td>\n",
       "      <td>1.00</td>\n",
       "      <td>1.00</td>\n",
       "      <td>62.00</td>\n",
       "      <td>Research and Public Policy</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>6.00</td>\n",
       "      <td>nan</td>\n",
       "      <td>1.00</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>5954</td>\n",
       "      <td>0.79</td>\n",
       "      <td>CN 2.1</td>\n",
       "      <td>1.00</td>\n",
       "      <td>ME</td>\n",
       "      <td>nan</td>\n",
       "      <td>16.38</td>\n",
       "      <td>1.00</td>\n",
       "      <td>Research and Public Policy</td>\n",
       "      <td>12967968.00</td>\n",
       "      <td>1.00</td>\n",
       "      <td>010202467</td>\n",
       "      <td>FY2014</td>\n",
       "      <td>21.00</td>\n",
       "      <td>0.95</td>\n",
       "      <td>0.95</td>\n",
       "      <td>22.00</td>\n",
       "      <td>1.00</td>\n",
       "      <td>1.00</td>\n",
       "      <td>1.00</td>\n",
       "      <td>4433997.00</td>\n",
       "      <td>7351259.00</td>\n",
       "      <td>23690097.00</td>\n",
       "      <td>0.50</td>\n",
       "      <td>1.00</td>\n",
       "      <td>1.00</td>\n",
       "      <td>0.00</td>\n",
       "      <td>0.00</td>\n",
       "      <td>0.00</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1.00</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>1.00</td>\n",
       "      <td>010202467</td>\n",
       "      <td>0.00</td>\n",
       "      <td>both</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "    Col1  A2011_data  A2016_data        ein     fye Form_990_FYE  \\\n",
       "0  50709           0           1  010202467  FY2014      2014-12   \n",
       "\n",
       "   SOX_policies  SOX_policies_all_binary  SOX_policies_binary   age  \\\n",
       "0          3.00                     1.00                 1.00 62.00   \n",
       "\n",
       "                     category  category_Animals  \\\n",
       "0  Research and Public Policy                 0   \n",
       "\n",
       "   category_Arts__Culture__Humaniti  category_Community_Development  \\\n",
       "0                                 0                               0   \n",
       "\n",
       "   category_Education  category_Environment  category_Health  \\\n",
       "0                   0                     0                0   \n",
       "\n",
       "   category_Human_Services  category_Human_and_Civil_Rights  \\\n",
       "0                        0                                0   \n",
       "\n",
       "   category_International  category_Religion  \\\n",
       "0                       0                  0   \n",
       "\n",
       "   category_Research_and_Public_Pol  complexity  complexity_2011  \\\n",
       "0                                 1        6.00              nan   \n",
       "\n",
       "   conflict_of_interest_policy_v2  donor_advisory  \\\n",
       "0                            1.00               0   \n",
       "\n",
       "   donor_advisory_2011_to_2016  donor_advisory_2016 org_id  \\\n",
       "0                            0                    0   5954   \n",
       "\n",
       "   program_efficiency ratings_system  records_retention_policy_v2 state  \\\n",
       "0                0.79         CN 2.1                         1.00    ME   \n",
       "\n",
       "   tot_rev  total_revenue_logged  whistleblower_policy_v2  \\\n",
       "0      nan                 16.38                     1.00   \n",
       "\n",
       "                    ncategory        revs  samused_      EIN_x     FYE  \\\n",
       "0  Research and Public Policy 12967968.00      1.00  010202467  FY2014   \n",
       "\n",
       "   independent_directors_num  independent_directors_pct  \\\n",
       "0                      21.00                       0.95   \n",
       "\n",
       "   independent_directors_pct_v2  voting_directors  outsourced_mgt  990_review  \\\n",
       "0                          0.95             22.00            1.00        1.00   \n",
       "\n",
       "   audit_committee  perm_rest_assets  temp_rest_assets  net_assets  \\\n",
       "0             1.00        4433997.00        7351259.00 23690097.00   \n",
       "\n",
       "   donor_restrictions  restricted_donations  tax_exempt_bond  taxable_revenue  \\\n",
       "0                0.50                  1.00             1.00             0.00   \n",
       "\n",
       "   taxable_revenue_binary  taxable_revenue_neg_or_pos_rev_binary  \\\n",
       "0                    0.00                                   0.00   \n",
       "\n",
       "  audited_financials independent_audited_fs federal_grant_audit_performed  \\\n",
       "0                  1                      1                             1   \n",
       "\n",
       "  federal_grant_audit_required  FeesForServicesAccounting_binary no_relations  \\\n",
       "0                            1                              1.00            0   \n",
       "\n",
       "  elected_board  government_grant      EIN_y  zero_FR _merge  \n",
       "0             1              1.00  010202467     0.00   both  "
      ]
     },
     "execution_count": 563,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "merged[:1]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 416,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "#merged.rename(columns={'FYE_x':'FYE'}, inplace=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 418,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "pd.set_option('display.float_format', lambda x: '%.2f' % x)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "##### Descriptives for our governance variables"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 439,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Col1</th>\n",
       "      <th>A2011_data</th>\n",
       "      <th>A2016_data</th>\n",
       "      <th>ein</th>\n",
       "      <th>fye</th>\n",
       "      <th>Form_990_FYE</th>\n",
       "      <th>SOX_policies</th>\n",
       "      <th>SOX_policies_all_binary</th>\n",
       "      <th>SOX_policies_binary</th>\n",
       "      <th>age</th>\n",
       "      <th>category</th>\n",
       "      <th>category_Animals</th>\n",
       "      <th>category_Arts__Culture__Humaniti</th>\n",
       "      <th>category_Community_Development</th>\n",
       "      <th>category_Education</th>\n",
       "      <th>category_Environment</th>\n",
       "      <th>category_Health</th>\n",
       "      <th>category_Human_Services</th>\n",
       "      <th>category_Human_and_Civil_Rights</th>\n",
       "      <th>category_International</th>\n",
       "      <th>category_Religion</th>\n",
       "      <th>category_Research_and_Public_Pol</th>\n",
       "      <th>complexity</th>\n",
       "      <th>complexity_2011</th>\n",
       "      <th>conflict_of_interest_policy_v2</th>\n",
       "      <th>donor_advisory</th>\n",
       "      <th>donor_advisory_2011_to_2016</th>\n",
       "      <th>donor_advisory_2016</th>\n",
       "      <th>org_id</th>\n",
       "      <th>program_efficiency</th>\n",
       "      <th>ratings_system</th>\n",
       "      <th>records_retention_policy_v2</th>\n",
       "      <th>state</th>\n",
       "      <th>tot_rev</th>\n",
       "      <th>total_revenue_logged</th>\n",
       "      <th>whistleblower_policy_v2</th>\n",
       "      <th>ncategory</th>\n",
       "      <th>revs</th>\n",
       "      <th>EIN</th>\n",
       "      <th>FYE</th>\n",
       "      <th>independent_directors_num</th>\n",
       "      <th>independent_directors_pct</th>\n",
       "      <th>voting_directors</th>\n",
       "      <th>outsourced_mgt</th>\n",
       "      <th>990_review</th>\n",
       "      <th>audit_committee</th>\n",
       "      <th>perm_rest_assets</th>\n",
       "      <th>temp_rest_assets</th>\n",
       "      <th>net_assets</th>\n",
       "      <th>donor_restrictions</th>\n",
       "      <th>tax_exempt_bond</th>\n",
       "      <th>taxable_revenue</th>\n",
       "      <th>taxable_revenue_binary</th>\n",
       "      <th>taxable_revenue_neg_or_pos_rev_binary</th>\n",
       "      <th>audited_financials</th>\n",
       "      <th>independent_audited_fs</th>\n",
       "      <th>federal_grant_audit_performed</th>\n",
       "      <th>federal_grant_audit_required</th>\n",
       "      <th>FeesForServicesAccounting_binary</th>\n",
       "      <th>_merge</th>\n",
       "      <th>independent_directors_pct_v2</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>50709</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>010202467</td>\n",
       "      <td>FY2014</td>\n",
       "      <td>2014-12</td>\n",
       "      <td>3.00</td>\n",
       "      <td>1.00</td>\n",
       "      <td>1.00</td>\n",
       "      <td>62.00</td>\n",
       "      <td>Research and Public Policy</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>6.00</td>\n",
       "      <td>nan</td>\n",
       "      <td>1.00</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>5954</td>\n",
       "      <td>0.79</td>\n",
       "      <td>CN 2.1</td>\n",
       "      <td>1.00</td>\n",
       "      <td>ME</td>\n",
       "      <td>nan</td>\n",
       "      <td>16.38</td>\n",
       "      <td>1.00</td>\n",
       "      <td>Research and Public Policy</td>\n",
       "      <td>12967968.00</td>\n",
       "      <td>010202467</td>\n",
       "      <td>FY2014</td>\n",
       "      <td>21.00</td>\n",
       "      <td>0.95</td>\n",
       "      <td>22.00</td>\n",
       "      <td>1.00</td>\n",
       "      <td>1.00</td>\n",
       "      <td>1.00</td>\n",
       "      <td>4433997.00</td>\n",
       "      <td>7351259.00</td>\n",
       "      <td>23690097.00</td>\n",
       "      <td>0.50</td>\n",
       "      <td>1.00</td>\n",
       "      <td>0.00</td>\n",
       "      <td>0.00</td>\n",
       "      <td>0.00</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1.00</td>\n",
       "      <td>both</td>\n",
       "      <td>0.95</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "    Col1  A2011_data  A2016_data        ein     fye Form_990_FYE  \\\n",
       "0  50709           0           1  010202467  FY2014      2014-12   \n",
       "\n",
       "   SOX_policies  SOX_policies_all_binary  SOX_policies_binary   age  \\\n",
       "0          3.00                     1.00                 1.00 62.00   \n",
       "\n",
       "                     category  category_Animals  \\\n",
       "0  Research and Public Policy                 0   \n",
       "\n",
       "   category_Arts__Culture__Humaniti  category_Community_Development  \\\n",
       "0                                 0                               0   \n",
       "\n",
       "   category_Education  category_Environment  category_Health  \\\n",
       "0                   0                     0                0   \n",
       "\n",
       "   category_Human_Services  category_Human_and_Civil_Rights  \\\n",
       "0                        0                                0   \n",
       "\n",
       "   category_International  category_Religion  \\\n",
       "0                       0                  0   \n",
       "\n",
       "   category_Research_and_Public_Pol  complexity  complexity_2011  \\\n",
       "0                                 1        6.00              nan   \n",
       "\n",
       "   conflict_of_interest_policy_v2  donor_advisory  \\\n",
       "0                            1.00               0   \n",
       "\n",
       "   donor_advisory_2011_to_2016  donor_advisory_2016 org_id  \\\n",
       "0                            0                    0   5954   \n",
       "\n",
       "   program_efficiency ratings_system  records_retention_policy_v2 state  \\\n",
       "0                0.79         CN 2.1                         1.00    ME   \n",
       "\n",
       "   tot_rev  total_revenue_logged  whistleblower_policy_v2  \\\n",
       "0      nan                 16.38                     1.00   \n",
       "\n",
       "                    ncategory        revs        EIN     FYE  \\\n",
       "0  Research and Public Policy 12967968.00  010202467  FY2014   \n",
       "\n",
       "   independent_directors_num  independent_directors_pct  voting_directors  \\\n",
       "0                      21.00                       0.95             22.00   \n",
       "\n",
       "   outsourced_mgt  990_review  audit_committee  perm_rest_assets  \\\n",
       "0            1.00        1.00             1.00        4433997.00   \n",
       "\n",
       "   temp_rest_assets  net_assets  donor_restrictions  tax_exempt_bond  \\\n",
       "0        7351259.00 23690097.00                0.50             1.00   \n",
       "\n",
       "   taxable_revenue  taxable_revenue_binary  \\\n",
       "0             0.00                    0.00   \n",
       "\n",
       "   taxable_revenue_neg_or_pos_rev_binary audited_financials  \\\n",
       "0                                   0.00                  1   \n",
       "\n",
       "  independent_audited_fs federal_grant_audit_performed  \\\n",
       "0                      1                             1   \n",
       "\n",
       "  federal_grant_audit_required  FeesForServicesAccounting_binary _merge  \\\n",
       "0                            1                              1.00   both   \n",
       "\n",
       "   independent_directors_pct_v2  \n",
       "0                          0.95  "
      ]
     },
     "execution_count": 439,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "merged[:1]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "##### Save DF"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 565,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "merged.to_pickle('model 4 data with e-file governance variables (n=8,238).pkl')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 566,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "merged.to_excel('model 4 data with e-file governance variables (n=8,238).xls')"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 2",
   "language": "python",
   "name": "python2"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 2
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython2",
   "version": "2.7.12"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 0
}