{
 "metadata": {
  "name": "",
  "signature": "sha256:f205a329c7dc4533eed822a6910d29c28459c04b5f2f91135432911bbb9e7bf9"
 },
 "nbformat": 3,
 "nbformat_minor": 0,
 "worksheets": [
  {
   "cells": [
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "import pandas as pd\n",
      "import glob\n",
      "\n",
      "pj = '/Users/danielmsheehan/Desktop/data/'\n",
      "pj = '/Users/danielmsheehan/Dropbox/data/'\n",
      "\n",
      "csvList = glob.glob(pj+'tables/'+\"*.csv\") #remove _sm\n",
      "\n",
      "x = []\n",
      "\n",
      "for i in csvList:\n",
      "    i = i.replace(pj+'tables/','').replace('.csv','') #remove _sm\n",
      "    #print i\n",
      "    x.append(i)\n",
      " \n",
      "print x\n",
      "\n",
      "df_list = []\n",
      "for i in x:\n",
      "    df = pd.read_csv(pj+'output/tables/d_'+i+'.csv', dtype={'geoid':object})\n",
      "    df['geoid'] = df['geoid'].astype(str)\n",
      "    df_list.append(df)\n",
      "#print df_list    \n",
      "dfc = pd.concat(df_list)\n",
      "dfc['geoid'] = dfc['geoid'].astype(str)\n",
      "dfc['type'] = 'd'\n",
      "\n",
      "dfc.columns = ['tuid','dist_roadbed','geoid','dist_bldg_hght','type']\n",
      "print dfc.dtypes\n",
      "dfc.to_csv(pj+'output/d_2013.csv', index=False)\n",
      "\n",
      "print len(dfc.index)"
     ],
     "language": "python",
     "metadata": {},
     "outputs": [
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "['20130101', '20130102', '20130103', '20130104', '20130105', '20130106', '20130107', '20130108', '20130109', '20130110', '20130111', '20130112', '20130113', '20130114', '20130115', '20130116', '20130117', '20130118', '20130119', '20130120', '20130121', '20130122', '20130123', '20130124', '20130125', '20130126', '20130127', '20130128', '20130129', '20130130', '20130131', '20130201', '20130202', '20130203', '20130204', '20130205', '20130206', '20130207', '20130208', '20130209', '20130210', '20130211', '20130212', '20130213', '20130214', '20130215', '20130216', '20130217', '20130218', '20130219', '20130220', '20130221', '20130222', '20130223', '20130224', '20130225', '20130226', '20130227', '20130228', '20130301', '20130302', '20130303', '20130304', '20130305', '20130306', '20130307', '20130308', '20130309', '20130310', '20130311', '20130312', '20130313', '20130314', '20130315', '20130316', '20130317', '20130318', '20130319', '20130320', '20130321', '20130322', '20130323', '20130324', '20130325', '20130326', '20130327', '20130328', '20130329', '20130330', '20130331', '20130401', '20130402', '20130403', '20130404', '20130405', '20130406', '20130407', '20130408', '20130409', '20130410', '20130411', '20130412', '20130413', '20130414', '20130415', '20130416', '20130417', '20130418', '20130419', '20130420', '20130421', '20130422', '20130423', '20130424', '20130425', '20130426', '20130427', '20130428', '20130429', '20130430', '20130501', '20130502', '20130503', '20130504', '20130505', '20130506', '20130507', '20130508', '20130509', '20130510', '20130511', '20130512', '20130513', '20130514', '20130515', '20130516', '20130517', '20130518', '20130519', '20130520', '20130521', '20130522', '20130523', '20130524', '20130525', '20130526', '20130527', '20130528', '20130529', '20130530', '20130531', '20130601', '20130602', '20130603', '20130604', '20130605', '20130606', '20130607', '20130608', '20130609', '20130610', '20130611', '20130612', '20130613', '20130614', '20130615', '20130616', '20130617', '20130618', '20130619', '20130620', '20130621', '20130622', '20130623', '20130624', '20130625', '20130626', '20130627', '20130628', '20130629', '20130630', '20130701', '20130702', '20130703', '20130704', '20130705', '20130706', '20130707', '20130708', '20130709', '20130710', '20130711', '20130712', '20130713', '20130714', '20130715', '20130716', '20130717', '20130718', '20130719', '20130720', '20130721', '20130722', '20130723', '20130724', '20130725', '20130726', '20130727', '20130728', '20130729', '20130730', '20130731', '20130801', '20130802', '20130803', '20130804', '20130805', '20130806', '20130807', '20130808', '20130809', '20130810', '20130811', '20130812', '20130813', '20130814', '20130815', '20130816', '20130817', '20130818', '20130819', '20130820', '20130821', '20130822', '20130823', '20130824', '20130825', '20130826', '20130827', '20130828', '20130829', '20130830', '20130831', '20130901', '20130902', '20130903', '20130904', '20130905', '20130906', '20130907', '20130908', '20130909', '20130910', '20130911', '20130912', '20130913', '20130914', '20130915', '20130916', '20130917', '20130918', '20130919', '20130920', '20130921', '20130922', '20130923', '20130924', '20130925', '20130926', '20130927', '20130928', '20130929', '20130930', '20131001', '20131002', '20131003', '20131004', '20131005', '20131006', '20131007', '20131008', '20131009', '20131010', '20131011', '20131012', '20131013', '20131014', '20131015', '20131016', '20131017', '20131018', '20131019', '20131020', '20131021', '20131022', '20131023', '20131024', '20131025', '20131026', '20131027', '20131028', '20131029', '20131030', '20131031', '20131101', '20131102', '20131103', '20131104', '20131105', '20131106', '20131107', '20131108', '20131109', '20131110', '20131111', '20131112', '20131113', '20131114', '20131115', '20131116', '20131117', '20131118', '20131119', '20131120', '20131121', '20131122', '20131123', '20131124', '20131125', '20131126', '20131127', '20131128', '20131129', '20131130', '20131201', '20131202', '20131203', '20131204', '20131205', '20131206', '20131207', '20131208', '20131209', '20131210', '20131211', '20131212', '20131213', '20131214', '20131215', '20131216', '20131217', '20131218', '20131219', '20131220', '20131221', '20131222', '20131223', '20131224', '20131225', '20131226', '20131227', '20131228', '20131229', '20131230', '20131231']\n",
        "tuid                                     int64\n",
        "dist_roadbed_dropoff_feet              float64\n",
        "geoid                                   object\n",
        "building_block_int_dis_tbl_bulkdens    float64\n",
        "dtype: object"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\n"
       ]
      }
     ],
     "prompt_number": 3
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "%reset"
     ],
     "language": "python",
     "metadata": {},
     "outputs": [
      {
       "name": "stdout",
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "Once deleted, variables cannot be recovered. Proceed (y/[n])? y\n"
       ]
      }
     ],
     "prompt_number": 4
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "import pandas as pd\n",
      "import glob\n",
      "\n",
      "pj = '/Users/danielmsheehan/Desktop/data/'\n",
      "pj = '/Users/danielmsheehan/Dropbox/data/'\n",
      "\n",
      "csvList = glob.glob(pj+'tables/'+\"*.csv\") #remove _sm\n",
      "\n",
      "x = []\n",
      "\n",
      "for i in csvList:\n",
      "    i = i.replace(pj+'tables/','').replace('.csv','') #remove _sm\n",
      "    #print i\n",
      "    x.append(i)\n",
      " \n",
      "print x\n",
      "\n",
      "df_list = []\n",
      "for i in x:\n",
      "    df = pd.read_csv(pj+'output/tables/p_'+i+'.csv', dtype={'geoid':object})\n",
      "    df['geoid'] = df['geoid'].astype(str)\n",
      "    df_list.append(df)\n",
      "#print df_list    \n",
      "dfc = pd.concat(df_list)\n",
      "dfc['geoid'] = dfc['geoid'].astype(str)\n",
      "dfc['type'] = 'p'\n",
      "\n",
      "dfc.columns = ['tuid','dist_roadbed','geoid','dist_bldg_hght','type']\n",
      "print dfc.dtypes\n",
      "dfc.to_csv(pj+'output/p_2013.csv', index=False)\n",
      "\n",
      "print len(dfc.index)"
     ],
     "language": "python",
     "metadata": {},
     "outputs": [
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "['20130101', '20130102', '20130103', '20130104', '20130105', '20130106', '20130107', '20130108', '20130109', '20130110', '20130111', '20130112', '20130113', '20130114', '20130115', '20130116', '20130117', '20130118', '20130119', '20130120', '20130121', '20130122', '20130123', '20130124', '20130125', '20130126', '20130127', '20130128', '20130129', '20130130', '20130131', '20130201', '20130202', '20130203', '20130204', '20130205', '20130206', '20130207', '20130208', '20130209', '20130210', '20130211', '20130212', '20130213', '20130214', '20130215', '20130216', '20130217', '20130218', '20130219', '20130220', '20130221', '20130222', '20130223', '20130224', '20130225', '20130226', '20130227', '20130228', '20130301', '20130302', '20130303', '20130304', '20130305', '20130306', '20130307', '20130308', '20130309', '20130310', '20130311', '20130312', '20130313', '20130314', '20130315', '20130316', '20130317', '20130318', '20130319', '20130320', '20130321', '20130322', '20130323', '20130324', '20130325', '20130326', '20130327', '20130328', '20130329', '20130330', '20130331', '20130401', '20130402', '20130403', '20130404', '20130405', '20130406', '20130407', '20130408', '20130409', '20130410', '20130411', '20130412', '20130413', '20130414', '20130415', '20130416', '20130417', '20130418', '20130419', '20130420', '20130421', '20130422', '20130423', '20130424', '20130425', '20130426', '20130427', '20130428', '20130429', '20130430', '20130501', '20130502', '20130503', '20130504', '20130505', '20130506', '20130507', '20130508', '20130509', '20130510', '20130511', '20130512', '20130513', '20130514', '20130515', '20130516', '20130517', '20130518', '20130519', '20130520', '20130521', '20130522', '20130523', '20130524', '20130525', '20130526', '20130527', '20130528', '20130529', '20130530', '20130531', '20130601', '20130602', '20130603', '20130604', '20130605', '20130606', '20130607', '20130608', '20130609', '20130610', '20130611', '20130612', '20130613', '20130614', '20130615', '20130616', '20130617', '20130618', '20130619', '20130620', '20130621', '20130622', '20130623', '20130624', '20130625', '20130626', '20130627', '20130628', '20130629', '20130630', '20130701', '20130702', '20130703', '20130704', '20130705', '20130706', '20130707', '20130708', '20130709', '20130710', '20130711', '20130712', '20130713', '20130714', '20130715', '20130716', '20130717', '20130718', '20130719', '20130720', '20130721', '20130722', '20130723', '20130724', '20130725', '20130726', '20130727', '20130728', '20130729', '20130730', '20130731', '20130801', '20130802', '20130803', '20130804', '20130805', '20130806', '20130807', '20130808', '20130809', '20130810', '20130811', '20130812', '20130813', '20130814', '20130815', '20130816', '20130817', '20130818', '20130819', '20130820', '20130821', '20130822', '20130823', '20130824', '20130825', '20130826', '20130827', '20130828', '20130829', '20130830', '20130831', '20130901', '20130902', '20130903', '20130904', '20130905', '20130906', '20130907', '20130908', '20130909', '20130910', '20130911', '20130912', '20130913', '20130914', '20130915', '20130916', '20130917', '20130918', '20130919', '20130920', '20130921', '20130922', '20130923', '20130924', '20130925', '20130926', '20130927', '20130928', '20130929', '20130930', '20131001', '20131002', '20131003', '20131004', '20131005', '20131006', '20131007', '20131008', '20131009', '20131010', '20131011', '20131012', '20131013', '20131014', '20131015', '20131016', '20131017', '20131018', '20131019', '20131020', '20131021', '20131022', '20131023', '20131024', '20131025', '20131026', '20131027', '20131028', '20131029', '20131030', '20131031', '20131101', '20131102', '20131103', '20131104', '20131105', '20131106', '20131107', '20131108', '20131109', '20131110', '20131111', '20131112', '20131113', '20131114', '20131115', '20131116', '20131117', '20131118', '20131119', '20131120', '20131121', '20131122', '20131123', '20131124', '20131125', '20131126', '20131127', '20131128', '20131129', '20131130', '20131201', '20131202', '20131203', '20131204', '20131205', '20131206', '20131207', '20131208', '20131209', '20131210', '20131211', '20131212', '20131213', '20131214', '20131215', '20131216', '20131217', '20131218', '20131219', '20131220', '20131221', '20131222', '20131223', '20131224', '20131225', '20131226', '20131227', '20131228', '20131229', '20131230', '20131231']\n",
        "tuid                                     int64\n",
        "dist_roadbed_pickup_feet               float64\n",
        "geoid                                   object\n",
        "building_block_int_dis_tbl_bulkdens    float64\n",
        "type                                    object\n",
        "dtype: object"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\n",
        "173200962"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\n"
       ]
      }
     ],
     "prompt_number": 5
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "%reset"
     ],
     "language": "python",
     "metadata": {},
     "outputs": [
      {
       "name": "stdout",
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "Once deleted, variables cannot be recovered. Proceed (y/[n])? y\n"
       ]
      }
     ],
     "prompt_number": 6
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "import pandas as pd\n",
      "import glob\n",
      "\n",
      "pj = '/Users/danielmsheehan/Desktop/data/'\n",
      "pj = '/Users/danielmsheehan/Dropbox/data/'\n",
      "\n",
      "csvList = glob.glob(pj+'output/'+\"*.csv\") #remove _sm\n",
      "\n",
      "x = []\n",
      "\n",
      "for i in csvList:\n",
      "    i = i.replace(pj+'output/','').replace('.csv','') #remove _sm\n",
      "    #print i\n",
      "    x.append(i)\n",
      " \n",
      "print x\n",
      "\n",
      "df_list = []\n",
      "for i in x:\n",
      "    df = pd.read_csv(pj+'output/'+i+'.csv', dtype={'geoid':object})\n",
      "    df['geoid'] = df['geoid'].astype(str)\n",
      "    df_list.append(df)\n",
      "#print df_list    \n",
      "dfc = pd.concat(df_list)\n",
      "\n",
      "dfc.to_csv(pj+'output/all/taxi_2013.csv', index=False)\n",
      "\n",
      "print dfc.dtypes\n",
      "print len(dfc.index)"
     ],
     "language": "python",
     "metadata": {},
     "outputs": [
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "['d_2013', 'p_2013']\n",
        "346405433"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\n"
       ]
      }
     ],
     "prompt_number": 9
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "%reset"
     ],
     "language": "python",
     "metadata": {},
     "outputs": [
      {
       "name": "stdout",
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "Once deleted, variables cannot be recovered. Proceed (y/[n])? y\n"
       ]
      }
     ],
     "prompt_number": 10
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [],
     "language": "python",
     "metadata": {},
     "outputs": []
    }
   ],
   "metadata": {}
  }
 ]
}