{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 1a. A demo of using pandas to bulk download files...\n",
    "Building off the method we just looked at of using Pandas to grab a single file, we see here how Python can be quite effective at downloading industrial water data for **all** US states. \n",
    "\n",
    "Compare running this script with doing the downloads by hand..."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#import os & pandas\n",
    "import os\n",
    "import pandas as pd\n",
    "\n",
    "#import the us package (install if necessary)\n",
    "try:\n",
    "    import us\n",
    "except:\n",
    "    !pip install us\n",
    "    print('installing the US package')\n",
    "    import us"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#Create a folder to hold all the downloads\n",
    "outFolder = \"WaterData\"\n",
    "if not os.path.exists(outFolder): os.mkdir(outFolder)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#Loop through each state, download it's data, and save to a local file\n",
    "for state in us.STATES:\n",
    "    stateAbbr = state.abbr.lower()\n",
    "    print(\"{}\".format(stateAbbr),end='>')\n",
    "    \n",
    "    #Update the url with the state code\n",
    "    theURL = 'https://waterdata.usgs.gov/{}/nwis/water_use?format=rdb&rdb_compression=value&wu_area=County&wu_year=ALL&wu_county=ALL&wu_category=IN&wu_county_nms=--ALL%2BCounties--&wu_category_nms=Industrial'\n",
    "    theURL = theURL.format(stateAbbr)\n",
    "    \n",
    "    #Get the data as a dataframe\n",
    "    dfState = pd.read_csv(theURL,delimiter='\\t',skiprows=list(range(49))+[50])\n",
    "    \n",
    "    #write df to csv in the WaterData folder\n",
    "    dfState.to_csv(\"WaterData/{}.csv\".format(stateAbbr),index=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.12"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}