{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Get Data\n",
    "This notebook includes the procedures for fetching:\n",
    "* The DC Fast Charging (\"DCFC\") locations from NREL servers via NREL's APIs, and\n",
    "* The North Carolina major road network, from OpenStreetMap via the osmnx package."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "#Import packages\n",
    "import os, shutil, requests\n",
    "import osmnx as ox\n",
    "import pandas as pd\n",
    "import geopandas as gpd"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 1. Fetch the DCFC data\n",
    "* Fetch the data, which includes latitude and longitude fields, filtered for NC into a Pandas dataframe.\n",
    "* Clean the data: remove unneeded columns and fiter for non-Tesla public charging records.\n",
    "* Convert to a geodataframe, save as a shapefile."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "#Get the NREL API key (stored as local file)\n",
    "key = open('Data/NRELkey.txt','r').readline()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "#Construct the service request\n",
    "baseURL = 'https://developer.nrel.gov/api/alt-fuel-stations/v1.json?'\n",
    "params = {\n",
    "    'api_key':key,                #Our key, to authenticate the service\n",
    "    'status':'E',                 #Request only currently open sites \n",
    "    'fuel_type':'ELEC',           #Request only Electric sites\n",
    "    'ev_charging_level':'dc_fast',#Request only DCFC sites\n",
    "    'state':'NC' ,                #Request only NC sites\n",
    "    'access':'public',            #Request only Public sites\n",
    "    'groups_with_access_code':'Public', #Request only sites open to Public\n",
    "    'limit':'all'                 #Don't cap results to first 200 (default)\n",
    "}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "86 records retrieved\n"
     ]
    }
   ],
   "source": [
    "#Execute the request, converting the response to a dataframe\n",
    "df_ALL = pd.DataFrame.from_dict(requests.get(baseURL,params).json()['fuel_stations'])\n",
    "print(f'{len(df_ALL)} records retrieved')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "#Drop excess columns\n",
    "dropCols = [col for col in df_ALL.columns if col[:3] in ('bd_','cng','hy_','ng_','e85','lng','lpg')]\n",
    "dropCols += [col for col in df_ALL.columns if col[-2:] == 'fr']\n",
    "df_ALL.drop(columns = dropCols,inplace=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "#Remove sites in the Tesla network\n",
    "df_DCFC = df_ALL.loc[df_ALL['ev_network'] != 'Tesla'].reset_index()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Creating NREL folder in Data directory\n"
     ]
    }
   ],
   "source": [
    "#Create a folder to hold NREL data\n",
    "if not os.path.exists('Data/NREL'):\n",
    "    print(\"Creating NREL folder in Data directory\")\n",
    "    os.mkdir('Data/NREL')\n",
    "else:\n",
    "    print(\"Using existing NREL folder in Data directory\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "68 records save to Data/NREL/DCFCsites.csv\n"
     ]
    }
   ],
   "source": [
    "#Save to a csv\n",
    "df_DCFC.to_csv('Data/NREL/DCFCsites.csv',index=False)\n",
    "print(f\"{len(df_DCFC)} records save to Data/NREL/DCFCsites.csv\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "#Convert to a geodataframe\n",
    "from shapely.geometry import Point\n",
    "geometry = [Point(xy) for xy in zip(df_DCFC.longitude, df_DCFC.latitude)]\n",
    "gdf_DCFC = gpd.GeoDataFrame(df_DCFC,geometry=geometry,crs='epsg:4326')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "68 features stored as Data/NREL/NC_DCFC.shp\n"
     ]
    }
   ],
   "source": [
    "gdf_DCFC[['id','geometry']].to_file('Data/NREL/NC_DCFC.shp')\n",
    "print(f\"{len(df_DCFC)} features stored as Data/NREL/NC_DCFC.shp\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 2. Fetch NC major roads as a graph and shapefile\n",
    "* Read NC major roads (motorway, primary) graphml into a graph object; fetch if not present"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Creating OSM folder in Data directory\n"
     ]
    }
   ],
   "source": [
    "#Create a folder to hold OSM data, if not there already\n",
    "if not os.path.exists('Data/OSM'):\n",
    "    print(\"Creating OSM folder in Data directory\")\n",
    "    os.mkdir('Data/OSM')\n",
    "else:\n",
    "    print(\"Using existing OSM folder in Data directory\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Fetching data. This may take a while...\n",
      "Saving file to Data/OSM/NC_highways_all.graphml\n"
     ]
    }
   ],
   "source": [
    "#Fetch the NC highway graph\n",
    "graph_filename = 'Data/OSM/NC_highways_all.graphml'\n",
    "if os.path.exists(graph_filename):\n",
    "    print(f\"Loading graph from {graph_filename}\")\n",
    "    nc_graph = ox.load_graphml(filename=os.path.basename(graph_filename),\n",
    "                               folder=os.path.dirname(graph_filename))\n",
    "else:\n",
    "    print(\"Fetching data. This may take a while...\")\n",
    "    nc_graph = ox.graph_from_place(\"North Carolina\",\n",
    "                                    network_type='drive',\n",
    "                                    simplify=True,\n",
    "                                    retain_all=False,\n",
    "                                    truncate_by_edge=True,\n",
    "                                    infrastructure='way[\"highway\"~\"motorway|trunk|primary\"]')\n",
    "    print(f\"Saving file to {graph_filename}\")\n",
    "    ox.save_graphml(nc_graph,\n",
    "                    filename=os.path.basename(graph_filename),\n",
    "                    folder=os.path.dirname(graph_filename))\n",
    "                    "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [],
   "source": [
    "#Save as a shapefile\n",
    "ox.save_graph_shapefile(G=nc_graph,filename=\"NC_highways\",folder=\"Data/OSM\")"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.6"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}