{
"cells": [
{
"cell_type": "code",
"execution_count": 52,
"metadata": {},
"outputs": [],
"source": [
"import multiprocessing as mp\n",
"# https://github.com/matplotlib/matplotlib/issues/15410#issuecomment-625027757\n",
"mp.set_start_method('forkserver')"
]
},
{
"cell_type": "code",
"execution_count": 81,
"metadata": {},
"outputs": [],
"source": [
"%matplotlib inline\n",
"import warnings\n",
"import pandas as pd\n",
"import matplotlib.pyplot as plt\n",
"from datetime import datetime\n",
"from functools import reduce\n",
"from vortexasdk import CargoTimeSeries, Products, Geographies\n",
"from vortexasdk.utils import convert_to_list\n",
"\n",
"warnings.filterwarnings(\"ignore\")\n",
"plt.rcParams['figure.figsize'] = (15, 10)\n",
"plt.rcParams.update({'font.size': 14})"
]
},
{
"cell_type": "code",
"execution_count": 82,
"metadata": {},
"outputs": [],
"source": [
"START_DATE = datetime(2019, 6, 10)\n",
"END_DATE = datetime(2020, 6, 10)\n",
"UNIT = 'b'"
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"# Define helper functions"
]
},
{
"cell_type": "code",
"execution_count": 83,
"metadata": {},
"outputs": [],
"source": [
"def get_product_id_exact(product_name):\n",
" if product_name is None:\n",
" return None\n",
" \n",
" products = [p.id for p in Products().search(product_name).to_list() if p.name==product_name] \n",
" assert len(products) == 1\n",
" return products[0]\n",
"\n",
"def get_geography_id_exact(geog_name):\n",
" if geog_name is None:\n",
" return None\n",
" \n",
" geogs = [g.id for g in Geographies().search(geog_name).to_list() if g.name==geog_name] \n",
" assert len(geogs) == 1\n",
" return geogs[0]"
]
},
{
"cell_type": "code",
"execution_count": 84,
"metadata": {},
"outputs": [],
"source": [
"def merge(data_frames):\n",
" return reduce(\n",
" lambda left, right: pd.merge(\n",
" left, right, left_index=True, right_index=True, how=\"outer\"\n",
" ),\n",
" data_frames,\n",
" )"
]
},
{
"cell_type": "code",
"execution_count": 85,
"metadata": {},
"outputs": [],
"source": [
"def plot_df(df, title=None, unit=UNIT):\n",
" df.plot(title=title, grid=True)\n",
" plt.xlabel('date')\n",
" plt.ylabel('k' + unit);"
]
},
{
"cell_type": "code",
"execution_count": 86,
"metadata": {},
"outputs": [],
"source": [
"def prepare_dataset(df_fs, product_names, destination_names, storage_names, filter_activity):\n",
"\n",
" # just keep key and value\n",
" df_fs = df_fs[['key', 'value']]\n",
"\n",
" # use kilo unit not unit\n",
" df_fs['value'] = df_fs['value'] / 1000\n",
"\n",
"\n",
" # rename columns\n",
" col_name = str((destination_names or \" \")) + \\\n",
" \" \" + str((storage_names) or \" \") + \\\n",
" \" \" + str((product_names) or \" \") + \\\n",
" \": \" + filter_activity\n",
"\n",
" df_fs = df_fs.rename(columns={'key': 'date', 'value': col_name})\n",
"\n",
" # remove time zone from timestamp\n",
" df_fs['date'] = pd.to_datetime(df_fs['date']).dt.tz_localize(None)\n",
"\n",
" return df_fs.set_index('date')"
]
},
{
"cell_type": "code",
"execution_count": 87,
"metadata": {},
"outputs": [],
"source": [
"def fetch_timeseries(filter_activity, product_names=None, destination_names=None, storage_names=None,\n",
" unit=UNIT, frequency='day', start_date=START_DATE, end_date=END_DATE):\n",
" \n",
" # Generate IDs\n",
" product_ids = [get_product_id_exact(name) for name in convert_to_list(product_names)]\n",
" destination_ids = [get_geography_id_exact(name) for name in convert_to_list(destination_names)]\n",
" storage_ids = [get_geography_id_exact(name) for name in convert_to_list(storage_names)]\n",
" \n",
" \n",
" # Load Data\n",
" df = CargoTimeSeries().search(timeseries_frequency=frequency,\n",
" timeseries_unit=unit,\n",
" disable_geographic_exclusion_rules=True,\n",
" filter_products=product_ids,\n",
" filter_destinations=destination_ids,\n",
" filter_storage_locations=storage_ids,\n",
" filter_activity=filter_activity,\n",
" filter_time_min=start_date,\n",
" filter_time_max=end_date).to_df()\n",
"\n",
" # Rename columns, set index etc\n",
" return prepare_dataset(df, product_names, destination_names, storage_names, filter_activity)\n",
" \n"
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"# Define our commonly used constants"
]
},
{
"cell_type": "code",
"execution_count": 88,
"metadata": {
"scrolled": false
},
"outputs": [],
"source": [
"clean = \"Clean Petroleum Products\"\n",
"naphtha = \"Naphtha\"\n",
"diesel_gasoil = \"Diesel/Gasoil\"\n",
"gasoline_blending_components = \"Gasoline/Blending Components\"\n",
"jet_kero = \"Jet/Kero\""
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"# Analysis Start"
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"### Load all global clean floating storage cargos"
]
},
{
"cell_type": "code",
"execution_count": 89,
"metadata": {},
"outputs": [
{
"data": {
"text/html": [
"\n",
"\n",
"
\n",
" \n",
" \n",
" | \n",
" Clean Petroleum Products: storing_state | \n",
"
\n",
" \n",
" | date | \n",
" | \n",
"
\n",
" \n",
" \n",
" \n",
" | 2019-06-10 | \n",
" 28275.824 | \n",
"
\n",
" \n",
" | 2019-06-11 | \n",
" 28572.234 | \n",
"
\n",
" \n",
" | 2019-06-12 | \n",
" 27440.642 | \n",
"
\n",
" \n",
" | 2019-06-13 | \n",
" 27306.334 | \n",
"
\n",
" \n",
" | 2019-06-14 | \n",
" 28177.139 | \n",
"
\n",
" \n",
"
\n",
"
"
]
},
"metadata": {
"needs_background": "light"
},
"output_type": "display_data"
}
],
"source": [
"plot_df(clean_fs, \"Global Clean Floating Storage\")"
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"### Let's look at the product split of these global floating storage cargos"
]
},
{
"cell_type": "code",
"execution_count": 91,
"metadata": {},
"outputs": [
{
"data": {
"text/html": [
"\n",
"\n",
"
\n",
" \n",
" \n",
" | \n",
" Gasoline/Blending Components: storing_state | \n",
" Diesel/Gasoil: storing_state | \n",
" Naphtha: storing_state | \n",
" Jet/Kero: storing_state | \n",
"
\n",
" \n",
" | date | \n",
" | \n",
" | \n",
" | \n",
" | \n",
"
\n",
" \n",
" \n",
" \n",
" | 2019-06-10 | \n",
" 7184.033 | \n",
" 11248.500 | \n",
" 1519.893 | \n",
" 896.613 | \n",
"
\n",
" \n",
" | 2019-06-11 | \n",
" 7056.326 | \n",
" 10627.893 | \n",
" 1442.768 | \n",
" 673.131 | \n",
"
\n",
" \n",
" | 2019-06-12 | \n",
" 7061.802 | \n",
" 10525.544 | \n",
" 766.018 | \n",
" 673.131 | \n",
"
\n",
" \n",
" | 2019-06-13 | \n",
" 6708.143 | \n",
" 9936.323 | \n",
" 766.018 | \n",
" 767.094 | \n",
"
\n",
" \n",
" | 2019-06-14 | \n",
" 6812.918 | \n",
" 10646.097 | \n",
" 766.018 | \n",
" 854.251 | \n",
"
\n",
" \n",
"
\n",
"
"
]
},
"metadata": {
"needs_background": "light"
},
"output_type": "display_data"
}
],
"source": [
"plot_df(df_merged)"
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"### Asia-only floating storage"
]
},
{
"cell_type": "code",
"execution_count": 93,
"metadata": {},
"outputs": [
{
"data": {
"text/html": [
"\n",
"\n",
"
\n",
" \n",
" \n",
" | \n",
" Asia Gasoline/Blending Components: storing_state | \n",
" Asia Diesel/Gasoil: storing_state | \n",
" Asia Naphtha: storing_state | \n",
" Asia Jet/Kero: storing_state | \n",
"
\n",
" \n",
" | date | \n",
" | \n",
" | \n",
" | \n",
" | \n",
"
\n",
" \n",
" \n",
" \n",
" | 2019-06-10 | \n",
" 393.260 | \n",
" 4422.283 | \n",
" 555.15 | \n",
" 462.041 | \n",
"
\n",
" \n",
" | 2019-06-11 | \n",
" 429.093 | \n",
" 4090.497 | \n",
" 555.15 | \n",
" 311.980 | \n",
"
\n",
" \n",
" | 2019-06-12 | \n",
" 330.501 | \n",
" 3648.004 | \n",
" 0.00 | \n",
" 311.980 | \n",
"
\n",
" \n",
" | 2019-06-13 | \n",
" 545.668 | \n",
" 3629.035 | \n",
" 0.00 | \n",
" 0.000 | \n",
"
\n",
" \n",
" | 2019-06-14 | \n",
" 545.668 | \n",
" 3503.981 | \n",
" 0.00 | \n",
" 0.000 | \n",
"
\n",
" \n",
"
\n",
"
"
]
},
"metadata": {
"needs_background": "light"
},
"output_type": "display_data"
}
],
"source": [
"plot_df(df_asia)"
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"### See how Diesel/Gasoil storage levels are split across Asian geographies"
]
},
{
"cell_type": "code",
"execution_count": 95,
"metadata": {},
"outputs": [
{
"data": {
"text/html": [
"\n",
"\n",
"
\n",
" \n",
" \n",
" | \n",
" South Korea Diesel/Gasoil: storing_state | \n",
" India Diesel/Gasoil: storing_state | \n",
" China Diesel/Gasoil: storing_state | \n",
" ['Singapore', 'Malaysia', 'Indonesia'] Diesel/Gasoil: storing_state | \n",
"
\n",
" \n",
" | date | \n",
" | \n",
" | \n",
" | \n",
" | \n",
"
\n",
" \n",
" \n",
" \n",
" | 2019-06-10 | \n",
" 56.23 | \n",
" 262.668 | \n",
" 2092.033 | \n",
" 1300.501 | \n",
"
\n",
" \n",
" | 2019-06-11 | \n",
" 0.00 | \n",
" 10.988 | \n",
" 2044.634 | \n",
" 1300.501 | \n",
"
\n",
" \n",
" | 2019-06-12 | \n",
" 0.00 | \n",
" 10.988 | \n",
" 2044.634 | \n",
" 1155.295 | \n",
"
\n",
" \n",
" | 2019-06-13 | \n",
" 0.00 | \n",
" 10.988 | \n",
" 2016.500 | \n",
" 941.291 | \n",
"
\n",
" \n",
" | 2019-06-14 | \n",
" 0.00 | \n",
" 10.988 | \n",
" 1843.022 | \n",
" 952.342 | \n",
"
\n",
" \n",
"
\n",
"