{
"cells": [
{
"cell_type": "markdown",
"metadata": {},
"source": [
"# Multi step model (simple encoder-decoder)\n",
"\n",
"In this notebook, we demonstrate how to:\n",
"- prepare time series data for training a RNN forecasting model\n",
"- get data in the required shape for the keras API\n",
"- implement a RNN model in keras to predict the next 3 steps ahead (time *t+1* to *t+3*) in the time series. This model uses a simple encoder decoder approach in which the final hidden state of the encoder is replicated across each time step of the decoder. \n",
"- enable early stopping to reduce the likelihood of model overfitting\n",
"- evaluate the model on a test dataset\n",
"\n",
"The data in this example is taken from the GEFCom2014 forecasting competition1 . It consists of 3 years of hourly electricity load and temperature values between 2012 and 2014. The task is to forecast future values of electricity load.\n",
"\n",
"1 Tao Hong, Pierre Pinson, Shu Fan, Hamidreza Zareipour, Alberto Troccoli and Rob J. Hyndman, \"Probabilistic energy forecasting: Global Energy Forecasting Competition 2014 and beyond\", International Journal of Forecasting, vol.32, no.3, pp 896-913, July-September, 2016."
]
},
{
"cell_type": "code",
"execution_count": 1,
"metadata": {
"collapsed": true
},
"outputs": [],
"source": [
"import os\n",
"import warnings\n",
"import matplotlib.pyplot as plt\n",
"import numpy as np\n",
"import pandas as pd\n",
"import datetime as dt\n",
"from collections import UserDict\n",
"from IPython.display import Image\n",
"%matplotlib inline\n",
"\n",
"from common.utils import load_data, mape, TimeSeriesTensor, create_evaluation_df\n",
"\n",
"pd.options.display.float_format = '{:,.2f}'.format\n",
"np.set_printoptions(precision=2)\n",
"warnings.filterwarnings(\"ignore\")"
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"Load data into Pandas dataframe"
]
},
{
"cell_type": "code",
"execution_count": 2,
"metadata": {},
"outputs": [
{
"data": {
"text/html": [
"\n",
"\n",
"
\n",
" \n",
" \n",
" load \n",
" temp \n",
" \n",
" \n",
" \n",
" \n",
" 2012-01-01 00:00:00 \n",
" 2,698.00 \n",
" 32.00 \n",
" \n",
" \n",
" 2012-01-01 01:00:00 \n",
" 2,558.00 \n",
" 32.67 \n",
" \n",
" \n",
" 2012-01-01 02:00:00 \n",
" 2,444.00 \n",
" 30.00 \n",
" \n",
" \n",
" 2012-01-01 03:00:00 \n",
" 2,402.00 \n",
" 31.00 \n",
" \n",
" \n",
" 2012-01-01 04:00:00 \n",
" 2,403.00 \n",
" 32.00 \n",
" \n",
" \n",
"
\n",
"
\n",
"\n",
"
\n",
" \n",
" \n",
" tensor \n",
" target \n",
" X \n",
" \n",
" \n",
" feature \n",
" y \n",
" load \n",
" temp \n",
" \n",
" \n",
" time step \n",
" t+1 \n",
" t+2 \n",
" t+3 \n",
" t-5 \n",
" t-4 \n",
" t-3 \n",
" t-2 \n",
" t-1 \n",
" t \n",
" t-5 \n",
" t-4 \n",
" t-3 \n",
" t-2 \n",
" t-1 \n",
" t \n",
" \n",
" \n",
" \n",
" \n",
" 2012-01-01 05:00:00 \n",
" 0.18 \n",
" 0.23 \n",
" 0.29 \n",
" 0.22 \n",
" 0.18 \n",
" 0.14 \n",
" 0.13 \n",
" 0.13 \n",
" 0.15 \n",
" 0.42 \n",
" 0.43 \n",
" 0.40 \n",
" 0.41 \n",
" 0.42 \n",
" 0.41 \n",
" \n",
" \n",
" 2012-01-01 06:00:00 \n",
" 0.23 \n",
" 0.29 \n",
" 0.35 \n",
" 0.18 \n",
" 0.14 \n",
" 0.13 \n",
" 0.13 \n",
" 0.15 \n",
" 0.18 \n",
" 0.43 \n",
" 0.40 \n",
" 0.41 \n",
" 0.42 \n",
" 0.41 \n",
" 0.40 \n",
" \n",
" \n",
" 2012-01-01 07:00:00 \n",
" 0.29 \n",
" 0.35 \n",
" 0.37 \n",
" 0.14 \n",
" 0.13 \n",
" 0.13 \n",
" 0.15 \n",
" 0.18 \n",
" 0.23 \n",
" 0.40 \n",
" 0.41 \n",
" 0.42 \n",
" 0.41 \n",
" 0.40 \n",
" 0.39 \n",
" \n",
" \n",
" 2012-01-01 08:00:00 \n",
" 0.35 \n",
" 0.37 \n",
" 0.37 \n",
" 0.13 \n",
" 0.13 \n",
" 0.15 \n",
" 0.18 \n",
" 0.23 \n",
" 0.29 \n",
" 0.41 \n",
" 0.42 \n",
" 0.41 \n",
" 0.40 \n",
" 0.39 \n",
" 0.39 \n",
" \n",
" \n",
" 2012-01-01 09:00:00 \n",
" 0.37 \n",
" 0.37 \n",
" 0.37 \n",
" 0.13 \n",
" 0.15 \n",
" 0.18 \n",
" 0.23 \n",
" 0.29 \n",
" 0.35 \n",
" 0.42 \n",
" 0.41 \n",
" 0.40 \n",
" 0.39 \n",
" 0.39 \n",
" 0.43 \n",
" \n",
" \n",
"
\n",
"
"
]
},
"execution_count": 9,
"metadata": {},
"output_type": "execute_result"
}
],
"source": [
"Image('./images/simple_encoder_decoder.png')"
]
},
{
"cell_type": "code",
"execution_count": 10,
"metadata": {},
"outputs": [
{
"name": "stderr",
"output_type": "stream",
"text": [
"Using TensorFlow backend.\n"
]
}
],
"source": [
"from keras.models import Model, Sequential\n",
"from keras.layers import GRU, Dense, RepeatVector, TimeDistributed, Flatten\n",
"from keras.callbacks import EarlyStopping"
]
},
{
"cell_type": "code",
"execution_count": 11,
"metadata": {
"collapsed": true
},
"outputs": [],
"source": [
"LATENT_DIM = 5\n",
"BATCH_SIZE = 32\n",
"EPOCHS = 10"
]
},
{
"cell_type": "code",
"execution_count": 12,
"metadata": {
"collapsed": true
},
"outputs": [],
"source": [
"model = Sequential()\n",
"model.add(GRU(LATENT_DIM, input_shape=(T, 2)))\n",
"model.add(RepeatVector(HORIZON))\n",
"model.add(GRU(LATENT_DIM, return_sequences=True))\n",
"model.add(TimeDistributed(Dense(1)))\n",
"model.add(Flatten())"
]
},
{
"cell_type": "code",
"execution_count": 13,
"metadata": {
"collapsed": true
},
"outputs": [],
"source": [
"model.compile(optimizer='RMSprop', loss='mse')"
]
},
{
"cell_type": "code",
"execution_count": 14,
"metadata": {},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"_________________________________________________________________\n",
"Layer (type) Output Shape Param # \n",
"=================================================================\n",
"gru_1 (GRU) (None, 5) 120 \n",
"_________________________________________________________________\n",
"repeat_vector_1 (RepeatVecto (None, 3, 5) 0 \n",
"_________________________________________________________________\n",
"gru_2 (GRU) (None, 3, 5) 165 \n",
"_________________________________________________________________\n",
"time_distributed_1 (TimeDist (None, 3, 1) 6 \n",
"_________________________________________________________________\n",
"flatten_1 (Flatten) (None, 3) 0 \n",
"=================================================================\n",
"Total params: 291\n",
"Trainable params: 291\n",
"Non-trainable params: 0\n",
"_________________________________________________________________\n"
]
}
],
"source": [
"model.summary()"
]
},
{
"cell_type": "code",
"execution_count": 15,
"metadata": {
"collapsed": true
},
"outputs": [],
"source": [
"earlystop = EarlyStopping(monitor='val_loss', min_delta=0, patience=5)"
]
},
{
"cell_type": "code",
"execution_count": 16,
"metadata": {},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"Train on 23368 samples, validate on 1461 samples\n",
"Epoch 1/50\n",
"23368/23368 [==============================] - 4s 185us/step - loss: 0.0217 - val_loss: 0.0053\n",
"Epoch 2/50\n",
"23368/23368 [==============================] - 3s 129us/step - loss: 0.0050 - val_loss: 0.0042\n",
"Epoch 3/50\n",
"23368/23368 [==============================] - 3s 147us/step - loss: 0.0044 - val_loss: 0.0039\n",
"Epoch 4/50\n",
"23368/23368 [==============================] - 3s 129us/step - loss: 0.0041 - val_loss: 0.0035\n",
"Epoch 5/50\n",
"23368/23368 [==============================] - 3s 134us/step - loss: 0.0039 - val_loss: 0.0033\n",
"Epoch 6/50\n",
"23368/23368 [==============================] - 3s 138us/step - loss: 0.0037 - val_loss: 0.0032\n",
"Epoch 7/50\n",
"23368/23368 [==============================] - 3s 127us/step - loss: 0.0035 - val_loss: 0.0029\n",
"Epoch 8/50\n",
"23368/23368 [==============================] - 3s 130us/step - loss: 0.0034 - val_loss: 0.0029\n",
"Epoch 9/50\n",
"23368/23368 [==============================] - 4s 156us/step - loss: 0.0033 - val_loss: 0.0037\n",
"Epoch 10/50\n",
"23368/23368 [==============================] - 3s 123us/step - loss: 0.0033 - val_loss: 0.0029\n",
"Epoch 11/50\n",
"23368/23368 [==============================] - 4s 177us/step - loss: 0.0032 - val_loss: 0.0026\n",
"Epoch 12/50\n",
"23368/23368 [==============================] - 5s 195us/step - loss: 0.0032 - val_loss: 0.0032\n",
"Epoch 13/50\n",
"23368/23368 [==============================] - 4s 167us/step - loss: 0.0032 - val_loss: 0.0027\n",
"Epoch 14/50\n",
"23368/23368 [==============================] - 3s 135us/step - loss: 0.0032 - val_loss: 0.0028\n",
"Epoch 15/50\n",
"23368/23368 [==============================] - 4s 174us/step - loss: 0.0031 - val_loss: 0.0041\n",
"Epoch 16/50\n",
"23368/23368 [==============================] - 4s 172us/step - loss: 0.0031 - val_loss: 0.0034\n"
]
},
{
"data": {
"text/plain": [
""
]
},
"execution_count": 16,
"metadata": {},
"output_type": "execute_result"
}
],
"source": [
"model.fit(train_inputs['X'],\n",
" train_inputs['target'],\n",
" batch_size=BATCH_SIZE,\n",
" epochs=EPOCHS,\n",
" validation_data=(valid_inputs['X'], valid_inputs['target']),\n",
" callbacks=[earlystop],\n",
" verbose=1)"
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"## Evaluate the model"
]
},
{
"cell_type": "code",
"execution_count": 17,
"metadata": {
"collapsed": true
},
"outputs": [],
"source": [
"look_back_dt = dt.datetime.strptime(test_start_dt, '%Y-%m-%d %H:%M:%S') - dt.timedelta(hours=T-1)\n",
"test = energy.copy()[test_start_dt:][['load', 'temp']]\n",
"test[['load', 'temp']] = X_scaler.transform(test)\n",
"test_inputs = TimeSeriesTensor(test, 'load', HORIZON, tensor_structure)"
]
},
{
"cell_type": "code",
"execution_count": 18,
"metadata": {
"collapsed": true
},
"outputs": [],
"source": [
"predictions = model.predict(test_inputs['X'])"
]
},
{
"cell_type": "code",
"execution_count": 19,
"metadata": {},
"outputs": [
{
"data": {
"text/plain": [
"array([[0.24, 0.31, 0.39],\n",
" [0.31, 0.39, 0.46],\n",
" [0.4 , 0.46, 0.51],\n",
" ...,\n",
" [0.62, 0.57, 0.51],\n",
" [0.58, 0.54, 0.48],\n",
" [0.54, 0.5 , 0.46]], dtype=float32)"
]
},
"execution_count": 19,
"metadata": {},
"output_type": "execute_result"
}
],
"source": [
"predictions"
]
},
{
"cell_type": "code",
"execution_count": 20,
"metadata": {},
"outputs": [
{
"data": {
"text/html": [
"\n",
"\n",
"
\n",
" \n",
" \n",
" timestamp \n",
" h \n",
" prediction \n",
" actual \n",
" \n",
" \n",
" \n",
" \n",
" 0 \n",
" 2014-11-01 05:00:00 \n",
" t+1 \n",
" 2,748.30 \n",
" 2,714.00 \n",
" \n",
" \n",
" 1 \n",
" 2014-11-01 06:00:00 \n",
" t+1 \n",
" 2,994.21 \n",
" 2,970.00 \n",
" \n",
" \n",
" 2 \n",
" 2014-11-01 07:00:00 \n",
" t+1 \n",
" 3,267.59 \n",
" 3,189.00 \n",
" \n",
" \n",
" 3 \n",
" 2014-11-01 08:00:00 \n",
" t+1 \n",
" 3,405.73 \n",
" 3,356.00 \n",
" \n",
" \n",
" 4 \n",
" 2014-11-01 09:00:00 \n",
" t+1 \n",
" 3,537.20 \n",
" 3,436.00 \n",
" \n",
" \n",
"
\n",
"
"
]
},
"metadata": {},
"output_type": "display_data"
}
],
"source": [
"plot_df = eval_df[(eval_df.timestamp<'2014-11-08') & (eval_df.h=='t+1')][['timestamp', 'actual']]\n",
"for t in range(1, HORIZON+1):\n",
" plot_df['t+'+str(t)] = eval_df[(eval_df.timestamp<'2014-11-08') & (eval_df.h=='t+'+str(t))]['prediction'].values\n",
"\n",
"fig = plt.figure(figsize=(15, 8))\n",
"ax = plt.plot(plot_df['timestamp'], plot_df['actual'], color='red', linewidth=4.0)\n",
"ax = fig.add_subplot(111)\n",
"ax.plot(plot_df['timestamp'], plot_df['t+1'], color='blue', linewidth=4.0, alpha=0.75)\n",
"ax.plot(plot_df['timestamp'], plot_df['t+2'], color='blue', linewidth=3.0, alpha=0.5)\n",
"ax.plot(plot_df['timestamp'], plot_df['t+3'], color='blue', linewidth=2.0, alpha=0.25)\n",
"plt.xlabel('timestamp', fontsize=12)\n",
"plt.ylabel('load', fontsize=12)\n",
"ax.legend(loc='best')\n",
"plt.show()"
]
}
],
"metadata": {
"kernelspec": {
"display_name": "Python 3.5",
"language": "python",
"name": "python3"
},
"language_info": {
"codemirror_mode": {
"name": "ipython",
"version": 3
},
"file_extension": ".py",
"mimetype": "text/x-python",
"name": "python",
"nbconvert_exporter": "python",
"pygments_lexer": "ipython3",
"version": "3.5.5"
}
},
"nbformat": 4,
"nbformat_minor": 2
}