{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "kernelspec": {
      "display_name": "Python 3",
      "language": "python",
      "name": "python3"
    },
    "language_info": {
      "codemirror_mode": {
        "name": "ipython",
        "version": 3
      },
      "file_extension": ".py",
      "mimetype": "text/x-python",
      "name": "python",
      "nbconvert_exporter": "python",
      "pygments_lexer": "ipython3",
      "version": "3.6.5"
    },
    "colab": {
      "name": "01-explore-data.ipynb",
      "provenance": []
    }
  },
  "cells": [
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "OsjvyRyVU3_G",
        "colab_type": "text"
      },
      "source": [
        "# Understanding the data\n",
        "\n",
        "In this first part, we load the data and perform some initial exploration on it. The main goal of this step is to acquire some basic knowledge about the data, how the various features are distributed, if there are missing values in it and so on."
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "_wmNcEoRU3_K",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "### imports\n",
        "import pandas as pd\n",
        "import seaborn as sns\n",
        "import matplotlib.pyplot as plt\n",
        "import numpy as np\n",
        "%matplotlib inline\n",
        "\n",
        "# load hourly data\n",
        "hourly_data = pd.read_csv('https://raw.githubusercontent.com/PacktWorkshops/The-Data-Analysis-Workshop/master/Chapter01/data/hour.csv')"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "qDc53XA2U3_P",
        "colab_type": "text"
      },
      "source": [
        "Check data format, number of missing values in the data and general statistics:"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "9QFrElsEU3_Q",
        "colab_type": "code",
        "colab": {},
        "outputId": "6fda5579-55de-478b-a00d-ede2cfecd346"
      },
      "source": [
        "# print some generic statistics about the data\n",
        "print(f\"Shape of data: {hourly_data.shape}\")\n",
        "print(f\"Number of missing values in the data: {hourly_data.isnull().sum().sum()}\")\n",
        "\n",
        "# get statistics on the numerical columns\n",
        "hourly_data.describe().T"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Shape of data: (17379, 17)\n",
            "Number of missing values in the data: 0\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "execute_result",
          "data": {
            "text/html": [
              "<div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>count</th>\n",
              "      <th>mean</th>\n",
              "      <th>std</th>\n",
              "      <th>min</th>\n",
              "      <th>25%</th>\n",
              "      <th>50%</th>\n",
              "      <th>75%</th>\n",
              "      <th>max</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>instant</th>\n",
              "      <td>17379.0</td>\n",
              "      <td>8690.000000</td>\n",
              "      <td>5017.029500</td>\n",
              "      <td>1.00</td>\n",
              "      <td>4345.5000</td>\n",
              "      <td>8690.0000</td>\n",
              "      <td>13034.5000</td>\n",
              "      <td>17379.0000</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>season</th>\n",
              "      <td>17379.0</td>\n",
              "      <td>2.501640</td>\n",
              "      <td>1.106918</td>\n",
              "      <td>1.00</td>\n",
              "      <td>2.0000</td>\n",
              "      <td>3.0000</td>\n",
              "      <td>3.0000</td>\n",
              "      <td>4.0000</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>yr</th>\n",
              "      <td>17379.0</td>\n",
              "      <td>0.502561</td>\n",
              "      <td>0.500008</td>\n",
              "      <td>0.00</td>\n",
              "      <td>0.0000</td>\n",
              "      <td>1.0000</td>\n",
              "      <td>1.0000</td>\n",
              "      <td>1.0000</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>mnth</th>\n",
              "      <td>17379.0</td>\n",
              "      <td>6.537775</td>\n",
              "      <td>3.438776</td>\n",
              "      <td>1.00</td>\n",
              "      <td>4.0000</td>\n",
              "      <td>7.0000</td>\n",
              "      <td>10.0000</td>\n",
              "      <td>12.0000</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>hr</th>\n",
              "      <td>17379.0</td>\n",
              "      <td>11.546752</td>\n",
              "      <td>6.914405</td>\n",
              "      <td>0.00</td>\n",
              "      <td>6.0000</td>\n",
              "      <td>12.0000</td>\n",
              "      <td>18.0000</td>\n",
              "      <td>23.0000</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>holiday</th>\n",
              "      <td>17379.0</td>\n",
              "      <td>0.028770</td>\n",
              "      <td>0.167165</td>\n",
              "      <td>0.00</td>\n",
              "      <td>0.0000</td>\n",
              "      <td>0.0000</td>\n",
              "      <td>0.0000</td>\n",
              "      <td>1.0000</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>weekday</th>\n",
              "      <td>17379.0</td>\n",
              "      <td>3.003683</td>\n",
              "      <td>2.005771</td>\n",
              "      <td>0.00</td>\n",
              "      <td>1.0000</td>\n",
              "      <td>3.0000</td>\n",
              "      <td>5.0000</td>\n",
              "      <td>6.0000</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>workingday</th>\n",
              "      <td>17379.0</td>\n",
              "      <td>0.682721</td>\n",
              "      <td>0.465431</td>\n",
              "      <td>0.00</td>\n",
              "      <td>0.0000</td>\n",
              "      <td>1.0000</td>\n",
              "      <td>1.0000</td>\n",
              "      <td>1.0000</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>weathersit</th>\n",
              "      <td>17379.0</td>\n",
              "      <td>1.425283</td>\n",
              "      <td>0.639357</td>\n",
              "      <td>1.00</td>\n",
              "      <td>1.0000</td>\n",
              "      <td>1.0000</td>\n",
              "      <td>2.0000</td>\n",
              "      <td>4.0000</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>temp</th>\n",
              "      <td>17379.0</td>\n",
              "      <td>0.496987</td>\n",
              "      <td>0.192556</td>\n",
              "      <td>0.02</td>\n",
              "      <td>0.3400</td>\n",
              "      <td>0.5000</td>\n",
              "      <td>0.6600</td>\n",
              "      <td>1.0000</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>atemp</th>\n",
              "      <td>17379.0</td>\n",
              "      <td>0.475775</td>\n",
              "      <td>0.171850</td>\n",
              "      <td>0.00</td>\n",
              "      <td>0.3333</td>\n",
              "      <td>0.4848</td>\n",
              "      <td>0.6212</td>\n",
              "      <td>1.0000</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>hum</th>\n",
              "      <td>17379.0</td>\n",
              "      <td>0.627229</td>\n",
              "      <td>0.192930</td>\n",
              "      <td>0.00</td>\n",
              "      <td>0.4800</td>\n",
              "      <td>0.6300</td>\n",
              "      <td>0.7800</td>\n",
              "      <td>1.0000</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>windspeed</th>\n",
              "      <td>17379.0</td>\n",
              "      <td>0.190098</td>\n",
              "      <td>0.122340</td>\n",
              "      <td>0.00</td>\n",
              "      <td>0.1045</td>\n",
              "      <td>0.1940</td>\n",
              "      <td>0.2537</td>\n",
              "      <td>0.8507</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>casual</th>\n",
              "      <td>17379.0</td>\n",
              "      <td>35.676218</td>\n",
              "      <td>49.305030</td>\n",
              "      <td>0.00</td>\n",
              "      <td>4.0000</td>\n",
              "      <td>17.0000</td>\n",
              "      <td>48.0000</td>\n",
              "      <td>367.0000</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>registered</th>\n",
              "      <td>17379.0</td>\n",
              "      <td>153.786869</td>\n",
              "      <td>151.357286</td>\n",
              "      <td>0.00</td>\n",
              "      <td>34.0000</td>\n",
              "      <td>115.0000</td>\n",
              "      <td>220.0000</td>\n",
              "      <td>886.0000</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>cnt</th>\n",
              "      <td>17379.0</td>\n",
              "      <td>189.463088</td>\n",
              "      <td>181.387599</td>\n",
              "      <td>1.00</td>\n",
              "      <td>40.0000</td>\n",
              "      <td>142.0000</td>\n",
              "      <td>281.0000</td>\n",
              "      <td>977.0000</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "</div>"
            ],
            "text/plain": [
              "              count         mean          std   min        25%        50%  \\\n",
              "instant     17379.0  8690.000000  5017.029500  1.00  4345.5000  8690.0000   \n",
              "season      17379.0     2.501640     1.106918  1.00     2.0000     3.0000   \n",
              "yr          17379.0     0.502561     0.500008  0.00     0.0000     1.0000   \n",
              "mnth        17379.0     6.537775     3.438776  1.00     4.0000     7.0000   \n",
              "hr          17379.0    11.546752     6.914405  0.00     6.0000    12.0000   \n",
              "holiday     17379.0     0.028770     0.167165  0.00     0.0000     0.0000   \n",
              "weekday     17379.0     3.003683     2.005771  0.00     1.0000     3.0000   \n",
              "workingday  17379.0     0.682721     0.465431  0.00     0.0000     1.0000   \n",
              "weathersit  17379.0     1.425283     0.639357  1.00     1.0000     1.0000   \n",
              "temp        17379.0     0.496987     0.192556  0.02     0.3400     0.5000   \n",
              "atemp       17379.0     0.475775     0.171850  0.00     0.3333     0.4848   \n",
              "hum         17379.0     0.627229     0.192930  0.00     0.4800     0.6300   \n",
              "windspeed   17379.0     0.190098     0.122340  0.00     0.1045     0.1940   \n",
              "casual      17379.0    35.676218    49.305030  0.00     4.0000    17.0000   \n",
              "registered  17379.0   153.786869   151.357286  0.00    34.0000   115.0000   \n",
              "cnt         17379.0   189.463088   181.387599  1.00    40.0000   142.0000   \n",
              "\n",
              "                   75%         max  \n",
              "instant     13034.5000  17379.0000  \n",
              "season          3.0000      4.0000  \n",
              "yr              1.0000      1.0000  \n",
              "mnth           10.0000     12.0000  \n",
              "hr             18.0000     23.0000  \n",
              "holiday         0.0000      1.0000  \n",
              "weekday         5.0000      6.0000  \n",
              "workingday      1.0000      1.0000  \n",
              "weathersit      2.0000      4.0000  \n",
              "temp            0.6600      1.0000  \n",
              "atemp           0.6212      1.0000  \n",
              "hum             0.7800      1.0000  \n",
              "windspeed       0.2537      0.8507  \n",
              "casual         48.0000    367.0000  \n",
              "registered    220.0000    886.0000  \n",
              "cnt           281.0000    977.0000  "
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 2
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "GwZrQdjybjKO",
        "colab_type": "text"
      },
      "source": [
        "Exercise 1.01: Preprocessing temporal and weather features "
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "7gGtv-TzU3_W",
        "colab_type": "code",
        "colab": {},
        "outputId": "8a79860f-0a5c-435a-c84f-859bdebf07d2"
      },
      "source": [
        "# create a copy of the original data\n",
        "preprocessed_data = hourly_data.copy()\n",
        "\n",
        "# tranform seasons\n",
        "seasons_mapping = {1: 'winter', 2: 'spring', 3: 'summer', 4: 'fall'}\n",
        "preprocessed_data['season'] = preprocessed_data['season'].apply(lambda x: seasons_mapping[x])\n",
        "\n",
        "# transform yr\n",
        "yr_mapping = {0: 2011, 1: 2012}\n",
        "preprocessed_data['yr'] = preprocessed_data['yr'].apply(lambda x: yr_mapping[x])\n",
        "\n",
        "# transform weekday\n",
        "weekday_mapping = {0: 'Sunday', 1: 'Monday', 2: 'Tuesday', 3: 'Wednesday', 4: 'Thursday', 5: 'Friday', 6: 'Saturday'}\n",
        "preprocessed_data['weekday'] = preprocessed_data['weekday'].apply(lambda x: weekday_mapping[x])\n",
        "\n",
        "# transform weathersit\n",
        "weather_mapping = {1: 'clear', 2: 'cloudy', 3: 'light_rain_snow', 4: 'heavy_rain_snow'}\n",
        "preprocessed_data['weathersit'] = preprocessed_data['weathersit'].apply(lambda x: weather_mapping[x]) \n",
        "\n",
        "# transorm hum and windspeed\n",
        "preprocessed_data['hum'] = preprocessed_data['hum']*100\n",
        "preprocessed_data['windspeed'] = preprocessed_data['windspeed']*67\n",
        "\n",
        "# visualize preprocessed columns\n",
        "cols = ['season', 'yr', 'weekday', 'weathersit', 'hum', 'windspeed']\n",
        "preprocessed_data[cols].sample(10, random_state=123)"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/html": [
              "<div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>season</th>\n",
              "      <th>yr</th>\n",
              "      <th>weekday</th>\n",
              "      <th>weathersit</th>\n",
              "      <th>hum</th>\n",
              "      <th>windspeed</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>5792</th>\n",
              "      <td>summer</td>\n",
              "      <td>2011</td>\n",
              "      <td>Saturday</td>\n",
              "      <td>clear</td>\n",
              "      <td>74.0</td>\n",
              "      <td>8.9981</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>7823</th>\n",
              "      <td>fall</td>\n",
              "      <td>2011</td>\n",
              "      <td>Sunday</td>\n",
              "      <td>clear</td>\n",
              "      <td>43.0</td>\n",
              "      <td>31.0009</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>15426</th>\n",
              "      <td>fall</td>\n",
              "      <td>2012</td>\n",
              "      <td>Tuesday</td>\n",
              "      <td>cloudy</td>\n",
              "      <td>77.0</td>\n",
              "      <td>6.0032</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>15028</th>\n",
              "      <td>fall</td>\n",
              "      <td>2012</td>\n",
              "      <td>Sunday</td>\n",
              "      <td>clear</td>\n",
              "      <td>51.0</td>\n",
              "      <td>22.0028</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>12290</th>\n",
              "      <td>spring</td>\n",
              "      <td>2012</td>\n",
              "      <td>Friday</td>\n",
              "      <td>cloudy</td>\n",
              "      <td>89.0</td>\n",
              "      <td>12.9980</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>3262</th>\n",
              "      <td>spring</td>\n",
              "      <td>2011</td>\n",
              "      <td>Friday</td>\n",
              "      <td>clear</td>\n",
              "      <td>64.0</td>\n",
              "      <td>7.0015</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>10763</th>\n",
              "      <td>spring</td>\n",
              "      <td>2012</td>\n",
              "      <td>Thursday</td>\n",
              "      <td>clear</td>\n",
              "      <td>42.0</td>\n",
              "      <td>23.9994</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>12384</th>\n",
              "      <td>spring</td>\n",
              "      <td>2012</td>\n",
              "      <td>Tuesday</td>\n",
              "      <td>light_rain_snow</td>\n",
              "      <td>82.0</td>\n",
              "      <td>11.0014</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>6051</th>\n",
              "      <td>summer</td>\n",
              "      <td>2011</td>\n",
              "      <td>Wednesday</td>\n",
              "      <td>clear</td>\n",
              "      <td>52.0</td>\n",
              "      <td>19.0012</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>948</th>\n",
              "      <td>winter</td>\n",
              "      <td>2011</td>\n",
              "      <td>Saturday</td>\n",
              "      <td>clear</td>\n",
              "      <td>80.0</td>\n",
              "      <td>0.0000</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "</div>"
            ],
            "text/plain": [
              "       season    yr    weekday       weathersit   hum  windspeed\n",
              "5792   summer  2011   Saturday            clear  74.0     8.9981\n",
              "7823     fall  2011     Sunday            clear  43.0    31.0009\n",
              "15426    fall  2012    Tuesday           cloudy  77.0     6.0032\n",
              "15028    fall  2012     Sunday            clear  51.0    22.0028\n",
              "12290  spring  2012     Friday           cloudy  89.0    12.9980\n",
              "3262   spring  2011     Friday            clear  64.0     7.0015\n",
              "10763  spring  2012   Thursday            clear  42.0    23.9994\n",
              "12384  spring  2012    Tuesday  light_rain_snow  82.0    11.0014\n",
              "6051   summer  2011  Wednesday            clear  52.0    19.0012\n",
              "948    winter  2011   Saturday            clear  80.0     0.0000"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 3
        }
      ]
    }
  ]
}