| Software | Version |
|---|
| Python | 3.7.1 64bit [Clang 4.0.1 (tags/RELEASE_401/final)] |
| IPython | 7.2.0 |
| OS | Darwin 18.2.0 x86_64 i386 64bit |
| pandas | 0.23.4 |
| Mon Apr 29 07:15:04 2019 PDT |
"
],
"text/latex": [
"\\begin{tabular}{|l|l|}\\hline\n",
"{\\bf Software} & {\\bf Version} \\\\ \\hline\\hline\n",
"Python & 3.7.1 64bit [Clang 4.0.1 (tags/RELEASE\\_401/final)] \\\\ \\hline\n",
"IPython & 7.2.0 \\\\ \\hline\n",
"OS & Darwin 18.2.0 x86\\_64 i386 64bit \\\\ \\hline\n",
"pandas & 0.23.4 \\\\ \\hline\n",
"\\hline \\multicolumn{2}{|l|}{Mon Apr 29 07:15:04 2019 PDT} \\\\ \\hline\n",
"\\end{tabular}\n"
],
"text/plain": [
"Software versions\n",
"Python 3.7.1 64bit [Clang 4.0.1 (tags/RELEASE_401/final)]\n",
"IPython 7.2.0\n",
"OS Darwin 18.2.0 x86_64 i386 64bit\n",
"pandas 0.23.4\n",
"Mon Apr 29 07:15:04 2019 PDT"
]
},
"execution_count": 26,
"metadata": {},
"output_type": "execute_result"
}
],
"source": [
"from IPython.display import HTML\n",
"HTML('')\n",
"%load_ext version_information\n",
"%version_information pandas"
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"# Goalie Pull Bayes Optimize\n",
"\n",
" - Parse the goalie pull stats we need\n",
" \n",
"### This script is for the current format: *2007/2008* and later"
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"## Parse HTML Stats Table"
]
},
{
"cell_type": "code",
"execution_count": 27,
"metadata": {},
"outputs": [],
"source": [
"import pandas as pd\n",
"import numpy as np\n",
"import requests\n",
"from bs4 import BeautifulSoup\n",
"import os\n",
"import re\n",
"import datetime\n",
"import time\n",
"from collections import OrderedDict\n",
"import glob\n",
"from tqdm import tqdm_notebook\n",
"from colorama import Fore, Style"
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"### Helper functions"
]
},
{
"cell_type": "code",
"execution_count": 28,
"metadata": {},
"outputs": [],
"source": [
"def is_int(x):\n",
" try:\n",
" int(x)\n",
" return True\n",
" except:\n",
" return False\n",
"\n",
"def parse_time(s):\n",
" _s = s.copy()\n",
" _s = _s.apply(lambda x:\n",
" datetime.datetime.strptime(x.split(' ')[0], '%M:%S')\n",
" )\n",
" _s = _s.apply(lambda x:\n",
" datetime.timedelta(\n",
" hours=x.hour,\n",
" minutes=x.minute,\n",
" seconds=x.second,\n",
" microseconds=x.microsecond\n",
" ))\n",
" return _s\n",
"\n",
"def parse_team(s):\n",
" _s = s.copy()\n",
" # Add '.' as a hack in case _s is empty\n",
" _s = (_s + '.').str.split().apply(lambda x: x[0])\n",
" return _s\n",
"\n",
"def parse_date(s):\n",
" _s = s.copy()\n",
" \n",
" # invalid parsing will be set as NaT\n",
" _s = pd.to_datetime(_s, errors='coerce')\n",
"\n",
" '''\n",
" _s = _s.apply(lambda x: (re.search(',(.*)', x.text.splitlines()[2])\n",
" .group(1)\n",
" .strip()))\n",
" _s = _s.apply(lambda x: datetime.datetime\\\n",
" .strptime(x.strip(), '%B %d, %Y'))\n",
" '''\n",
" \n",
" return _s\n",
"\n",
"def goalie_pull_timedelta(s1, s2):\n",
" out = []\n",
" for d1, d2 in zip(s1.values, s2.values):\n",
" try:\n",
" o = d1 - d2\n",
" except:\n",
" o = float('nan')\n",
" out.append(o)\n",
" return out"
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"### Main Functions"
]
},
{
"cell_type": "code",
"execution_count": 29,
"metadata": {},
"outputs": [],
"source": [
"def get_game_df(\n",
" soup,\n",
" local_html=True,\n",
") -> pd.DataFrame:\n",
" '''\n",
" Returns the parsed HTML stats table for some game and returns\n",
" a DataFrame.\n",
" \n",
" local_html : bool\n",
" Set True if using local HTML file. Otherwise set False\n",
" for requested HTML from nhl.com\n",
" '''\n",
"\n",
" if not local_html:\n",
" raise NotImplementedError('Only local HTML supported')\n",
"\n",
" # Parse the main table\n",
" data = []\n",
" for row in soup.find_all('tr', {'class': 'evenColor'}):\n",
" d = []\n",
" for i, row_data in enumerate(row.find_all('td')):\n",
" if 'class' in row_data.attrs:\n",
" classes = ' '.join(row_data.attrs['class'])\n",
" if 'bborder' not in classes:\n",
" continue\n",
" else:\n",
" continue\n",
"\n",
" if i == 3:\n",
" # Parse the time\n",
" row_data_text = ' '.join([str(s) for s in row_data.contents])\n",
"\n",
" elif i >= 6:\n",
" # Parse the players on ice\n",
" row_data_text = ' '.join(re.findall('[a-zA-Z]+', row_data.text.strip()))\n",
"\n",
" else:\n",
" # Standard parse\n",
" row_data_text = row_data.text.strip()\n",
" \n",
" d.append(row_data_text)\n",
"\n",
" if len(d) >= 8:\n",
" data.append(d[:8])\n",
" \n",
" cols = ['#', 'per', 'type', 'time', 'event', 'description', 'visitor_on_ice', 'home_on_ice']\n",
" df = pd.DataFrame(data, columns=cols)\n",
" \n",
" good_rows = df['#'].apply(is_int)\n",
" df = df.loc[good_rows]\n",
" df['#'] = df['#'].astype(int)\n",
" df['per'] = df['per'].astype(int)\n",
" df['time'] = parse_time(df['time'])\n",
" df['team'] = parse_team(df['description'])\n",
" \n",
" # Get the team info\n",
" team_names = []\n",
" for row_data in soup.find_all('td', {'class': 'heading + bborder'}):\n",
" if 'on ice' in row_data.text.lower():\n",
" team_names.append(row_data.text.split()[0])\n",
" if len(team_names) == 2:\n",
" break\n",
" if len(team_names) != 2:\n",
" team_names = ['', '']\n",
" team_info = {'visitor': team_names[0], 'home': team_names[1]}\n",
" \n",
" return df, team_info"
]
},
{
"cell_type": "code",
"execution_count": 30,
"metadata": {},
"outputs": [],
"source": [
"def goalie_pull_game_search(\n",
" game_df: pd.DataFrame,\n",
" team_info: dict,\n",
" verbose=False,\n",
") -> list:\n",
" '''\n",
" Search through a DataFrame stats table for goalie pulls, and\n",
" finds empty net goals / games that end with a goalie pulled.\n",
" \n",
" team_info : dict\n",
" Visitor and away team names.\n",
" '''\n",
" out = []\n",
" pull_threshold = datetime.timedelta(minutes=15)\n",
"\n",
" prev_row = None\n",
" prev_row_players = ['', '']\n",
" pull_team = ''\n",
" pull_switch = False\n",
" goal_switch = False\n",
" o = {}\n",
" \n",
" # Break once we hit the final row of the 3rd period\n",
" i_final_row = game_df[game_df.per == 3].index[-1]\n",
"\n",
" for i, row in game_df.iterrows():\n",
" if verbose:\n",
" print(f'Looking at row {i}, # = {row[\"#\"]}')\n",
" if prev_row is None:\n",
" prev_row = row.copy()\n",
" if i == i_final_row:\n",
" # Check if game ended 5 on 6\n",
" if o:\n",
" print(Fore.RED + f'Game end with no goalie in net!' + Style.RESET_ALL)\n",
" o['game_end_time'] = row.time\n",
" out.append(o.copy())\n",
" break\n",
" if row.per != 3:\n",
" if verbose:\n",
" print('Not in 3rd period, skipping row')\n",
" continue\n",
" \n",
" visitor_g_pull = 'G' not in row.visitor_on_ice\n",
" home_g_pull = 'G' not in row.home_on_ice\n",
" candidate_row = all((row.visitor_on_ice.strip(),\n",
" row.home_on_ice.strip()))\n",
" if not any((visitor_g_pull, home_g_pull)):\n",
" # No goalie is pulled. Set switch off\n",
" pull_switch = False\n",
" o = {}\n",
" \n",
" if (visitor_g_pull or home_g_pull) and candidate_row:\n",
" if not pull_switch:\n",
" # The goalie was just pulled\n",
" pull_switch = True\n",
" o = {}\n",
" if visitor_g_pull:\n",
" pull_team = 'visitor'\n",
" elif home_g_pull:\n",
" pull_team = 'home'\n",
" else:\n",
" raise ValueError('Home or away team must have goalie pulled')\n",
" o['team_name'] = team_info.get(pull_team, '')\n",
" o['pull_period'] = row.per\n",
" o['pull_time'] = (row.time + prev_row.time) / 2\n",
" if verbose:\n",
" print(f'team, period, time = \\n{o}')\n",
" \n",
" # Search for an event (goal for / against / end of game)\n",
" if pull_switch:\n",
" if verbose:\n",
" print('Searching for a goal or end of game')\n",
" if row.event.lower() == 'goal':\n",
" print(f'Found goal after a pull, setting goal against time as {row.time}')\n",
" if row.team == o['team_name']:\n",
" o['goal_for_time'] = row.time\n",
" else:\n",
" o['goal_against_time'] = row.time\n",
" \n",
" if row.time == o['pull_time']:\n",
" print(f'Goal time equal to pull time {row.time}. '\n",
" 'Ignoring it and searching again.')\n",
" continue\n",
" \n",
" if ((pull_team == 'visitor' and visitor_g_pull)\n",
" or (pull_team == 'home' and home_g_pull)):\n",
" # It's a true empty net goal\n",
" print(Fore.RED + f'Found empty net goal:\\n{o}' + Style.RESET_ALL)\n",
" out.append(o.copy())\n",
" else:\n",
" print('Goalie was back in the net')\n",
" pass\n",
" o = {}\n",
" pull_switch = False\n",
"\n",
" prev_row = row.copy()\n",
"\n",
" return out\n",
"\n",
"\n",
"def get_game_meta(soup) -> dict:\n",
" '''\n",
" Get game metadata like the date and game number.\n",
" '''\n",
" try:\n",
" center_el = soup.find_all('table', {'id': 'GameInfo'})[0]\n",
" game_number = center_el.find_all('tr')[6].text.split()[-1].lstrip('0')\n",
" date = center_el.find_all('tr')[3].text.strip()\n",
" return {'game_number': game_number, 'date': date}\n",
" except:\n",
" print('Failed to get meta data :(')\n",
" return {}"
]
},
{
"cell_type": "code",
"execution_count": 31,
"metadata": {},
"outputs": [],
"source": [
"def parse_game(\n",
" soup: BeautifulSoup,\n",
" cols: list = None,\n",
" season='',\n",
" verbose=False\n",
") -> list:\n",
" '''\n",
" Search for empty net goals for, against and game ends,\n",
" for a given game. Input the HTML as a BeautifulSoup object.\n",
" '''\n",
" out = []\n",
" \n",
" # Default columns\n",
" if cols is None:\n",
" cols = [\n",
" 'season',\n",
" 'game_number',\n",
" 'team_name',\n",
" 'date',\n",
" 'pull_period',\n",
" 'pull_time',\n",
" 'goal_for_time',\n",
" 'goal_against_time',\n",
" 'game_end_time',\n",
" 'goal_for_timedelta',\n",
" 'goal_against_timedelta',\n",
" 'game_end_timedelta',\n",
" ]\n",
"\n",
" # Get the game stats df \n",
" game_df, team_info = get_game_df(soup)\n",
" \n",
" # Get the game metadata\n",
" meta_data = get_game_meta(soup)\n",
" if season:\n",
" meta_data['season'] = season\n",
" \n",
" # Check for goalie pulls\n",
" visitor_goalie_pull_idx = ~(game_df.visitor_on_ice.str.contains('G'))\n",
" home_goalie_pull_idx = ~(game_df.home_on_ice.str.contains('G'))\n",
" if not (visitor_goalie_pull_idx.sum() + home_goalie_pull_idx.sum()):\n",
" return []\n",
" \n",
" # For each goalie pull, determine the outcome\n",
" game_5on6_data = goalie_pull_game_search(\n",
" game_df, team_info, verbose=verbose\n",
" )\n",
" \n",
" if not game_5on6_data:\n",
" return []\n",
" \n",
" out = []\n",
" for row in game_5on6_data:\n",
" row.update(meta_data)\n",
" out.append([row.get(col, float('nan')) for col in cols])\n",
"\n",
" return out\n",
"\n",
"\n",
"def make_final_df(\n",
" data: list,\n",
" cols: list,\n",
") -> pd.DataFrame:\n",
" '''\n",
" Return table with columns:\n",
" - season\n",
" - game number\n",
" - team name\n",
" - date\n",
" - period of pull\n",
" - time of pull\n",
" - time of goal for\n",
" - time of goal against\n",
" - time of goal end\n",
" - timedelta of goal for (since pull)\n",
" - timedelta of goal against (since pull)\n",
" - timedelta of game end (since pull)\n",
" \n",
" Each row represents a goalie pull event.\n",
" Goalie pull events that don't result in\n",
" one of the above are ignored.\n",
" '''\n",
" df = pd.DataFrame(data, columns=cols)\n",
" df['date'] = parse_date(df.date.fillna('').astype(str))\n",
" df['goal_for_timedelta'] = goalie_pull_timedelta(df.goal_for_time, df.pull_time)\n",
" df['goal_against_timedelta'] = goalie_pull_timedelta(df.goal_against_time, df.pull_time)\n",
" df['game_end_timedelta'] = goalie_pull_timedelta(df.game_end_time, df.pull_time)\n",
" \n",
" return df"
]
},
{
"cell_type": "code",
"execution_count": 32,
"metadata": {},
"outputs": [],
"source": [
"def parse_game_range(\n",
" seasons: list,\n",
" test: bool = False,\n",
" verbose: bool = False\n",
") -> pd.DataFrame:\n",
" '''\n",
" Parse every game for a given season.\n",
" \n",
" Folder structure is:\n",
" ../../data/raw/html/{season}/{game_number}.html\n",
" \n",
" '''\n",
" \n",
" root_data_path = '../../data/raw/html'\n",
" if not os.path.exists(root_data_path):\n",
" print(f'Root data path not found ({root_data_path})')\n",
" return None\n",
" \n",
" cols = [\n",
" 'season',\n",
" 'game_number',\n",
" 'team_name',\n",
" 'date',\n",
" 'pull_period',\n",
" 'pull_time',\n",
" 'goal_for_time',\n",
" 'goal_against_time',\n",
" 'game_end_time',\n",
" 'goal_for_timedelta',\n",
" 'goal_against_timedelta',\n",
" 'game_end_timedelta',\n",
" ] \n",
" data = []\n",
" for season in seasons: \n",
" search_string = os.path.join(root_data_path, season, '*.html')\n",
" html_files = sorted(glob.glob(search_string),\n",
" key=lambda x: int(re.match(r'(\\d+)', os.path.split(x)[-1]).group(1)))\n",
" print(f'Found {len(html_files)} files')\n",
" i = 0\n",
" for file in tqdm_notebook(html_files):\n",
" i += 1\n",
" if test:\n",
" return_condition = i%20 == 0\n",
" if return_condition: \n",
" print(f'Testing mode - stopping script at {return_condition}')\n",
" return data, make_final_df(data, cols)\n",
" \n",
" print(f'Processing file {file}')\n",
" try:\n",
" if season in ('20072008', '20082009',):\n",
" parser = 'html.parser'\n",
" else:\n",
" parser = 'lxml'\n",
" with open(file, 'r') as f:\n",
" page_text = f.read()\n",
" soup = BeautifulSoup(page_text, parser)\n",
" except Exception as e:\n",
" print(f'Unable to read/parse file {file}')\n",
" print(str(e))\n",
" continue\n",
" \n",
" try:\n",
" d = parse_game(soup, cols, season, verbose)\n",
" if not d:\n",
" continue\n",
" except Exception as e:\n",
" print(f'Unable to parse game for file {file}')\n",
" print(str(e))\n",
" continue\n",
"\n",
" data += d\n",
" \n",
" df = make_final_df(data, cols)\n",
" return df"
]
},
{
"cell_type": "code",
"execution_count": 33,
"metadata": {},
"outputs": [],
"source": [
"def test_parse_game_range(season, game_number, verbose=True):\n",
" '''\n",
" Parse a specific game with verbose output.\n",
" '''\n",
" \n",
" cols = [\n",
" 'season',\n",
" 'game_number',\n",
" 'team_name',\n",
" 'date',\n",
" 'pull_period',\n",
" 'pull_time',\n",
" 'goal_for_time',\n",
" 'goal_against_time',\n",
" 'game_end_time',\n",
" 'goal_for_timedelta',\n",
" 'goal_against_timedelta',\n",
" 'game_end_timedelta',\n",
" ] \n",
" data = []\n",
" \n",
" file = f'../../data/raw/html/{season}/{game_number}.html'\n",
" print(f'Processing file {file}')\n",
" with open(file, 'r') as f:\n",
" try:\n",
" page_text = f.read()\n",
" soup = BeautifulSoup(page_text, 'lxml')\n",
" except Exception as e:\n",
" print(f'Unable to read/parse file {file}')\n",
" raise e\n",
"\n",
" try:\n",
" d = parse_game(soup, cols, season, verbose)\n",
" if not d:\n",
" print('No return from parse_game')\n",
" except Exception as e:\n",
" print(f'Unable to parse game for file {file}')\n",
" raise e\n",
"\n",
" data = d\n",
" df = make_final_df(data, cols)\n",
" return df"
]
},
{
"cell_type": "code",
"execution_count": 9,
"metadata": {},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"Processing file ../../data/raw/html/20072008/20.html\n",
"No return from parse_game\n"
]
},
{
"data": {
"text/html": [
"