{ "cells": [ { "cell_type": "markdown", "metadata": {}, "source": [ "# League of legends crawler" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "#### primary site for this project: [https://tracker.gg/lol](https://tracker.gg/lol)" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "This crawler is built using [selenium](https://www.selenium.dev/documentation/) using the firefox webdriver" ] }, { "cell_type": "code", "execution_count": 1, "metadata": {}, "outputs": [], "source": [ "from selenium import webdriver\n", "from selenium.webdriver.firefox.service import Service as FirefoxService\n", "from webdriver_manager.firefox import GeckoDriverManager\n", "from selenium.webdriver.common.by import By" ] }, { "cell_type": "code", "execution_count": 2, "metadata": {}, "outputs": [ { "name": "stderr", "output_type": "stream", "text": [ "[WDM] - Downloading: 19.0kB [00:00, 2.11MB/s] \n" ] } ], "source": [ "driver = webdriver.Firefox(service=FirefoxService(GeckoDriverManager().install()))\n", "\n", "driver.get(\"https://tracker.gg/lol/leaderboards/stats/all/LeaguePoints?region=NA&queueType=RANKED_SOLO_5x5\")\n", "\n", "driver.implicitly_wait(3)" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "### North America Leaderboard" ] }, { "cell_type": "code", "execution_count": 3, "metadata": {}, "outputs": [], "source": [ "na_leaderboard = driver.find_elements(By.TAG_NAME, 'tr')" ] }, { "cell_type": "code", "execution_count": 4, "metadata": {}, "outputs": [], "source": [ "# python lists to store data\n", "\n", "overall_position = []\n", "player_name = []\n", "rank = []\n", "league_points = []\n", "win_rate = []\n", "matches_played = []" ] }, { "cell_type": "code", "execution_count": 5, "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "['5', '7', '8', '13', '14', '17', '18', '18', '21', '23', '26', '27', '31', '34', '37', '39', '48', '48', '61', '65', '66', '68', '72', '75', '75', '77', '82', '82', '84', '89', '89', '94', '95', '96']\n", "['Nim', 'Neo', 'NIKJOJO', 'noinvade', 'Not', 'niewang', 'nathanwemiss', 'NALCS', 'NaCl', 'MónkeyD', 'noones', 'neopets', 'Notorious', 'nodravenday', 'No', 'newbury', 'Nice', 'Ninja', 'Not', 'Nostalgia', 'Never', 'no1care', 'NIGHTMARE', 'Need', 'Niek', 'Nickji', 'navel', 'Mystical', 'NOOB', 'not', 'MySensei', 'Not', 'NanoSpeck', 'NiMama']\n", "['Chimpsky', 'NanChangGodE', '1', 'pls', 'Hans', 'zii', 'u', 'YI', 'Tim', 'Luffy', 'fault', 'lover', 'Malli', '1', 'Focus', 'park', 'Bunz', 'Doggy', 'Viable', 'Lee', 'Story', 'me', 'BABY', 'Therapy', 'Joel', '18', 'orange', 'Firefly', 'Fredy41OO', 'skip', 'PzZzang', 'aIöne', 'v2', 'Boom']\n", "['Challenger', 'Grandmaster', 'Grandmaster', 'Grandmaster', 'Grandmaster', 'Grandmaster', 'Grandmaster', 'Grandmaster', 'Grandmaster', 'Master', 'Master', 'Master', 'Master', 'Master', 'Master', 'Master', 'Master', 'Master', 'Master', 'Master', 'Master', 'Master', 'Master', 'Master', 'Master', 'Master', 'Master', 'Master', 'Master', 'Master', 'Master', 'Master', 'Master', 'Master']\n", "['954', '822', '802', '658', '637', '557', '552', '552', '547', '509', '435', '433', '379', '345', '309', '308', '211', '211', '93', '75', '73', '65', '45', '44', '44', '36', '29', '29', '23', '18', '18', '13', '2', '1']\n", "['0%', '0%', '0%', '0%', '0%', '0%', '0%', '0%', '0%', '0%', '0%', '0%', '0%', '0%', '0%', '0%', '0%', '0%', '0%', '0%', '0%', '0%', '0%', '0%', '0%', '0%', '0%', '0%', '0%', '0%', '0%', '0%', '0%', '0%']\n" ] } ], "source": [ "for player in na_leaderboard:\n", "\n", " new_list = player.text.split()\n", "\n", " if len(new_list) == 6:\n", "\n", " overall_position.append(new_list[0])\n", " player_name.append(new_list[1])\n", " rank.append(new_list[2])\n", " league_points.append(new_list[3])\n", " win_rate.append(new_list[4])\n", " matches_played.append(new_list[5])\n", "\n", " elif len(new_list) == 7:\n", "\n", " two_name_player = new_list[1] + new_list[2]\n", "\n", " overall_position.append(new_list[0])\n", " player_name.append(two_name_player)\n", " rank.append(new_list[3])\n", " league_points.append(new_list[4])\n", " win_rate.append(new_list[5])\n", " matches_played.append(new_list[6])\n", "\n", " elif len(new_list) == 8:\n", "\n", " three_name_player = new_list[1] + new_list[2] + new_list[3]\n", "\n", " overall_position.append(new_list[0])\n", " player_name.append(three_name_player)\n", " rank.append(new_list[4])\n", " league_points.append(new_list[5])\n", " win_rate.append(new_list[6])\n", " matches_played.append(new_list[7])\n", "\n", "print(overall_position)\n", "print(player_name)\n", "print(rank)\n", "print(league_points)\n", "print(win_rate)\n", "print(matches_played)" ] }, { "cell_type": "code", "execution_count": 6, "metadata": {}, "outputs": [], "source": [ "# check if lists are empty\n", "\n", "assert (overall_position != []), 'list is empty'\n", "assert (player_name != []), 'list is empty'\n", "assert (rank != []), 'list is empty'\n", "assert (league_points != []), 'list is empty'\n", "assert (win_rate != []), 'list is empty'\n", "assert (matches_played != []), 'list is empty'" ] }, { "cell_type": "code", "execution_count": 7, "metadata": {}, "outputs": [], "source": [ "# closing the driver session\n", "\n", "driver.quit()" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "## Generate data frame and csv file" ] }, { "cell_type": "code", "execution_count": 8, "metadata": {}, "outputs": [], "source": [ "import pandas as pd" ] }, { "cell_type": "code", "execution_count": 9, "metadata": {}, "outputs": [ { "data": { "text/html": [ "
\n", "\n", "\n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", "
overall_positionplayer_namerankleague_pointswin_ratematches_played
05NimChimpskyChallenger9540%
17NeoNanChangGodEGrandmaster8220%
28NIKJOJO1Grandmaster8020%
313noinvadeplsGrandmaster6580%
414NotHansGrandmaster6370%
517niewangziiGrandmaster5570%
618nathanwemissuGrandmaster5520%
718NALCSYIGrandmaster5520%
821NaClTimGrandmaster5470%
923MónkeyDLuffyMaster5090%
1026noonesfaultMaster4350%
1127neopetsloverMaster4330%
1231NotoriousMalliMaster3790%
1334nodravenday1Master3450%
1437NoFocusMaster3090%
1539newburyparkMaster3080%
1648NiceBunzMaster2110%
1748NinjaDoggyMaster2110%
1861NotViableMaster930%
1965NostalgiaLeeMaster750%
2066NeverStoryMaster730%
2168no1caremeMaster650%
2272NIGHTMAREBABYMaster450%
2375NeedTherapyMaster440%
2475NiekJoelMaster440%
2577Nickji18Master360%
2682navelorangeMaster290%
2782MysticalFireflyMaster290%
2884NOOBFredy41OOMaster230%
2989notskipMaster180%
3089MySenseiPzZzangMaster180%
3194NotaIöneMaster130%
3295NanoSpeckv2Master20%
3396NiMamaBoomMaster10%
\n", "
" ], "text/plain": [ " overall_position player_name rank league_points win_rate \\\n", "0 5 Nim Chimpsky Challenger 954 \n", "1 7 Neo NanChangGodE Grandmaster 822 \n", "2 8 NIKJOJO 1 Grandmaster 802 \n", "3 13 noinvade pls Grandmaster 658 \n", "4 14 Not Hans Grandmaster 637 \n", "5 17 niewang zii Grandmaster 557 \n", "6 18 nathanwemiss u Grandmaster 552 \n", "7 18 NALCS YI Grandmaster 552 \n", "8 21 NaCl Tim Grandmaster 547 \n", "9 23 MónkeyD Luffy Master 509 \n", "10 26 noones fault Master 435 \n", "11 27 neopets lover Master 433 \n", "12 31 Notorious Malli Master 379 \n", "13 34 nodravenday 1 Master 345 \n", "14 37 No Focus Master 309 \n", "15 39 newbury park Master 308 \n", "16 48 Nice Bunz Master 211 \n", "17 48 Ninja Doggy Master 211 \n", "18 61 Not Viable Master 93 \n", "19 65 Nostalgia Lee Master 75 \n", "20 66 Never Story Master 73 \n", "21 68 no1care me Master 65 \n", "22 72 NIGHTMARE BABY Master 45 \n", "23 75 Need Therapy Master 44 \n", "24 75 Niek Joel Master 44 \n", "25 77 Nickji 18 Master 36 \n", "26 82 navel orange Master 29 \n", "27 82 Mystical Firefly Master 29 \n", "28 84 NOOB Fredy41OO Master 23 \n", "29 89 not skip Master 18 \n", "30 89 MySensei PzZzang Master 18 \n", "31 94 Not aIöne Master 13 \n", "32 95 NanoSpeck v2 Master 2 \n", "33 96 NiMama Boom Master 1 \n", "\n", " matches_played \n", "0 0% \n", "1 0% \n", "2 0% \n", "3 0% \n", "4 0% \n", "5 0% \n", "6 0% \n", "7 0% \n", "8 0% \n", "9 0% \n", "10 0% \n", "11 0% \n", "12 0% \n", "13 0% \n", "14 0% \n", "15 0% \n", "16 0% \n", "17 0% \n", "18 0% \n", "19 0% \n", "20 0% \n", "21 0% \n", "22 0% \n", "23 0% \n", "24 0% \n", "25 0% \n", "26 0% \n", "27 0% \n", "28 0% \n", "29 0% \n", "30 0% \n", "31 0% \n", "32 0% \n", "33 0% " ] }, "execution_count": 9, "metadata": {}, "output_type": "execute_result" } ], "source": [ "# create a dataframe\n", "\n", "df = pd.DataFrame({\n", " \"overall_position\": overall_position,\n", " \"player_name\": player_name,\n", " \"rank\": rank,\n", " \"league_points\": league_points,\n", " \"win_rate\": win_rate,\n", " \"matches_played\": matches_played\n", "})\n", "\n", "df" ] }, { "cell_type": "code", "execution_count": 10, "metadata": {}, "outputs": [], "source": [ "# create csv file\n", "\n", "df.to_csv(\"na_leaderboard.csv\", index=False)" ] } ], "metadata": { "kernelspec": { "display_name": "Python 3.10.5 ('.env': venv)", "language": "python", "name": "python3" }, "language_info": { "codemirror_mode": { "name": "ipython", "version": 3 }, "file_extension": ".py", "mimetype": "text/x-python", "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", "version": "3.10.5" }, "orig_nbformat": 4, "vscode": { "interpreter": { "hash": "323ffd80a3d1bd87652eef821899663beb3891630d214e0d2358d356c8194408" } } }, "nbformat": 4, "nbformat_minor": 2 }