{ "cells": [ { "cell_type": "markdown", "metadata": {}, "source": [ "# League of legends crawler" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "#### primary site for this project: [https://tracker.gg/lol](https://tracker.gg/lol)" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "This crawler is built using [selenium](https://www.selenium.dev/documentation/) using the firefox webdriver" ] }, { "cell_type": "code", "execution_count": 1, "metadata": {}, "outputs": [], "source": [ "from selenium import webdriver\n", "from selenium.webdriver.firefox.service import Service as FirefoxService\n", "from webdriver_manager.firefox import GeckoDriverManager\n", "from selenium.webdriver.common.by import By" ] }, { "cell_type": "code", "execution_count": 2, "metadata": {}, "outputs": [ { "name": "stderr", "output_type": "stream", "text": [ "[WDM] - Downloading: 19.0kB [00:00, 2.11MB/s] \n" ] } ], "source": [ "driver = webdriver.Firefox(service=FirefoxService(GeckoDriverManager().install()))\n", "\n", "driver.get(\"https://tracker.gg/lol/leaderboards/stats/all/LeaguePoints?region=NA&queueType=RANKED_SOLO_5x5\")\n", "\n", "driver.implicitly_wait(3)" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "### North America Leaderboard" ] }, { "cell_type": "code", "execution_count": 3, "metadata": {}, "outputs": [], "source": [ "na_leaderboard = driver.find_elements(By.TAG_NAME, 'tr')" ] }, { "cell_type": "code", "execution_count": 4, "metadata": {}, "outputs": [], "source": [ "# python lists to store data\n", "\n", "overall_position = []\n", "player_name = []\n", "rank = []\n", "league_points = []\n", "win_rate = []\n", "matches_played = []" ] }, { "cell_type": "code", "execution_count": 5, "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "['5', '7', '8', '13', '14', '17', '18', '18', '21', '23', '26', '27', '31', '34', '37', '39', '48', '48', '61', '65', '66', '68', '72', '75', '75', '77', '82', '82', '84', '89', '89', '94', '95', '96']\n", "['Nim', 'Neo', 'NIKJOJO', 'noinvade', 'Not', 'niewang', 'nathanwemiss', 'NALCS', 'NaCl', 'MónkeyD', 'noones', 'neopets', 'Notorious', 'nodravenday', 'No', 'newbury', 'Nice', 'Ninja', 'Not', 'Nostalgia', 'Never', 'no1care', 'NIGHTMARE', 'Need', 'Niek', 'Nickji', 'navel', 'Mystical', 'NOOB', 'not', 'MySensei', 'Not', 'NanoSpeck', 'NiMama']\n", "['Chimpsky', 'NanChangGodE', '1', 'pls', 'Hans', 'zii', 'u', 'YI', 'Tim', 'Luffy', 'fault', 'lover', 'Malli', '1', 'Focus', 'park', 'Bunz', 'Doggy', 'Viable', 'Lee', 'Story', 'me', 'BABY', 'Therapy', 'Joel', '18', 'orange', 'Firefly', 'Fredy41OO', 'skip', 'PzZzang', 'aIöne', 'v2', 'Boom']\n", "['Challenger', 'Grandmaster', 'Grandmaster', 'Grandmaster', 'Grandmaster', 'Grandmaster', 'Grandmaster', 'Grandmaster', 'Grandmaster', 'Master', 'Master', 'Master', 'Master', 'Master', 'Master', 'Master', 'Master', 'Master', 'Master', 'Master', 'Master', 'Master', 'Master', 'Master', 'Master', 'Master', 'Master', 'Master', 'Master', 'Master', 'Master', 'Master', 'Master', 'Master']\n", "['954', '822', '802', '658', '637', '557', '552', '552', '547', '509', '435', '433', '379', '345', '309', '308', '211', '211', '93', '75', '73', '65', '45', '44', '44', '36', '29', '29', '23', '18', '18', '13', '2', '1']\n", "['0%', '0%', '0%', '0%', '0%', '0%', '0%', '0%', '0%', '0%', '0%', '0%', '0%', '0%', '0%', '0%', '0%', '0%', '0%', '0%', '0%', '0%', '0%', '0%', '0%', '0%', '0%', '0%', '0%', '0%', '0%', '0%', '0%', '0%']\n" ] } ], "source": [ "for player in na_leaderboard:\n", "\n", " new_list = player.text.split()\n", "\n", " if len(new_list) == 6:\n", "\n", " overall_position.append(new_list[0])\n", " player_name.append(new_list[1])\n", " rank.append(new_list[2])\n", " league_points.append(new_list[3])\n", " win_rate.append(new_list[4])\n", " matches_played.append(new_list[5])\n", "\n", " elif len(new_list) == 7:\n", "\n", " two_name_player = new_list[1] + new_list[2]\n", "\n", " overall_position.append(new_list[0])\n", " player_name.append(two_name_player)\n", " rank.append(new_list[3])\n", " league_points.append(new_list[4])\n", " win_rate.append(new_list[5])\n", " matches_played.append(new_list[6])\n", "\n", " elif len(new_list) == 8:\n", "\n", " three_name_player = new_list[1] + new_list[2] + new_list[3]\n", "\n", " overall_position.append(new_list[0])\n", " player_name.append(three_name_player)\n", " rank.append(new_list[4])\n", " league_points.append(new_list[5])\n", " win_rate.append(new_list[6])\n", " matches_played.append(new_list[7])\n", "\n", "print(overall_position)\n", "print(player_name)\n", "print(rank)\n", "print(league_points)\n", "print(win_rate)\n", "print(matches_played)" ] }, { "cell_type": "code", "execution_count": 6, "metadata": {}, "outputs": [], "source": [ "# check if lists are empty\n", "\n", "assert (overall_position != []), 'list is empty'\n", "assert (player_name != []), 'list is empty'\n", "assert (rank != []), 'list is empty'\n", "assert (league_points != []), 'list is empty'\n", "assert (win_rate != []), 'list is empty'\n", "assert (matches_played != []), 'list is empty'" ] }, { "cell_type": "code", "execution_count": 7, "metadata": {}, "outputs": [], "source": [ "# closing the driver session\n", "\n", "driver.quit()" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "## Generate data frame and csv file" ] }, { "cell_type": "code", "execution_count": 8, "metadata": {}, "outputs": [], "source": [ "import pandas as pd" ] }, { "cell_type": "code", "execution_count": 9, "metadata": {}, "outputs": [ { "data": { "text/html": [ "
\n", " | overall_position | \n", "player_name | \n", "rank | \n", "league_points | \n", "win_rate | \n", "matches_played | \n", "
---|---|---|---|---|---|---|
0 | \n", "5 | \n", "Nim | \n", "Chimpsky | \n", "Challenger | \n", "954 | \n", "0% | \n", "
1 | \n", "7 | \n", "Neo | \n", "NanChangGodE | \n", "Grandmaster | \n", "822 | \n", "0% | \n", "
2 | \n", "8 | \n", "NIKJOJO | \n", "1 | \n", "Grandmaster | \n", "802 | \n", "0% | \n", "
3 | \n", "13 | \n", "noinvade | \n", "pls | \n", "Grandmaster | \n", "658 | \n", "0% | \n", "
4 | \n", "14 | \n", "Not | \n", "Hans | \n", "Grandmaster | \n", "637 | \n", "0% | \n", "
5 | \n", "17 | \n", "niewang | \n", "zii | \n", "Grandmaster | \n", "557 | \n", "0% | \n", "
6 | \n", "18 | \n", "nathanwemiss | \n", "u | \n", "Grandmaster | \n", "552 | \n", "0% | \n", "
7 | \n", "18 | \n", "NALCS | \n", "YI | \n", "Grandmaster | \n", "552 | \n", "0% | \n", "
8 | \n", "21 | \n", "NaCl | \n", "Tim | \n", "Grandmaster | \n", "547 | \n", "0% | \n", "
9 | \n", "23 | \n", "MónkeyD | \n", "Luffy | \n", "Master | \n", "509 | \n", "0% | \n", "
10 | \n", "26 | \n", "noones | \n", "fault | \n", "Master | \n", "435 | \n", "0% | \n", "
11 | \n", "27 | \n", "neopets | \n", "lover | \n", "Master | \n", "433 | \n", "0% | \n", "
12 | \n", "31 | \n", "Notorious | \n", "Malli | \n", "Master | \n", "379 | \n", "0% | \n", "
13 | \n", "34 | \n", "nodravenday | \n", "1 | \n", "Master | \n", "345 | \n", "0% | \n", "
14 | \n", "37 | \n", "No | \n", "Focus | \n", "Master | \n", "309 | \n", "0% | \n", "
15 | \n", "39 | \n", "newbury | \n", "park | \n", "Master | \n", "308 | \n", "0% | \n", "
16 | \n", "48 | \n", "Nice | \n", "Bunz | \n", "Master | \n", "211 | \n", "0% | \n", "
17 | \n", "48 | \n", "Ninja | \n", "Doggy | \n", "Master | \n", "211 | \n", "0% | \n", "
18 | \n", "61 | \n", "Not | \n", "Viable | \n", "Master | \n", "93 | \n", "0% | \n", "
19 | \n", "65 | \n", "Nostalgia | \n", "Lee | \n", "Master | \n", "75 | \n", "0% | \n", "
20 | \n", "66 | \n", "Never | \n", "Story | \n", "Master | \n", "73 | \n", "0% | \n", "
21 | \n", "68 | \n", "no1care | \n", "me | \n", "Master | \n", "65 | \n", "0% | \n", "
22 | \n", "72 | \n", "NIGHTMARE | \n", "BABY | \n", "Master | \n", "45 | \n", "0% | \n", "
23 | \n", "75 | \n", "Need | \n", "Therapy | \n", "Master | \n", "44 | \n", "0% | \n", "
24 | \n", "75 | \n", "Niek | \n", "Joel | \n", "Master | \n", "44 | \n", "0% | \n", "
25 | \n", "77 | \n", "Nickji | \n", "18 | \n", "Master | \n", "36 | \n", "0% | \n", "
26 | \n", "82 | \n", "navel | \n", "orange | \n", "Master | \n", "29 | \n", "0% | \n", "
27 | \n", "82 | \n", "Mystical | \n", "Firefly | \n", "Master | \n", "29 | \n", "0% | \n", "
28 | \n", "84 | \n", "NOOB | \n", "Fredy41OO | \n", "Master | \n", "23 | \n", "0% | \n", "
29 | \n", "89 | \n", "not | \n", "skip | \n", "Master | \n", "18 | \n", "0% | \n", "
30 | \n", "89 | \n", "MySensei | \n", "PzZzang | \n", "Master | \n", "18 | \n", "0% | \n", "
31 | \n", "94 | \n", "Not | \n", "aIöne | \n", "Master | \n", "13 | \n", "0% | \n", "
32 | \n", "95 | \n", "NanoSpeck | \n", "v2 | \n", "Master | \n", "2 | \n", "0% | \n", "
33 | \n", "96 | \n", "NiMama | \n", "Boom | \n", "Master | \n", "1 | \n", "0% | \n", "