{ "cells": [ { "cell_type": "markdown", "metadata": {}, "source": [ "# Title: msticpy - Folium Map Plotting\n", "\n", "## Introduction\n", "This module contains a class that wraps the `folium` package to plot geo-location data.\n", "\n", "Read the [Folium documentation](https://python-visualization.github.io/folium/)\n" ] }, { "cell_type": "markdown", "metadata": { "toc": true }, "source": [ "

Table of Contents

\n", "
" ] }, { "cell_type": "code", "execution_count": 1, "metadata": { "ExecuteTime": { "end_time": "2020-02-08T01:01:41.418156Z", "start_time": "2020-02-08T01:01:39.342857Z" } }, "outputs": [], "source": [ "# Imports\n", "import sys\n", "MIN_REQ_PYTHON = (3,6)\n", "if sys.version_info < MIN_REQ_PYTHON:\n", " print('Check the Kernel->Change Kernel menu and ensure that Python 3.6')\n", " print('or later is selected as the active kernel.')\n", " sys.exit(\"Python %s.%s or later is required.\\n\" % MIN_REQ_PYTHON)\n", "\n", "\n", "from IPython.display import display\n", "import pandas as pd\n", "\n", "import msticpy.sectools as sectools\n", "from msticpy.nbtools import *\n", "from msticpy.nbtools.entityschema import IpAddress, GeoLocation\n", "from msticpy.nbtools.foliummap import FoliumMap" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "## FoliumMap class\n", "\n", "```\n", "FoliumMap(\n", " title: str = 'layer1',\n", " zoom_start: float = 2.5,\n", " tiles=None,\n", " width: str = '100%',\n", " height: str = '100%',\n", " location: list = None,\n", ")\n", "Wrapper class for Folium/Leaflet mapping.\n", "\n", "Parameters\n", "----------\n", "title : str, optional\n", " Name of the layer (the default is 'layer1')\n", "zoom_start : int, optional\n", " The zoom level of the map (the default is 7)\n", "tiles : [type], optional\n", " Custom set of tiles or tile URL (the default is None)\n", "width : str, optional\n", " Map display width (the default is '100%')\n", "height : str, optional\n", " Map display height (the default is '100%')\n", "location : list, optional\n", " Location to center map on\n", "\n", "Attributes\n", "----------\n", "folium_map : folium.Map\n", "```" ] }, { "cell_type": "code", "execution_count": 2, "metadata": { "ExecuteTime": { "end_time": "2020-02-08T01:01:41.435147Z", "start_time": "2020-02-08T01:01:41.419155Z" }, "scrolled": true }, "outputs": [ { "data": { "text/html": [ "
Make this Notebook Trusted to load map: File -> Trust Notebook
" ], "text/plain": [ "" ] }, "execution_count": 2, "metadata": {}, "output_type": "execute_result" } ], "source": [ "folium_map = FoliumMap(width=\"50%\", height=\"50%\", location=(47.5982328,-122.331), zoom_start=14)\n", "folium_map" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "The underlying folium map object is accessible as the `folium_map` attribute" ] }, { "cell_type": "code", "execution_count": 3, "metadata": { "ExecuteTime": { "end_time": "2020-02-08T01:01:41.441144Z", "start_time": "2020-02-08T01:01:41.436631Z" } }, "outputs": [ { "data": { "text/plain": [ "folium.folium.Map" ] }, "execution_count": 3, "metadata": {}, "output_type": "execute_result" } ], "source": [ "type(folium_map.folium_map)" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "## Adding IP Entities to the map\n", "\n", "```\n", "fol_map.add_ip_cluster(\n", " ip_entities: Iterable[msticpy.nbtools.entityschema.IpAddress],\n", " **kwargs,\n", ")\n", "\n", "Add a collection of IP Entities to the map.\n", "\n", "Parameters\n", "----------\n", "ip_entities : Iterable[IpAddress]\n", " a iterable of IpAddress Entities\n", "\n", "Other Parameters\n", "----------------\n", " kwargs: icon properties to use for displaying this cluster\n", "```" ] }, { "cell_type": "code", "execution_count": 4, "metadata": { "ExecuteTime": { "end_time": "2020-02-08T01:01:41.464131Z", "start_time": "2020-02-08T01:01:41.442143Z" } }, "outputs": [ { "data": { "text/html": [ "
Make this Notebook Trusted to load map: File -> Trust Notebook
" ], "text/plain": [ "" ] }, "execution_count": 4, "metadata": {}, "output_type": "execute_result" } ], "source": [ "import pickle\n", "with open(b\"data/ip_entities.pkl\", \"rb\") as fh:\n", " ip_entities = pickle.load(fh)\n", "ip_entities = [ip for ip in ip_entities if ip.Location and ip.Location.Latitude]\n", "\n", "folium_map = FoliumMap(zoom_start=9)\n", "folium_map.add_ip_cluster(ip_entities=ip_entities, color='orange')\n", "folium_map.center_map()\n", "folium_map" ] }, { "cell_type": "code", "execution_count": 13, "metadata": { "ExecuteTime": { "end_time": "2020-02-08T01:01:41.488150Z", "start_time": "2020-02-08T01:01:41.465129Z" } }, "outputs": [ { "data": { "text/html": [ "
\n", "\n", "\n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", "
AllExtIPsAdditionalDataTypeCountryCodeCountryNameStateCityLongitudeLatitudeIpAddress
065.55.44.109{}geolocationUSUnited StatesVirginiaBoydton-78.375036.653465.55.44.109
113.71.172.128{}geolocationCACanadaOntarioToronto-79.419543.664413.71.172.128
213.71.172.130{}geolocationCACanadaOntarioToronto-79.419543.664413.71.172.130
340.124.45.19{}geolocationUSUnited StatesTexasSan Antonio-98.493529.424740.124.45.19
4104.43.212.12{}geolocationUSUnited StatesIowaDes Moines-93.611241.6006104.43.212.12
\n", "
" ], "text/plain": [ " AllExtIPs AdditionalData Type CountryCode CountryName \\\n", "0 65.55.44.109 {} geolocation US United States \n", "1 13.71.172.128 {} geolocation CA Canada \n", "2 13.71.172.130 {} geolocation CA Canada \n", "3 40.124.45.19 {} geolocation US United States \n", "4 104.43.212.12 {} geolocation US United States \n", "\n", " State City Longitude Latitude IpAddress \n", "0 Virginia Boydton -78.3750 36.6534 65.55.44.109 \n", "1 Ontario Toronto -79.4195 43.6644 13.71.172.128 \n", "2 Ontario Toronto -79.4195 43.6644 13.71.172.130 \n", "3 Texas San Antonio -98.4935 29.4247 40.124.45.19 \n", "4 Iowa Des Moines -93.6112 41.6006 104.43.212.12 " ] }, "execution_count": 13, "metadata": {}, "output_type": "execute_result" } ], "source": [ "# Read in some data\n", "geo_loc_df = pd.read_csv(\"data/ip_locs.csv\", index_col=0)\n", "geo_loc_df.head()" ] }, { "cell_type": "code", "execution_count": 14, "metadata": { "ExecuteTime": { "end_time": "2020-02-08T01:01:41.548161Z", "start_time": "2020-02-08T01:01:41.489152Z" } }, "outputs": [ { "data": { "text/plain": [ "[IpAddress(Address=65.55.44.109, Location={ 'AdditionalData': {},\n", " 'City': 'Boydton',\n", " 'C...),\n", " IpAddress(Address=13.71.172.128, Location={ 'AdditionalData': {},\n", " 'City': 'Toronto',\n", " '...),\n", " IpAddress(Address=13.71.172.130, Location={ 'AdditionalData': {},\n", " 'City': 'Toronto',\n", " '...),\n", " IpAddress(Address=40.124.45.19, Location={ 'AdditionalData': {},\n", " 'City': 'San Antonio',\n", " ...),\n", " IpAddress(Address=104.43.212.12, Location={ 'AdditionalData': {},\n", " 'City': 'Des Moines',\n", " ...)]" ] }, "execution_count": 14, "metadata": {}, "output_type": "execute_result" } ], "source": [ "# Create IP and GeoLocation Entities from the dataframe\n", "def create_ip_entity(row):\n", " ip_ent = IpAddress(Address=row[\"AllExtIPs\"])\n", " geo_loc = create_geo_entity(row)\n", " ip_ent.Location = geo_loc\n", " return ip_ent\n", "\n", "def create_geo_entity(row):\n", " # get subset of fields for GeoLocation\n", " loc_props = row[[\"CountryCode\", \"CountryName\",\"State\", \"City\", \"Longitude\", \"Latitude\"]]\n", " geo_loc = GeoLocation(**loc_props.to_dict())\n", " return geo_loc\n", "\n", "geo_locs = list(geo_loc_df.apply(create_geo_entity, axis=1).values)\n", "ip_ents = list(geo_loc_df.apply(create_ip_entity, axis=1).values)\n", "ip_ents[:5]" ] }, { "cell_type": "code", "execution_count": 15, "metadata": { "ExecuteTime": { "end_time": "2020-02-08T01:26:09.269390Z", "start_time": "2020-02-08T01:26:09.261435Z" } }, "outputs": [ { "data": { "text/plain": [ "array([(36.6534, -78.375), (43.6644, -79.4195), (43.6644, -79.4195),\n", " (29.4247, -98.4935), (41.6006, -93.6112), (41.1399, -104.8193),\n", " (36.6534, -78.375), (43.6644, -79.4195), (38.7095, -78.1539),\n", " (38.7095, -78.1539), (36.6534, -78.375), (47.6742, -122.1243),\n", " (38.7095, -78.1539), (53.3338, -6.2488), (29.4247, -98.4935),\n", " (41.6006, -93.6112), (41.6006, -93.6112), (41.6006, -93.6112),\n", " (38.7095, -78.1539), (36.6534, -78.375), (36.6534, -78.375),\n", " (37.3388, -121.8914), (37.3388, -121.8914), (38.7095, -78.1539),\n", " (38.7095, -78.1539), (38.7095, -78.1539), (38.7095, -78.1539),\n", " (38.7095, -78.1539), (38.7095, -78.1539), (38.7095, -78.1539),\n", " (53.3338, -6.2488), (41.6006, -93.6112), (38.7095, -78.1539),\n", " (38.7095, -78.1539), (38.7095, -78.1539), (36.6534, -78.375),\n", " (38.7095, -78.1539), (41.6006, -93.6112), (41.6006, -93.6112),\n", " (41.6006, -93.6112), (38.7095, -78.1539), (37.3388, -121.8914),\n", " (37.3388, -121.8914), (38.7095, -78.1539), (41.6006, -93.6112),\n", " (40.4953, -111.9439), (41.6006, -93.6112), (41.6006, -93.6112),\n", " (36.6534, -78.375), (38.7095, -78.1539), (38.7095, -78.1539)],\n", " dtype=object)" ] }, "execution_count": 15, "metadata": {}, "output_type": "execute_result" } ], "source": [ "geo_loc_df.apply(lambda x: (x.Latitude, x.Longitude), axis=1).values" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "### Plot IPAddress entities with location data" ] }, { "cell_type": "code", "execution_count": 16, "metadata": { "ExecuteTime": { "end_time": "2020-02-08T01:01:41.607128Z", "start_time": "2020-02-08T01:01:41.550160Z" }, "scrolled": true }, "outputs": [ { "data": { "text/html": [ "
Make this Notebook Trusted to load map: File -> Trust Notebook
" ], "text/plain": [ "" ] }, "execution_count": 16, "metadata": {}, "output_type": "execute_result" } ], "source": [ "fmap_ips = FoliumMap()\n", "fmap_ips.add_ip_cluster(ip_entities=ip_ents[:20], color='blue')\n", "fmap_ips.center_map()\n", "fmap_ips" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "### Use different colors and icons" ] }, { "cell_type": "code", "execution_count": 17, "metadata": { "ExecuteTime": { "end_time": "2020-02-08T01:01:41.677086Z", "start_time": "2020-02-08T01:01:41.609124Z" }, "scrolled": true }, "outputs": [ { "data": { "text/html": [ "
Make this Notebook Trusted to load map: File -> Trust Notebook
" ], "text/plain": [ "" ] }, "execution_count": 17, "metadata": {}, "output_type": "execute_result" } ], "source": [ "fmap_ips.add_ip_cluster(ip_entities=ip_ents[30:40], color='red', icon=\"flash\")\n", "fmap_ips.center_map()\n", "fmap_ips" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "## Custom Icons\n", "\n", "By default folium uses the information icon (i).\n", "Icons can be taken from the default Bootstrap set. See the default list here [glyphicons](https://www.w3schools.com/icons/bootstrap_icons_glyphicons.asp)\n", "\n", "Alternatively you can use icons from the [Font Awesome collection](https://fontawesome.com/icons?d=gallery)\n", "by adding prefx=\"fa\" and icon=\"icon_name\" to the call to add_ip_cluster or add_geo_cluster." ] }, { "cell_type": "code", "execution_count": 18, "metadata": { "ExecuteTime": { "end_time": "2020-02-08T01:01:41.806012Z", "start_time": "2020-02-08T01:01:41.678087Z" } }, "outputs": [ { "data": { "text/html": [ "
Make this Notebook Trusted to load map: File -> Trust Notebook
" ], "text/plain": [ "" ] }, "execution_count": 18, "metadata": {}, "output_type": "execute_result" } ], "source": [ "fmap_ips.add_geoloc_cluster(geo_locations=geo_locs[40:50], color='darkblue', icon=\"desktop\", prefix=\"fa\")\n", "fmap_ips.center_map()\n", "fmap_ips" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "## Utility Functions" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "### Calculate center point of entity locations" ] }, { "cell_type": "code", "execution_count": 19, "metadata": { "ExecuteTime": { "end_time": "2020-02-08T01:01:41.815007Z", "start_time": "2020-02-08T01:01:41.807012Z" } }, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "(38.7095, -78.375)\n", "(39.70266078431372, -85.0484019607843)\n", "(40.15505, -78.26445)\n", "(39.9247, -79.4195)\n", "(39.9247, -79.4195)\n" ] } ], "source": [ "from msticpy.nbtools.foliummap import get_map_center, get_center_ip_entities, get_center_geo_locs\n", "\n", "print(get_center_geo_locs(geo_locs))\n", "print(get_center_geo_locs(geo_locs, mode=\"mean\"))\n", "\n", "# get_map_center Will accept iterable of any entity type that is either\n", "# an IpAddress entity or an entity that has properties of type IpAddress\n", "print(get_map_center(ip_ents[30:40]))\n", "print(get_map_center(ip_ents[:20]))\n", "\n", "print(get_center_ip_entities(ip_ents[:20])) " ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "### Calculate distance between entity locations" ] }, { "cell_type": "code", "execution_count": 27, "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "Distance between\n", "{ 'AdditionalData': {},\n", " 'Address': '65.55.44.109',\n", " 'Location': { 'AdditionalData': {},\n", " 'City': 'Boydton',\n", " 'CountryCode': 'US',\n", " 'CountryName': 'United States',\n", " 'Latitude': 36.6534,\n", " 'Longitude': -78.375,\n", " 'State': 'Virginia',\n", " 'Type': 'geolocation',\n", " 'edges': set()},\n", " 'ThreatIntelligence': [],\n", " 'Type': 'ipaddress',\n", " 'edges': set()} { 'AdditionalData': {},\n", " 'Address': '13.71.172.128',\n", " 'Location': { 'AdditionalData': {},\n", " 'City': 'Toronto',\n", " 'CountryCode': 'CA',\n", " 'CountryName': 'Canada',\n", " 'Latitude': 43.6644,\n", " 'Longitude': -79.4195,\n", " 'State': 'Ontario',\n", " 'Type': 'geolocation',\n", " 'edges': set()},\n", " 'ThreatIntelligence': [],\n", " 'Type': 'ipaddress',\n", " 'edges': set()}\n", "\n", " 784.604908273247 km\n", "Distance between\n", "{ 'AdditionalData': {},\n", " 'Address': '65.55.44.109',\n", " 'Location': { 'AdditionalData': {},\n", " 'City': 'Boydton',\n", " 'CountryCode': 'US',\n", " 'CountryName': 'United States',\n", " 'Latitude': 36.6534,\n", " 'Longitude': -78.375,\n", " 'State': 'Virginia',\n", " 'Type': 'geolocation',\n", " 'edges': set()},\n", " 'ThreatIntelligence': [],\n", " 'Type': 'ipaddress',\n", " 'edges': set()} { 'AdditionalData': {},\n", " 'Address': '40.77.226.250',\n", " 'Location': { 'AdditionalData': {},\n", " 'City': 'Dublin',\n", " 'CountryCode': 'IE',\n", " 'CountryName': 'Ireland',\n", " 'Latitude': 53.3338,\n", " 'Longitude': -6.2488,\n", " 'State': 'Leinster',\n", " 'Type': 'geolocation',\n", " 'edges': set()},\n", " 'ThreatIntelligence': [],\n", " 'Type': 'ipaddress',\n", " 'edges': set()}\n", "\n", " 5699.044784950343 km\n" ] } ], "source": [ "from msticpy.sectools.geoip import entity_distance\n", "print(\"Distance between\")\n", "print(ip_ents[0], ip_ents[1])\n", "print(\"\\n\", entity_distance(ip_ents[0], ip_ents[1]), \"km\")\n", "\n", "print(\"Distance between\")\n", "print(ip_ents[0], ip_ents[13])\n", "print(\"\\n\", entity_distance(ip_ents[0], ip_ents[13]), \"km\")" ] } ], "metadata": { "hide_input": false, "kernelspec": { "display_name": "Python (condadev)", "language": "python", "name": "condadev" }, "language_info": { "codemirror_mode": { "name": "ipython", "version": 3 }, "file_extension": ".py", "mimetype": "text/x-python", "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", "version": "3.7.9" }, "toc": { "base_numbering": 1, "nav_menu": {}, "number_sections": false, "sideBar": true, "skip_h1_title": true, "title_cell": "Table of Contents", "title_sidebar": "Contents", "toc_cell": true, "toc_position": {}, "toc_section_display": true, "toc_window_display": true }, "varInspector": { "cols": { "lenName": 16, "lenType": 16, "lenVar": 40 }, "kernels_config": { "python": { "delete_cmd_postfix": "", "delete_cmd_prefix": "del ", "library": "var_list.py", "varRefreshCmd": "print(var_dic_list())" }, "r": { "delete_cmd_postfix": ") ", "delete_cmd_prefix": "rm(", "library": "var_list.r", "varRefreshCmd": "cat(var_dic_list()) " } }, "types_to_exclude": [ "module", "function", "builtin_function_or_method", "instance", "_Feature" ], "window_display": false }, "widgets": { "application/vnd.jupyter.widget-state+json": { "state": {}, "version_major": 2, "version_minor": 0 } } }, "nbformat": 4, "nbformat_minor": 4 }