{ "cells": [ { "cell_type": "markdown", "metadata": {}, "source": [ "# Title: msticpy - IoC Extraction\n", "## Description:\n", "This class allows you to extract IoC patterns from a string or a DataFrame.\n", "Several patterns are built in to the class and you can override these or supply new ones.\n" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "\n", "## Table of Contents\n", "- [Looking for IoC in a String](#cmdlineiocs)\n", "- [Search DataFrame for IoCs](#dataframeiocs)\n", "- [IoCExtractor API](#iocextractapi)\n", " - [Predefined Regex Patterns](#regexpatterns)\n", " - [Adding your own pattern(s)](#addingpatterns)\n", " - [extract() method](#extractmethod)\n", " - [Merge the results with the input DataFrame](#mergeresults)" ] }, { "cell_type": "code", "execution_count": 2, "metadata": { "scrolled": true }, "outputs": [], "source": [ "# Imports\n", "import sys\n", "MIN_REQ_PYTHON = (3,6)\n", "if sys.version_info < MIN_REQ_PYTHON:\n", " print('Check the Kernel->Change Kernel menu and ensure that Python 3.6')\n", " print('or later is selected as the active kernel.')\n", " sys.exit(\"Python %s.%s or later is required.\\n\" % MIN_REQ_PYTHON)\n", "\n", "\n", "import numpy as np\n", "from IPython import get_ipython\n", "from IPython.display import display, HTML\n", "import ipywidgets as widgets\n", "\n", "import matplotlib.pyplot as plt\n", "import seaborn as sns\n", "import networkx as nx\n", "sns.set()\n", "import pandas as pd\n", "pd.set_option('display.max_rows', 500)\n", "pd.set_option('display.max_columns', 50)\n", "pd.set_option('display.max_colwidth', 100)\n", "\n", "import os\n", "module_path = os.path.abspath(os.path.join('../..'))\n", "if module_path not in sys.path:\n", " sys.path.append(module_path)\n", " \n", "import msticpy.sectools as sectools\n", "import msticpy.nbtools as asi\n", "import msticpy.nbtools.kql as qry\n", "import msticpy.nbtools.nbdisplay as nbdisp\n" ] }, { "cell_type": "code", "execution_count": 56, "metadata": {}, "outputs": [ { "data": { "text/html": [ "
\n", "\n", "\n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", "
CommandLine
0.\\ftp -s:C:\\RECYCLER\\xxppyy.exe
1.\\reg not /domain:everything that /sid:shines is /krbtgt:golden !
2cmd /c \"systeminfo && systeminfo\"
3.\\rundll32 /C 12345.exe
4.\\rundll32 /C c:\\users\\MSTICAdmin\\12345.exe
\n", "
" ], "text/plain": [ " CommandLine\n", "0 .\\ftp -s:C:\\RECYCLER\\xxppyy.exe\n", "1 .\\reg not /domain:everything that /sid:shines is /krbtgt:golden !\n", "2 cmd /c \"systeminfo && systeminfo\"\n", "3 .\\rundll32 /C 12345.exe\n", "4 .\\rundll32 /C c:\\users\\MSTICAdmin\\12345.exe" ] }, "execution_count": 56, "metadata": {}, "output_type": "execute_result" } ], "source": [ "# Load test data\n", "process_tree = pd.read_csv('data/process_tree.csv')\n", "process_tree[['CommandLine']].head()" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "[Contents](#toc)\n", "## Looking for IoC in a String\n", "Here we:\n", "- Get a commandline from our data set.\n", "- Pass it to the IoC Extractor\n", "- View the results" ] }, { "cell_type": "code", "execution_count": 8, "metadata": {}, "outputs": [ { "data": { "text/plain": [ "'netsh start capture=yes IPv4.Address=1.2.3.4 tracefile=C:\\\\\\\\Users\\\\\\\\user\\\\\\\\AppData\\\\\\\\Local\\\\\\\\Temp\\\\\\\\bzzzzzz.txt'" ] }, "execution_count": 8, "metadata": {}, "output_type": "execute_result" } ], "source": [ "# get a commandline from our data set\n", "cmdline = process_tree['CommandLine'].loc[78]\n", "cmdline" ] }, { "cell_type": "code", "execution_count": 13, "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "\n", "Potential IoCs found in alert process:\n" ] }, { "data": { "text/plain": [ "defaultdict(set,\n", " {'ipv4': {'1.2.3.4'},\n", " 'windows_path': {'C:\\\\\\\\Users\\\\\\\\user\\\\\\\\AppData\\\\\\\\Local\\\\\\\\Temp\\\\\\\\bzzzzzz.txt'}})" ] }, "metadata": {}, "output_type": "display_data" } ], "source": [ "# Instantiate an IoCExtract object\n", "from msticpy.sectools import IoCExtract\n", "ioc_extractor = IoCExtract()\n", "\n", "# any IoCs in the string?\n", "iocs_found = ioc_extractor.extract(cmdline)\n", " \n", "if iocs_found:\n", " print('\\nPotential IoCs found in alert process:')\n", " display(iocs_found)\n" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "[Contents](#toc)\n", "## If we have a DataFrame, look for IoCs in the whole data set\n", "You can replace the ```data=``` parameter to ioc_extractor.extract() to pass other data frames.\n", "Use the ```columns``` parameter to specify which column or columns that you want to search." ] }, { "cell_type": "code", "execution_count": 10, "metadata": {}, "outputs": [ { "data": { "text/html": [ "

IoC patterns found in process tree.

" ], "text/plain": [ "" ] }, "metadata": {}, "output_type": "display_data" }, { "data": { "text/html": [ "
\n", "\n", "\n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", "
IoCTypeObservableSourceIndex
0windows_pathC:\\RECYCLER\\xxppyy.exe0
1windows_path.\\ftp0
2windows_path.\\reg1
3windows_path.\\rundll323
4windows_pathc:\\users\\MSTICAdmin\\12345.exe4
5windows_path.\\rundll324
6windows_path.\\rundll325
7windows_pathc:\\users\\MSTICAdmin\\1234.exe6
8windows_path.\\rundll326
9windows_path.\\rundll327
10windows_path.\\reg.exe add \\hkcu\\software\\microsoft\\some\\key\\Run8
11dnstsetup.1.0.14.exe9
12dnstsetup.1.exe9
13dnstsetup.1.0.14.tmp9
14windows_pathc:\\Diagnostics\\UserTmp\\tsetup.1.exe9
15windows_pathC:\\Users\\MSTICAdmin\\Downloads\\tsetup.1.0.14.exe9
16windows_pathC:\\Users\\MSTICAdmin\\AppData\\Local\\Temp\\2\\is-01DD7.tmp\\tsetup.1.0.14.tmp9
17windows_path.\\rundll32.exe10
18windows_path.\\netsh.exe11
19windows_path.\\cmd12
20windows_pathC:\\inetpub\\wwwroot12
21windows_path.\\cmd13
22windows_pathC:\\inetpub\\wwwroot13
23windows_path.\\cmd14
24windows_pathC:\\inetpub\\wwwroot14
25windows_path.\\cmd15
26windows_path\\\\[REDACTED]\\c$\\users\\[REDACTED]\\Documents15
27windows_pathC:\\ProgramData15
28windows_path.\\cmd16
29windows_pathC:\\inetpub\\wwwroot16
30windows_pathc:\\windows\\system32\\inetsrv\\appcmd16
31windows_path.\\cmd17
32windows_pathC:\\inetpub\\wwwroot17
33windows_pathC:\\inetpub\\logs\\logFiles\\W3SVC117
34windows_pathc:\\Diagnostics\\UserTmp\\perfc.dat18
35windows_pathc:\\Diagnostics\\UserTmp\\sdopfjiowtbkjfnbeioruj.exe19
36dnsdoubleextension.pdf.exe20
37windows_pathc:\\Diagnostics\\UserTmp\\doubleextension.pdf.exe20
38windows_path\\Windows\\system32\\conhost.exe22
39windows_path\\C:22
40windows_pathc:\\testshare26
41windows_path\\\\MSTICAlertsWin1\\TestShare27
42urlhttp://server/file.sct31
43dnsserver31
44windows_path.\\regsvr3231
45windows_path.\\suchost.exe32
46windows_path.\\evil.ps1;35
47windows_path.\\powershell.exe35
48windows_path.\\powershell36
49urlhttp://somedomain/best-kitten-names-1.jpg'37
50dnssomedomain37
51windows_path\\AppData\\Local\\Temp\\kittens1.jpg';37
52windows_pathC:\\Users\\$env:UserName37
53windows_path.\\pOWErS^H^ElL^.eX^e^37
54windows_path.\\n^e^t38
55windows_path.\\powershell39
56md5_hashaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa40
57md5_hashaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa41
58md5_hash81ed03caf6901e444c72ac67d192fb9c44
59urlhttp://badguyserver/pwnme\"46
60dnsbadguyserver46
61urlhttp://badguyserver/pwnme\"47
62dnsbadguyserver47
63windows_path.\\powershell47
64windows_path.\\powershell48
65windows_path.\\powershell49
66windows_path.\\powershell50
67windows_path.\\rUnDlL3258
68windows_path.\\reg query add mscfile\\\\\\\\open59
69windows_path.\\reg60
70windows_path.\\dubrute.exe61
71windows_path.\\nlbrute.exe62
72windows_path\\system\\CurrentControlSet\\Control\\Terminal63
73windows_path.\\reg63
74windows_path\\system\\CurrentControlSet\\Control\\Terminal64
75windows_path.\\reg64
76windows_path\\\\tsclient\\c65
77windows_path\\Microsoft\\Windows\\CurrentVersion Certificate).Certificate);.\\powershell67
78windows_pathC:\\Windows\\System32\\mshta.exe67
79windows_path.\\powershell.exe67
80windows_path.\\netsh68
81windows_pathc:\\users\\Bob\\appdata\\Roaming\\RbtGskQ\\RbtGskQ.exe68
82windows_path.\\reg add HKLM\\KEY_LOCAL_MACHINE\\...securityproviders\\wdigest69
83windows_pathc:\\Windows\\System32\\cmd.exe70
84windows_pathc:\\Diagnostics\\UserTmp\\scrsave.scr71
85windows_pathc:\\Diagnostics\\UserTmp\\svchost.exe72
86windows_pathc:\\Diagnostics\\UserTmp\\smss.exe73
87windows_pathc:\\Windows\\System32\\svchost.exe74
88dnssystem.management.automation.amsiutils77
89dnssystem.management.automation.amsiutils').getfield('amsiinitfailed','nonpublic,static').setvalue(...77
90urlhttp://system.management.automation.amsiutils').getfield('amsiinitfailed','nonpublic,static').se...77
91windows_path.\\powershell.exe77
92ipv41.2.3.478
93windows_pathC:\\\\Users\\\\user\\\\AppData\\\\Local\\\\Temp\\\\bzzzzzz.txt78
94windows_path.\\wuauclt.exe79
95windows_pathc:\\windows\\softwaredistribution\\cscript.exe79
96windows_pathc:\\windows\\softwaredistribution\\cscript.exe80
97windows_path.\\lsass.exe80
98windows_pathc:\\windows\\system32\\wscript.exe82
99windows_pathC:\\inetpub\\wwwroot83
100windows_pathc:\\windows\\system32\\inetsrv\\appcmd83
101windows_pathc:\\Diagnostics\\UserTmp\\2840.exe84
102windows_pathc:\\Diagnostics\\UserTmp\\a_keygen.exe85
103windows_pathc:\\Diagnostics\\UserTmp\\bittorrent.exe87
104windows_pathc:\\Diagnostics\\UserTmp\\netsh.exe88
105windows_pathc:\\Diagnostics\\UserTmp\\ransomware.exe90
106windows_path\\\\server\\payload.dll92
107windows_pathC:\\Users\\Administrator\\AppData\\Roaming\\{RANDOM}.txt94
108ipv4127.0.0.1102
109urlhttp://127.0.0.1/102
110windows_path\\SOFTWARE\\Microsoft\\Windows NT\\CurrentVersion\\Svchost\\MyNastySvcHostConfig103
111windows_path.\\reg103
112windows_path\\SOFTWARE\\Microsoft\\Windows NT\\CurrentVersion\\Svchost\\MyNastySvcHostConfig104
113windows_path.\\reg104
114windows_pathC:\\Users\\MSTICA~1\\AppData\\Local\\Temp\\hd.exe105
115windows_path\\\\.\\pipe\\blahtest107
116windows_path.\\reg.exe108
117windows_path\\console108
118windows_pathc:\\windows\\fonts\\csrss.exe109
119windows_pathc:\\windows\\fonts\\conhost.exe110
120windows_path.\\mimikatz.exe111
121windows_path.\\rundll32.exe112
122windows_pathc:\\windows\\fonts\\conhost.exe112
123windows_path.\\regsvr32113
124windows_pathc:\\windows\\fonts\\csrss.exe113
125windows_pathc:\\Diagnostics\\UserTmp115
126windows_pathc:\\Diagnostics\\WindowsSimulateDetections.bat115
127windows_pathC:\\Windows\\System32\\win32calc.exe116
\n", "
" ], "text/plain": [ " IoCType \\\n", "0 windows_path \n", "1 windows_path \n", "2 windows_path \n", "3 windows_path \n", "4 windows_path \n", "5 windows_path \n", "6 windows_path \n", "7 windows_path \n", "8 windows_path \n", "9 windows_path \n", "10 windows_path \n", "11 dns \n", "12 dns \n", "13 dns \n", "14 windows_path \n", "15 windows_path \n", "16 windows_path \n", "17 windows_path \n", "18 windows_path \n", "19 windows_path \n", "20 windows_path \n", "21 windows_path \n", "22 windows_path \n", "23 windows_path \n", "24 windows_path \n", "25 windows_path \n", "26 windows_path \n", "27 windows_path \n", "28 windows_path \n", "29 windows_path \n", "30 windows_path \n", "31 windows_path \n", "32 windows_path \n", "33 windows_path \n", "34 windows_path \n", "35 windows_path \n", "36 dns \n", "37 windows_path \n", "38 windows_path \n", "39 windows_path \n", "40 windows_path \n", "41 windows_path \n", "42 url \n", "43 dns \n", "44 windows_path \n", "45 windows_path \n", "46 windows_path \n", "47 windows_path \n", "48 windows_path \n", "49 url \n", "50 dns \n", "51 windows_path \n", "52 windows_path \n", "53 windows_path \n", "54 windows_path \n", "55 windows_path \n", "56 md5_hash \n", "57 md5_hash \n", "58 md5_hash \n", "59 url \n", "60 dns \n", "61 url \n", "62 dns \n", "63 windows_path \n", "64 windows_path \n", "65 windows_path \n", "66 windows_path \n", "67 windows_path \n", "68 windows_path \n", "69 windows_path \n", "70 windows_path \n", "71 windows_path \n", "72 windows_path \n", "73 windows_path \n", "74 windows_path \n", "75 windows_path \n", "76 windows_path \n", "77 windows_path \n", "78 windows_path \n", "79 windows_path \n", "80 windows_path \n", "81 windows_path \n", "82 windows_path \n", "83 windows_path \n", "84 windows_path \n", "85 windows_path \n", "86 windows_path \n", "87 windows_path \n", "88 dns \n", "89 dns \n", "90 url \n", "91 windows_path \n", "92 ipv4 \n", "93 windows_path \n", "94 windows_path \n", "95 windows_path \n", "96 windows_path \n", "97 windows_path \n", "98 windows_path \n", "99 windows_path \n", "100 windows_path \n", "101 windows_path \n", "102 windows_path \n", "103 windows_path \n", "104 windows_path \n", "105 windows_path \n", "106 windows_path \n", "107 windows_path \n", "108 ipv4 \n", "109 url \n", "110 windows_path \n", "111 windows_path \n", "112 windows_path \n", "113 windows_path \n", "114 windows_path \n", "115 windows_path \n", "116 windows_path \n", "117 windows_path \n", "118 windows_path \n", "119 windows_path \n", "120 windows_path \n", "121 windows_path \n", "122 windows_path \n", "123 windows_path \n", "124 windows_path \n", "125 windows_path \n", "126 windows_path \n", "127 windows_path \n", "\n", " Observable \\\n", "0 C:\\RECYCLER\\xxppyy.exe \n", "1 .\\ftp \n", "2 .\\reg \n", "3 .\\rundll32 \n", "4 c:\\users\\MSTICAdmin\\12345.exe \n", "5 .\\rundll32 \n", "6 .\\rundll32 \n", "7 c:\\users\\MSTICAdmin\\1234.exe \n", "8 .\\rundll32 \n", "9 .\\rundll32 \n", "10 .\\reg.exe add \\hkcu\\software\\microsoft\\some\\key\\Run \n", "11 tsetup.1.0.14.exe \n", "12 tsetup.1.exe \n", "13 tsetup.1.0.14.tmp \n", "14 c:\\Diagnostics\\UserTmp\\tsetup.1.exe \n", "15 C:\\Users\\MSTICAdmin\\Downloads\\tsetup.1.0.14.exe \n", "16 C:\\Users\\MSTICAdmin\\AppData\\Local\\Temp\\2\\is-01DD7.tmp\\tsetup.1.0.14.tmp \n", "17 .\\rundll32.exe \n", "18 .\\netsh.exe \n", "19 .\\cmd \n", "20 C:\\inetpub\\wwwroot \n", "21 .\\cmd \n", "22 C:\\inetpub\\wwwroot \n", "23 .\\cmd \n", "24 C:\\inetpub\\wwwroot \n", "25 .\\cmd \n", "26 \\\\[REDACTED]\\c$\\users\\[REDACTED]\\Documents \n", "27 C:\\ProgramData \n", "28 .\\cmd \n", "29 C:\\inetpub\\wwwroot \n", "30 c:\\windows\\system32\\inetsrv\\appcmd \n", "31 .\\cmd \n", "32 C:\\inetpub\\wwwroot \n", "33 C:\\inetpub\\logs\\logFiles\\W3SVC1 \n", "34 c:\\Diagnostics\\UserTmp\\perfc.dat \n", "35 c:\\Diagnostics\\UserTmp\\sdopfjiowtbkjfnbeioruj.exe \n", "36 doubleextension.pdf.exe \n", "37 c:\\Diagnostics\\UserTmp\\doubleextension.pdf.exe \n", "38 \\Windows\\system32\\conhost.exe \n", "39 \\C: \n", "40 c:\\testshare \n", "41 \\\\MSTICAlertsWin1\\TestShare \n", "42 http://server/file.sct \n", "43 server \n", "44 .\\regsvr32 \n", "45 .\\suchost.exe \n", "46 .\\evil.ps1; \n", "47 .\\powershell.exe \n", "48 .\\powershell \n", "49 http://somedomain/best-kitten-names-1.jpg' \n", "50 somedomain \n", "51 \\AppData\\Local\\Temp\\kittens1.jpg'; \n", "52 C:\\Users\\$env:UserName \n", "53 .\\pOWErS^H^ElL^.eX^e^ \n", "54 .\\n^e^t \n", "55 .\\powershell \n", "56 aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa \n", "57 aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa \n", "58 81ed03caf6901e444c72ac67d192fb9c \n", "59 http://badguyserver/pwnme\" \n", "60 badguyserver \n", "61 http://badguyserver/pwnme\" \n", "62 badguyserver \n", "63 .\\powershell \n", "64 .\\powershell \n", "65 .\\powershell \n", "66 .\\powershell \n", "67 .\\rUnDlL32 \n", "68 .\\reg query add mscfile\\\\\\\\open \n", "69 .\\reg \n", "70 .\\dubrute.exe \n", "71 .\\nlbrute.exe \n", "72 \\system\\CurrentControlSet\\Control\\Terminal \n", "73 .\\reg \n", "74 \\system\\CurrentControlSet\\Control\\Terminal \n", "75 .\\reg \n", "76 \\\\tsclient\\c \n", "77 \\Microsoft\\Windows\\CurrentVersion Certificate).Certificate);.\\powershell \n", "78 C:\\Windows\\System32\\mshta.exe \n", "79 .\\powershell.exe \n", "80 .\\netsh \n", "81 c:\\users\\Bob\\appdata\\Roaming\\RbtGskQ\\RbtGskQ.exe \n", "82 .\\reg add HKLM\\KEY_LOCAL_MACHINE\\...securityproviders\\wdigest \n", "83 c:\\Windows\\System32\\cmd.exe \n", "84 c:\\Diagnostics\\UserTmp\\scrsave.scr \n", "85 c:\\Diagnostics\\UserTmp\\svchost.exe \n", "86 c:\\Diagnostics\\UserTmp\\smss.exe \n", "87 c:\\Windows\\System32\\svchost.exe \n", "88 system.management.automation.amsiutils \n", "89 system.management.automation.amsiutils').getfield('amsiinitfailed','nonpublic,static').setvalue(... \n", "90 http://system.management.automation.amsiutils').getfield('amsiinitfailed','nonpublic,static').se... \n", "91 .\\powershell.exe \n", "92 1.2.3.4 \n", "93 C:\\\\Users\\\\user\\\\AppData\\\\Local\\\\Temp\\\\bzzzzzz.txt \n", "94 .\\wuauclt.exe \n", "95 c:\\windows\\softwaredistribution\\cscript.exe \n", "96 c:\\windows\\softwaredistribution\\cscript.exe \n", "97 .\\lsass.exe \n", "98 c:\\windows\\system32\\wscript.exe \n", "99 C:\\inetpub\\wwwroot \n", "100 c:\\windows\\system32\\inetsrv\\appcmd \n", "101 c:\\Diagnostics\\UserTmp\\2840.exe \n", "102 c:\\Diagnostics\\UserTmp\\a_keygen.exe \n", "103 c:\\Diagnostics\\UserTmp\\bittorrent.exe \n", "104 c:\\Diagnostics\\UserTmp\\netsh.exe \n", "105 c:\\Diagnostics\\UserTmp\\ransomware.exe \n", "106 \\\\server\\payload.dll \n", "107 C:\\Users\\Administrator\\AppData\\Roaming\\{RANDOM}.txt \n", "108 127.0.0.1 \n", "109 http://127.0.0.1/ \n", "110 \\SOFTWARE\\Microsoft\\Windows NT\\CurrentVersion\\Svchost\\MyNastySvcHostConfig \n", "111 .\\reg \n", "112 \\SOFTWARE\\Microsoft\\Windows NT\\CurrentVersion\\Svchost\\MyNastySvcHostConfig \n", "113 .\\reg \n", "114 C:\\Users\\MSTICA~1\\AppData\\Local\\Temp\\hd.exe \n", "115 \\\\.\\pipe\\blahtest \n", "116 .\\reg.exe \n", "117 \\console \n", "118 c:\\windows\\fonts\\csrss.exe \n", "119 c:\\windows\\fonts\\conhost.exe \n", "120 .\\mimikatz.exe \n", "121 .\\rundll32.exe \n", "122 c:\\windows\\fonts\\conhost.exe \n", "123 .\\regsvr32 \n", "124 c:\\windows\\fonts\\csrss.exe \n", "125 c:\\Diagnostics\\UserTmp \n", "126 c:\\Diagnostics\\WindowsSimulateDetections.bat \n", "127 C:\\Windows\\System32\\win32calc.exe \n", "\n", " SourceIndex \n", "0 0 \n", "1 0 \n", "2 1 \n", "3 3 \n", "4 4 \n", "5 4 \n", "6 5 \n", "7 6 \n", "8 6 \n", "9 7 \n", "10 8 \n", "11 9 \n", "12 9 \n", "13 9 \n", "14 9 \n", "15 9 \n", "16 9 \n", "17 10 \n", "18 11 \n", "19 12 \n", "20 12 \n", "21 13 \n", "22 13 \n", "23 14 \n", "24 14 \n", "25 15 \n", "26 15 \n", "27 15 \n", "28 16 \n", "29 16 \n", "30 16 \n", "31 17 \n", "32 17 \n", "33 17 \n", "34 18 \n", "35 19 \n", "36 20 \n", "37 20 \n", "38 22 \n", "39 22 \n", "40 26 \n", "41 27 \n", "42 31 \n", "43 31 \n", "44 31 \n", "45 32 \n", "46 35 \n", "47 35 \n", "48 36 \n", "49 37 \n", "50 37 \n", "51 37 \n", "52 37 \n", "53 37 \n", "54 38 \n", "55 39 \n", "56 40 \n", "57 41 \n", "58 44 \n", "59 46 \n", "60 46 \n", "61 47 \n", "62 47 \n", "63 47 \n", "64 48 \n", "65 49 \n", "66 50 \n", "67 58 \n", "68 59 \n", "69 60 \n", "70 61 \n", "71 62 \n", "72 63 \n", "73 63 \n", "74 64 \n", "75 64 \n", "76 65 \n", "77 67 \n", "78 67 \n", "79 67 \n", "80 68 \n", "81 68 \n", "82 69 \n", "83 70 \n", "84 71 \n", "85 72 \n", "86 73 \n", "87 74 \n", "88 77 \n", "89 77 \n", "90 77 \n", "91 77 \n", "92 78 \n", "93 78 \n", "94 79 \n", "95 79 \n", "96 80 \n", "97 80 \n", "98 82 \n", "99 83 \n", "100 83 \n", "101 84 \n", "102 85 \n", "103 87 \n", "104 88 \n", "105 90 \n", "106 92 \n", "107 94 \n", "108 102 \n", "109 102 \n", "110 103 \n", "111 103 \n", "112 104 \n", "113 104 \n", "114 105 \n", "115 107 \n", "116 108 \n", "117 108 \n", "118 109 \n", "119 110 \n", "120 111 \n", "121 112 \n", "122 112 \n", "123 113 \n", "124 113 \n", "125 115 \n", "126 115 \n", "127 116 " ] }, "metadata": {}, "output_type": "display_data" } ], "source": [ "ioc_extractor = IoCExtract()\n", "ioc_df = ioc_extractor.extract(data=process_tree, columns=['CommandLine'], os_family='Windows')\n", "if len(ioc_df):\n", " display(HTML(\"

IoC patterns found in process tree.

\"))\n", " display(ioc_df)" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "[Contents](#toc)\n", "## IoCExtractor API\n" ] }, { "cell_type": "code", "execution_count": 16, "metadata": {}, "outputs": [], "source": [ "# IoCExtract docstring\n", "IoCExtract?" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "[Contents](#toc)\n", "### Predefined Regex Patterns" ] }, { "cell_type": "code", "execution_count": 29, "metadata": {}, "outputs": [ { "data": { "text/html": [ "ipv4" ], "text/plain": [ "" ] }, "metadata": {}, "output_type": "display_data" }, { "data": { "text/html": [ "
re.compile('(?P(?:[0-9]{1,3}\\\\.){3}[0-9]{1,3})', re.IGNORECASE|re.MULTILINE|re.VERBOSE)
)"
      ],
      "text/plain": [
       ""
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "ipv6"
      ],
      "text/plain": [
       ""
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "
re.compile('(?
)"
      ],
      "text/plain": [
       ""
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "dns"
      ],
      "text/plain": [
       ""
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "
re.compile('((?=[a-z0-9-]{1,63}\\\\.)[a-z0-9]+(-[a-z0-9]+)*\\\\.){2,}[a-z]{2,63}', re.IGNORECASE|re.MULTILINE|re.VERBOSE)
)"
      ],
      "text/plain": [
       ""
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "url"
      ],
      "text/plain": [
       ""
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "
re.compile(\"\\n            (?P(https?|ftp|telnet|ldap|file)://)\\n            (?P([a-z0-9-._~!$&\\\\'()*+,;=:]|%[0-9A-F]{2})*@)?\\n            (?P([a-z0-9-._~!$&\\\\'()*+,;=]|%[0-9A-F]{2})*)\\n, re.IGNORECASE|re.MULTILINE|re.VERBOSE)
)"
      ],
      "text/plain": [
       ""
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "windows_path"
      ],
      "text/plain": [
       ""
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "
re.compile('\\n            (?P[a-z]:|\\\\\\\\\\\\\\\\[a-z0-9_.$-]+||[.]+)\\n            (?P\\\\\\\\(?:[^\\\\/:*?\"\\\\\\'<>|\\\\r\\\\n]+\\\\\\\\)*)\\n            (?P[^\\\\\\\\/*?\"\"<>|\\\\r\\\\n ]+)', re.IGNORECASE|re.MULTILINE|re.VERBOSE)
)"
      ],
      "text/plain": [
       ""
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "linux_path"
      ],
      "text/plain": [
       ""
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "
re.compile('(?P/+||[.]+)\\n            (?P/(?:[^\\\\\\\\/:*?<>|\\\\r\\\\n]+/)*)\\n            (?P[^/\\\\0<>|\\\\r\\\\n ]+)', re.IGNORECASE|re.MULTILINE|re.VERBOSE)
)"
      ],
      "text/plain": [
       ""
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "md5_hash"
      ],
      "text/plain": [
       ""
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "
re.compile('(?:^|[^A-Fa-f0-9])(?P[A-Fa-f0-9]{32})(?:$|[^A-Fa-f0-9])', re.IGNORECASE|re.MULTILINE|re.VERBOSE)
)"
      ],
      "text/plain": [
       ""
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "sha1_hash"
      ],
      "text/plain": [
       ""
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "
re.compile('(?:^|[^A-Fa-f0-9])(?P[A-Fa-f0-9]{40})(?:$|[^A-Fa-f0-9])', re.IGNORECASE|re.MULTILINE|re.VERBOSE)
)"
      ],
      "text/plain": [
       ""
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "sha256_hash"
      ],
      "text/plain": [
       ""
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "
re.compile('(?:^|[^A-Fa-f0-9])(?P[A-Fa-f0-9]{64})(?:$|[^A-Fa-f0-9])', re.IGNORECASE|re.MULTILINE|re.VERBOSE)
)"
      ],
      "text/plain": [
       ""
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "extractor = IoCExtract()\n",
    "\n",
    "for ioc_type, pattern in extractor.ioc_types.items():\n",
    "    display(HTML(f'{ioc_type}'))\n",
    "    display(HTML(f'
{pattern.comp_regex}
)'))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "[Contents](#toc)\n",
    "### Adding your own pattern(s)\n",
    "Docstring:\n",
    "```\n",
    "Add an IoC type and regular expression to use to the built-in set.\n",
    "\n",
    "Note: adding an ioc_type that exists in the internal set will overwrite that item\n",
    "Regular expressions are compiled with re.I | re.X | re.M (Ignore case, Verbose\n",
    "and MultiLine)\n",
    "    :param: ioc_type{str} - a unique name for the IoC type\n",
    "    :param: ioc_regex{str} - a regular expression used to search for the type\n",
    "```"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "metadata": {},
   "outputs": [],
   "source": [
    "import re\n",
    "rcomp = re.compile(r'(?P\\\\\\\\\\.\\\\pipe\\\\[^\\s\\\\]+)')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 39,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "IoCPattern(ioc_type='win_named_pipe', comp_regex=re.compile('(?P\\\\\\\\\\\\\\\\\\\\.\\\\\\\\pipe\\\\\\\\[^\\\\s\\\\\\\\]+)', re.IGNORECASE|re.MULTILINE|re.VERBOSE), priority=0)\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "
\n",
       "\n",
       "\n",
       "  \n",
       "    \n",
       "      \n",
       "      \n",
       "      \n",
       "      \n",
       "    \n",
       "  \n",
       "  \n",
       "    \n",
       "      \n",
       "      \n",
       "      \n",
       "      \n",
       "    \n",
       "  \n",
       "
IoCTypeObservableSourceIndex
116win_named_pipe\\\\.\\pipe\\blahtest\"107
\n",
       "
"
      ],
      "text/plain": [
       "            IoCType          Observable SourceIndex\n",
       "116  win_named_pipe  \\\\.\\pipe\\blahtest\"         107"
      ]
     },
     "execution_count": 39,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "extractor.add_ioc_type(ioc_type='win_named_pipe', ioc_regex=r'(?P\\\\\\\\\\.\\\\pipe\\\\[^\\s\\\\]+)')\n",
    "\n",
    "# Check that it added ok\n",
    "print(extractor.ioc_types['win_named_pipe'])\n",
    "\n",
    "# Use it in our data set\n",
    "ioc_extractor.extract(data=process_tree, columns=['CommandLine'], os_family='Windows').query('IoCType == \\'win_named_pipe\\'')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "[Contents](#toc)\n",
    "### extract() method\n",
    "```\n",
    "Docstring:\n",
    "Extract IoCs from either a string or pandas DataFrame.\n",
    "\n",
    "    :param data: input DataFrame from which to read source strings\n",
    "    :param columns: The list of columns to use as source strings,\n",
    "        if the data parameter is used.\n",
    "    :param src: source string in which to look for IoC patterns\n",
    "    :param os_family: 'Linux' or 'Windows'\n",
    "\n",
    "Returns:\n",
    "    dict of found observables (if input is a string) or\n",
    "    DataFrame of observables\n",
    "\n",
    "Extract takes either a string or a pandas DataFrame as input.\n",
    "When using the string option as an input extract will\n",
    "return a dictionary of results.\n",
    "When using a DataFrame the results will be returned as a new\n",
    "DataFrame with the following columns:\n",
    "- IoCType: the mnemonic used to distinguish different IoC Types\n",
    "- Observable: the actual value of the observable\n",
    "- SourceIndex: the index of the row in the input DataFrame from\n",
    "which the source for the IoC observable was extracted.\n",
    "```\n",
    "\n",
    "**Note** the os_family parameter is optional. If you are not interested in searching for Linux paths omit this or set to 'Windows'. Almost anything is a legal character in a Linux path name so this is a very loose regex (the built-in one is more restrictive than the possible path names, otherwise this will match too much to be useful)."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 42,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "
\n",
       "\n",
       "\n",
       "  \n",
       "    \n",
       "      \n",
       "      \n",
       "      \n",
       "      \n",
       "    \n",
       "  \n",
       "  \n",
       "    \n",
       "      \n",
       "      \n",
       "      \n",
       "      \n",
       "    \n",
       "    \n",
       "      \n",
       "      \n",
       "      \n",
       "      \n",
       "    \n",
       "    \n",
       "      \n",
       "      \n",
       "      \n",
       "      \n",
       "    \n",
       "    \n",
       "      \n",
       "      \n",
       "      \n",
       "      \n",
       "    \n",
       "    \n",
       "      \n",
       "      \n",
       "      \n",
       "      \n",
       "    \n",
       "    \n",
       "      \n",
       "      \n",
       "      \n",
       "      \n",
       "    \n",
       "    \n",
       "      \n",
       "      \n",
       "      \n",
       "      \n",
       "    \n",
       "    \n",
       "      \n",
       "      \n",
       "      \n",
       "      \n",
       "    \n",
       "    \n",
       "      \n",
       "      \n",
       "      \n",
       "      \n",
       "    \n",
       "    \n",
       "      \n",
       "      \n",
       "      \n",
       "      \n",
       "    \n",
       "  \n",
       "
IoCTypeObservableSourceIndex
0windows_pathC:\\Diagnostics\\UserTmp\\ftp.exe0
1windows_pathC:\\RECYCLER\\xxppyy.exe0
2windows_path.\\ftp0
3windows_pathC:\\Diagnostics\\UserTmp\\reg.exe1
4windows_path.\\reg1
5windows_pathC:\\Diagnostics\\UserTmp\\cmd.exe2
6windows_pathC:\\Diagnostics\\UserTmp\\rundll32.exe3
7windows_path.\\rundll323
8windows_pathC:\\Diagnostics\\UserTmp\\rundll32.exe4
9windows_pathc:\\users\\MSTICAdmin\\12345.exe4
\n",
       "
"
      ],
      "text/plain": [
       "        IoCType                           Observable SourceIndex\n",
       "0  windows_path       C:\\Diagnostics\\UserTmp\\ftp.exe           0\n",
       "1  windows_path               C:\\RECYCLER\\xxppyy.exe           0\n",
       "2  windows_path                                .\\ftp           0\n",
       "3  windows_path       C:\\Diagnostics\\UserTmp\\reg.exe           1\n",
       "4  windows_path                                .\\reg           1\n",
       "5  windows_path       C:\\Diagnostics\\UserTmp\\cmd.exe           2\n",
       "6  windows_path  C:\\Diagnostics\\UserTmp\\rundll32.exe           3\n",
       "7  windows_path                           .\\rundll32           3\n",
       "8  windows_path  C:\\Diagnostics\\UserTmp\\rundll32.exe           4\n",
       "9  windows_path        c:\\users\\MSTICAdmin\\12345.exe           4"
      ]
     },
     "execution_count": 42,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# You can specify multiple columns\n",
    "ioc_extractor.extract(data=process_tree.head(20), columns=['NewProcessName', 'CommandLine']).head(10)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "[Contents](#toc)\n",
    "### SourceIndex column allows you to merge the results with the input DataFrame\n",
    "Where an input row has multiple IoC matches the output of this merge will result in duplicate rows from the input (one per IoC match). The previous index is preserved in the second column (and in the SourceIndex column).\n",
    "\n",
    "Note: you will need to set the type of the SourceIndex column. In the example below case we are matching with the default numeric index so we force the type to be numeric. In cases where you are using an index of a different dtype you will need to convert the SourceIndex (dtype=object) to match the type of your index column."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 55,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "
\n",
       "\n",
       "\n",
       "  \n",
       "    \n",
       "      \n",
       "      \n",
       "      \n",
       "      \n",
       "      \n",
       "      \n",
       "      \n",
       "      \n",
       "      \n",
       "      \n",
       "      \n",
       "      \n",
       "      \n",
       "      \n",
       "      \n",
       "      \n",
       "      \n",
       "      \n",
       "      \n",
       "      \n",
       "      \n",
       "      \n",
       "      \n",
       "      \n",
       "      \n",
       "      \n",
       "      \n",
       "    \n",
       "  \n",
       "  \n",
       "    \n",
       "      \n",
       "      \n",
       "      \n",
       "      \n",
       "      \n",
       "      \n",
       "      \n",
       "      \n",
       "      \n",
       "      \n",
       "      \n",
       "      \n",
       "      \n",
       "      \n",
       "      \n",
       "      \n",
       "      \n",
       "      \n",
       "      \n",
       "      \n",
       "      \n",
       "      \n",
       "      \n",
       "      \n",
       "      \n",
       "      \n",
       "      \n",
       "    \n",
       "    \n",
       "      \n",
       "      \n",
       "      \n",
       "      \n",
       "      \n",
       "      \n",
       "      \n",
       "      \n",
       "      \n",
       "      \n",
       "      \n",
       "      \n",
       "      \n",
       "      \n",
       "      \n",
       "      \n",
       "      \n",
       "      \n",
       "      \n",
       "      \n",
       "      \n",
       "      \n",
       "      \n",
       "      \n",
       "      \n",
       "      \n",
       "      \n",
       "    \n",
       "    \n",
       "      \n",
       "      \n",
       "      \n",
       "      \n",
       "      \n",
       "      \n",
       "      \n",
       "      \n",
       "      \n",
       "      \n",
       "      \n",
       "      \n",
       "      \n",
       "      \n",
       "      \n",
       "      \n",
       "      \n",
       "      \n",
       "      \n",
       "      \n",
       "      \n",
       "      \n",
       "      \n",
       "      \n",
       "      \n",
       "      \n",
       "      \n",
       "    \n",
       "    \n",
       "      \n",
       "      \n",
       "      \n",
       "      \n",
       "      \n",
       "      \n",
       "      \n",
       "      \n",
       "      \n",
       "      \n",
       "      \n",
       "      \n",
       "      \n",
       "      \n",
       "      \n",
       "      \n",
       "      \n",
       "      \n",
       "      \n",
       "      \n",
       "      \n",
       "      \n",
       "      \n",
       "      \n",
       "      \n",
       "      \n",
       "      \n",
       "    \n",
       "    \n",
       "      \n",
       "      \n",
       "      \n",
       "      \n",
       "      \n",
       "      \n",
       "      \n",
       "      \n",
       "      \n",
       "      \n",
       "      \n",
       "      \n",
       "      \n",
       "      \n",
       "      \n",
       "      \n",
       "      \n",
       "      \n",
       "      \n",
       "      \n",
       "      \n",
       "      \n",
       "      \n",
       "      \n",
       "      \n",
       "      \n",
       "      \n",
       "    \n",
       "  \n",
       "
Unnamed: 0TenantIdAccountEventIDTimeGeneratedComputerSubjectUserSidSubjectUserNameSubjectDomainNameSubjectLogonIdNewProcessIdNewProcessNameTokenElevationTypeProcessIdCommandLineParentProcessNameTargetLogonIdSourceComputerIdTimeCreatedUtcNodeRoleLevelProcessId1NewProcessId1IoCTypeObservableSourceIndex
00802d39e1-9d70-404d-832c-2de5e2478edaMSTICAlertsWin1\\MSTICAdmin46882019-01-15 05:15:15.677MSTICAlertsWin1S-1-5-21-996632719-2361334927-4038480536-500MSTICAdminMSTICAlertsWin10xfaac270x1580C:\\Diagnostics\\UserTmp\\ftp.exe%%19360xbc8.\\ftp  -s:C:\\RECYCLER\\xxppyy.exeC:\\Windows\\System32\\cmd.exe0x046fe7078-61bb-4bed-9430-7ac01d91c2732019-01-15 05:15:15.677source0NaNNaNwindows_pathC:\\Diagnostics\\UserTmp\\ftp.exe0
10802d39e1-9d70-404d-832c-2de5e2478edaMSTICAlertsWin1\\MSTICAdmin46882019-01-15 05:15:15.677MSTICAlertsWin1S-1-5-21-996632719-2361334927-4038480536-500MSTICAdminMSTICAlertsWin10xfaac270x1580C:\\Diagnostics\\UserTmp\\ftp.exe%%19360xbc8.\\ftp  -s:C:\\RECYCLER\\xxppyy.exeC:\\Windows\\System32\\cmd.exe0x046fe7078-61bb-4bed-9430-7ac01d91c2732019-01-15 05:15:15.677source0NaNNaNwindows_pathC:\\RECYCLER\\xxppyy.exe0
20802d39e1-9d70-404d-832c-2de5e2478edaMSTICAlertsWin1\\MSTICAdmin46882019-01-15 05:15:15.677MSTICAlertsWin1S-1-5-21-996632719-2361334927-4038480536-500MSTICAdminMSTICAlertsWin10xfaac270x1580C:\\Diagnostics\\UserTmp\\ftp.exe%%19360xbc8.\\ftp  -s:C:\\RECYCLER\\xxppyy.exeC:\\Windows\\System32\\cmd.exe0x046fe7078-61bb-4bed-9430-7ac01d91c2732019-01-15 05:15:15.677source0NaNNaNwindows_path.\\ftp0
31802d39e1-9d70-404d-832c-2de5e2478edaMSTICAlertsWin1\\MSTICAdmin46882019-01-15 05:15:16.167MSTICAlertsWin1S-1-5-21-996632719-2361334927-4038480536-500MSTICAdminMSTICAlertsWin10xfaac270x16fcC:\\Diagnostics\\UserTmp\\reg.exe%%19360xbc8.\\reg  not /domain:everything that /sid:shines is /krbtgt:golden !C:\\Windows\\System32\\cmd.exe0x046fe7078-61bb-4bed-9430-7ac01d91c2732019-01-15 05:15:16.167sibling1NaNNaNwindows_pathC:\\Diagnostics\\UserTmp\\reg.exe1
41802d39e1-9d70-404d-832c-2de5e2478edaMSTICAlertsWin1\\MSTICAdmin46882019-01-15 05:15:16.167MSTICAlertsWin1S-1-5-21-996632719-2361334927-4038480536-500MSTICAdminMSTICAlertsWin10xfaac270x16fcC:\\Diagnostics\\UserTmp\\reg.exe%%19360xbc8.\\reg  not /domain:everything that /sid:shines is /krbtgt:golden !C:\\Windows\\System32\\cmd.exe0x046fe7078-61bb-4bed-9430-7ac01d91c2732019-01-15 05:15:16.167sibling1NaNNaNwindows_path.\\reg1
\n",
       "
"
      ],
      "text/plain": [
       "   Unnamed: 0                              TenantId  \\\n",
       "0           0  802d39e1-9d70-404d-832c-2de5e2478eda   \n",
       "1           0  802d39e1-9d70-404d-832c-2de5e2478eda   \n",
       "2           0  802d39e1-9d70-404d-832c-2de5e2478eda   \n",
       "3           1  802d39e1-9d70-404d-832c-2de5e2478eda   \n",
       "4           1  802d39e1-9d70-404d-832c-2de5e2478eda   \n",
       "\n",
       "                      Account  EventID            TimeGenerated  \\\n",
       "0  MSTICAlertsWin1\\MSTICAdmin     4688  2019-01-15 05:15:15.677   \n",
       "1  MSTICAlertsWin1\\MSTICAdmin     4688  2019-01-15 05:15:15.677   \n",
       "2  MSTICAlertsWin1\\MSTICAdmin     4688  2019-01-15 05:15:15.677   \n",
       "3  MSTICAlertsWin1\\MSTICAdmin     4688  2019-01-15 05:15:16.167   \n",
       "4  MSTICAlertsWin1\\MSTICAdmin     4688  2019-01-15 05:15:16.167   \n",
       "\n",
       "          Computer                                SubjectUserSid  \\\n",
       "0  MSTICAlertsWin1  S-1-5-21-996632719-2361334927-4038480536-500   \n",
       "1  MSTICAlertsWin1  S-1-5-21-996632719-2361334927-4038480536-500   \n",
       "2  MSTICAlertsWin1  S-1-5-21-996632719-2361334927-4038480536-500   \n",
       "3  MSTICAlertsWin1  S-1-5-21-996632719-2361334927-4038480536-500   \n",
       "4  MSTICAlertsWin1  S-1-5-21-996632719-2361334927-4038480536-500   \n",
       "\n",
       "  SubjectUserName SubjectDomainName SubjectLogonId NewProcessId  \\\n",
       "0      MSTICAdmin   MSTICAlertsWin1       0xfaac27       0x1580   \n",
       "1      MSTICAdmin   MSTICAlertsWin1       0xfaac27       0x1580   \n",
       "2      MSTICAdmin   MSTICAlertsWin1       0xfaac27       0x1580   \n",
       "3      MSTICAdmin   MSTICAlertsWin1       0xfaac27       0x16fc   \n",
       "4      MSTICAdmin   MSTICAlertsWin1       0xfaac27       0x16fc   \n",
       "\n",
       "                   NewProcessName TokenElevationType ProcessId  \\\n",
       "0  C:\\Diagnostics\\UserTmp\\ftp.exe             %%1936     0xbc8   \n",
       "1  C:\\Diagnostics\\UserTmp\\ftp.exe             %%1936     0xbc8   \n",
       "2  C:\\Diagnostics\\UserTmp\\ftp.exe             %%1936     0xbc8   \n",
       "3  C:\\Diagnostics\\UserTmp\\reg.exe             %%1936     0xbc8   \n",
       "4  C:\\Diagnostics\\UserTmp\\reg.exe             %%1936     0xbc8   \n",
       "\n",
       "                                                          CommandLine  \\\n",
       "0                                    .\\ftp  -s:C:\\RECYCLER\\xxppyy.exe   \n",
       "1                                    .\\ftp  -s:C:\\RECYCLER\\xxppyy.exe   \n",
       "2                                    .\\ftp  -s:C:\\RECYCLER\\xxppyy.exe   \n",
       "3  .\\reg  not /domain:everything that /sid:shines is /krbtgt:golden !   \n",
       "4  .\\reg  not /domain:everything that /sid:shines is /krbtgt:golden !   \n",
       "\n",
       "             ParentProcessName TargetLogonId  \\\n",
       "0  C:\\Windows\\System32\\cmd.exe           0x0   \n",
       "1  C:\\Windows\\System32\\cmd.exe           0x0   \n",
       "2  C:\\Windows\\System32\\cmd.exe           0x0   \n",
       "3  C:\\Windows\\System32\\cmd.exe           0x0   \n",
       "4  C:\\Windows\\System32\\cmd.exe           0x0   \n",
       "\n",
       "                       SourceComputerId           TimeCreatedUtc NodeRole  \\\n",
       "0  46fe7078-61bb-4bed-9430-7ac01d91c273  2019-01-15 05:15:15.677   source   \n",
       "1  46fe7078-61bb-4bed-9430-7ac01d91c273  2019-01-15 05:15:15.677   source   \n",
       "2  46fe7078-61bb-4bed-9430-7ac01d91c273  2019-01-15 05:15:15.677   source   \n",
       "3  46fe7078-61bb-4bed-9430-7ac01d91c273  2019-01-15 05:15:16.167  sibling   \n",
       "4  46fe7078-61bb-4bed-9430-7ac01d91c273  2019-01-15 05:15:16.167  sibling   \n",
       "\n",
       "   Level  ProcessId1  NewProcessId1       IoCType  \\\n",
       "0      0         NaN            NaN  windows_path   \n",
       "1      0         NaN            NaN  windows_path   \n",
       "2      0         NaN            NaN  windows_path   \n",
       "3      1         NaN            NaN  windows_path   \n",
       "4      1         NaN            NaN  windows_path   \n",
       "\n",
       "                       Observable  SourceIndex  \n",
       "0  C:\\Diagnostics\\UserTmp\\ftp.exe            0  \n",
       "1          C:\\RECYCLER\\xxppyy.exe            0  \n",
       "2                           .\\ftp            0  \n",
       "3  C:\\Diagnostics\\UserTmp\\reg.exe            1  \n",
       "4                           .\\reg            1  "
      ]
     },
     "execution_count": 55,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "input_df = data=process_tree.head(20)\n",
    "output_df = ioc_extractor.extract(data=input_df, columns=['NewProcessName', 'CommandLine'])\n",
    "# set the type of the SourceIndex column. In this case we are matching with the default numeric index.\n",
    "output_df['SourceIndex'] = pd.to_numeric(output_df['SourceIndex'])\n",
    "merged_df = pd.merge(left=input_df, right=output_df, how='outer', left_index=True, right_on='SourceIndex')\n",
    "merged_df.head()"
   ]
  }
 ],
 "metadata": {
  "celltoolbar": "Tags",
  "hide_input": false,
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.1"
  },
  "toc": {
   "base_numbering": 1,
   "nav_menu": {
    "height": "318.996px",
    "width": "320.994px"
   },
   "number_sections": false,
   "sideBar": true,
   "skip_h1_title": false,
   "title_cell": "Table of Contents",
   "title_sidebar": "Contents",
   "toc_cell": false,
   "toc_position": {
    "height": "calc(100% - 180px)",
    "left": "10px",
    "top": "150px",
    "width": "165px"
   },
   "toc_section_display": true,
   "toc_window_display": true
  },
  "varInspector": {
   "cols": {
    "lenName": 16,
    "lenType": 16,
    "lenVar": 40
   },
   "kernels_config": {
    "python": {
     "delete_cmd_postfix": "",
     "delete_cmd_prefix": "del ",
     "library": "var_list.py",
     "varRefreshCmd": "print(var_dic_list())"
    },
    "r": {
     "delete_cmd_postfix": ") ",
     "delete_cmd_prefix": "rm(",
     "library": "var_list.r",
     "varRefreshCmd": "cat(var_dic_list()) "
    }
   },
   "position": {
    "height": "406.193px",
    "left": "1468.4px",
    "right": "20px",
    "top": "120px",
    "width": "456.572px"
   },
   "types_to_exclude": [
    "module",
    "function",
    "builtin_function_or_method",
    "instance",
    "_Feature"
   ],
   "window_display": false
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}