{ "cells": [ { "cell_type": "markdown", "metadata": {}, "source": [ "# MSTICpy - Data Uploaders\n", "\n", "### Description\n", "This notebook provides a guided example of using the Log Analytics and Splunk Data Uploader included with MSTICpy.

\n", "Contents:\n", "- How to instanciate Uploaders\n", "- Uploading DataFrames\n", "- Uploading Files\n", "- Uploading Folders\n", "\n", "You must have msticpy installed with the Azure components to run this notebook:\n", "```\n", "%pip install --upgrade msticpy[azure]\n", "```\n" ] }, { "cell_type": "code", "execution_count": 1, "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "Processing imports....\n", "Checking configuration....\n", "No errors found.\n", "No warnings found.\n", "Setting options....\n" ] }, { "data": { "text/html": [ "

Notebook setup complete

" ], "text/plain": [ "" ] }, "metadata": {}, "output_type": "display_data" } ], "source": [ "#Setup\n", "from msticpy.init import nbinit\n", "extra_imports = [\"msticpy.data.uploaders.splunk_uploader, SplunkUploader\", \n", " \"msticpy.data.uploaders.loganalytics_uploader, LAUploader\"]\n", "\n", "nbinit.init_notebook(\n", " namespace=globals(),\n", " extra_imports=extra_imports,\n", ");\n", "\n", "WIDGET_DEFAULTS = {\n", " \"layout\": widgets.Layout(width=\"95%\"),\n", " \"style\": {\"description_width\": \"initial\"},\n", "}" ] }, { "cell_type": "code", "execution_count": 2, "metadata": {}, "outputs": [ { "data": { "text/html": [ "
\n", "\n", "\n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", "
Unnamed: 0TenantIdTimeGeneratedFlowStartTimeFlowEndTimeFlowIntervalEndTimeFlowTypeResourceGroupVMNameVMIPAddressPublicIPsSrcIPDestIPL4ProtocolL7ProtocolDestPortFlowDirectionAllowedOutFlowsAllowedInFlowsDeniedInFlowsDeniedOutFlowsRemoteRegionVMRegionAllExtIPsTotalAllowedFlows
088152b1ab41-869e-4138-9e40-2a4457f09bf02019-02-12 14:22:40.6972019-02-12 13:00:07.0002019-02-12 13:45:08.0002019-02-12 14:00:00.000AzurePublicasihuntomsworkspacergmsticalertswin110.0.3.5['65.55.44.109']NaNNaNThttps443.0O4.00.00.00.0eastus2eastus65.55.44.1094.0
187752b1ab41-869e-4138-9e40-2a4457f09bf02019-02-12 14:22:40.6812019-02-12 13:00:48.0002019-02-12 13:58:33.0002019-02-12 14:00:00.000AzurePublicasihuntomsworkspacergmsticalertswin110.0.3.5['13.71.172.130', '13.71.172.128']NaNNaNThttps443.0O18.00.00.00.0canadacentraleastus13.71.172.12818.0
\n", "
" ], "text/plain": [ " Unnamed: 0 TenantId TimeGenerated \\\n", "0 881 52b1ab41-869e-4138-9e40-2a4457f09bf0 2019-02-12 14:22:40.697 \n", "1 877 52b1ab41-869e-4138-9e40-2a4457f09bf0 2019-02-12 14:22:40.681 \n", "\n", " FlowStartTime FlowEndTime FlowIntervalEndTime \\\n", "0 2019-02-12 13:00:07.000 2019-02-12 13:45:08.000 2019-02-12 14:00:00.000 \n", "1 2019-02-12 13:00:48.000 2019-02-12 13:58:33.000 2019-02-12 14:00:00.000 \n", "\n", " FlowType ResourceGroup VMName VMIPAddress \\\n", "0 AzurePublic asihuntomsworkspacerg msticalertswin1 10.0.3.5 \n", "1 AzurePublic asihuntomsworkspacerg msticalertswin1 10.0.3.5 \n", "\n", " PublicIPs SrcIP DestIP L4Protocol L7Protocol \\\n", "0 ['65.55.44.109'] NaN NaN T https \n", "1 ['13.71.172.130', '13.71.172.128'] NaN NaN T https \n", "\n", " DestPort FlowDirection AllowedOutFlows AllowedInFlows DeniedInFlows \\\n", "0 443.0 O 4.0 0.0 0.0 \n", "1 443.0 O 18.0 0.0 0.0 \n", "\n", " DeniedOutFlows RemoteRegion VMRegion AllExtIPs TotalAllowedFlows \n", "0 0.0 eastus2 eastus 65.55.44.109 4.0 \n", "1 0.0 canadacentral eastus 13.71.172.128 18.0 " ] }, "execution_count": 2, "metadata": {}, "output_type": "execute_result" } ], "source": [ "# Load some sample data\n", "df = pd.read_csv('https://raw.githubusercontent.com/microsoft/msticpy/master/tests/testdata/az_net_flows.csv', parse_dates=['TimeGenerated'])\n", "df.head(2)" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "## LogAnalytics Data Uploader\n", "Below we collect some details required for our uploader, instanciate our LogAnalytics data uploader and pass our DataFrame loaded above to be uploaded.\n", "We are setting the debug flag on our uploader so we can get some additional details on our upload progress." ] }, { "cell_type": "code", "execution_count": 3, "metadata": {}, "outputs": [ { "data": { "application/vnd.jupyter.widget-view+json": { "model_id": "8d7a06200a5c403db4f6b39c9666738b", "version_major": 2, "version_minor": 0 }, "text/plain": [ "Text(value='', description='Workspace ID:')" ] }, "metadata": {}, "output_type": "display_data" }, { "data": { "application/vnd.jupyter.widget-view+json": { "model_id": "421afec5cfff4917b0396f935a9c8dd5", "version_major": 2, "version_minor": 0 }, "text/plain": [ "Password(description='Workspace Key:')" ] }, "metadata": {}, "output_type": "display_data" } ], "source": [ "la_ws_id = widgets.Text(description='Workspace ID:')\n", "la_ws_key = widgets.Password(description='Workspace Key:')\n", "display(la_ws_id)\n", "display(la_ws_key)" ] }, { "cell_type": "code", "execution_count": 3, "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "Upload response code: 200\n", "Upload to upload_demo complete\n" ] } ], "source": [ "# Instanciate our Uploader\n", "la_up = LAUploader(workspace=la_ws_id.value, workspace_secret=la_ws_key.value, debug=True)\n", "# Upload our DataFrame\n", "la_up.upload_df(data=df, table_name='upload_demo')" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "### Upload a file\n", "We can now upload a file to our Workspace using the same Uploader. We simply pass the path to the file we want to upload, and we can also pass a table name for the data to be uploaded to." ] }, { "cell_type": "code", "execution_count": 5, "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "Upload response code: 200\n", "Upload to upload_demo complete\n" ] } ], "source": [ "la_up.upload_file(file_path='data/alertlist.csv', table_name='upload_demo')" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "### Upload a folder\n", "We can now upload a file to our Workspace using the same Uploader. We simply pass the the path to the folder we want to upload file from. In this case we aren't going to pass a table name, in which case the name will be generated automatically for each file from the file's name. With a folder we get a progress bar showing the progress uploading each file." ] }, { "cell_type": "code", "execution_count": 6, "metadata": {}, "outputs": [ { "data": { "application/vnd.jupyter.widget-view+json": { "model_id": "b59b3e2a33a14d96bce3e34220ea84fc", "version_major": 2, "version_minor": 0 }, "text/plain": [ "HBox(children=(FloatProgress(value=0.0, description='Files', max=10.0, style=ProgressStyle(description_width='…" ] }, "metadata": {}, "output_type": "display_data" }, { "name": "stdout", "output_type": "stream", "text": [ "Upload response code: 200\n", "Upload to alertlist complete\n", "Upload response code: 200\n", "Upload to az_net_flows complete\n", "Upload response code: 200\n", "Upload to demo_exchange_data complete\n", "Upload response code: 200\n", "Upload to host_logons complete\n", "Upload response code: 200\n", "Upload to ip_locs complete\n", "Upload response code: 200\n", "Upload to processes_on_host complete\n", "Upload response code: 200\n", "Upload to raw_network complete\n", "Upload response code: 200\n", "Upload to sample_alerts complete\n", "Upload response code: 200\n", "Upload to TimeSeriesDemo complete\n", "Upload response code: 200\n", "Upload to ti_data complete\n", "\n" ] } ], "source": [ "la_up.upload_folder(folder_path='data/')" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "## Splunk Data Uploader\n", "The Splunk Uploader functions in the same manner as the LogAnalytics one.
\n", "Below we collect some details required for our uploader, instanciate our Splunk data uploader and pass our DataFrame loaded above to be uploaded.\n", "We are setting the debug flag on our uploader so we can get some additional details on our upload progress.
\n", "When uploading our DataFrame the only difference is that as well as providing a table name (which is represneted as sourcetype in Splunk), we also need to pass a Splunk index that we want to data uploaded to. Also as Splunk uploads data a line at a time we get a progress bar for the file as it uploads." ] }, { "cell_type": "code", "execution_count": 7, "metadata": {}, "outputs": [ { "data": { "application/vnd.jupyter.widget-view+json": { "model_id": "3366d467cc4746bf83c9da070e56ecf7", "version_major": 2, "version_minor": 0 }, "text/plain": [ "Text(value='', description='Splunk host')" ] }, "metadata": {}, "output_type": "display_data" }, { "data": { "application/vnd.jupyter.widget-view+json": { "model_id": "d5d66a08cc4340f5b4f924b78a05734b", "version_major": 2, "version_minor": 0 }, "text/plain": [ "Text(value='', description='Username')" ] }, "metadata": {}, "output_type": "display_data" }, { "data": { "application/vnd.jupyter.widget-view+json": { "model_id": "55d2699324d14e69a87cf89ea73a7cb6", "version_major": 2, "version_minor": 0 }, "text/plain": [ "Password(description='Password')" ] }, "metadata": {}, "output_type": "display_data" } ], "source": [ "sp_host = widgets.Text(description='Splunk host')\n", "sp_user = widgets.Text(description='Username')\n", "sp_pwrd = widgets.Password(description='Password')\n", "display(sp_host)\n", "display(sp_user)\n", "display(sp_pwrd)" ] }, { "cell_type": "code", "execution_count": 3, "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "connected\n" ] }, { "data": { "application/vnd.jupyter.widget-view+json": { "model_id": "f1bda0f665da4e449fbc172f02955e59", "version_major": 2, "version_minor": 0 }, "text/plain": [ "HBox(children=(FloatProgress(value=0.0, description='Rows', max=460.0, style=ProgressStyle(description_width='…" ] }, "metadata": {}, "output_type": "display_data" }, { "name": "stdout", "output_type": "stream", "text": [ "\n", "Upload complete\n" ] } ], "source": [ "# Instanciate our Uploader\n", "spup = SplunkUploader(username=sp_user.value, host=sp_host.value, password=sp_pwrd.value, debug=True)\n", "# Upload our DataFrame\n", "spup.upload_df(data=df, table_name='upload_test', index_name='upload_test')" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "### Upload a file\n", "We can now upload a file to our Workspace using the same Uploader. We simply pass the path to the file we want to upload along with the index name, and we can also pass a table name for the data to be uploaded to." ] }, { "cell_type": "code", "execution_count": 4, "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "Exception reporting mode: Verbose\n" ] }, { "data": { "application/vnd.jupyter.widget-view+json": { "model_id": "140a61fa3eda4b1793f6f4f9aff7d847", "version_major": 2, "version_minor": 0 }, "text/plain": [ "HBox(children=(FloatProgress(value=0.0, description='Rows', max=189.0, style=ProgressStyle(description_width='…" ] }, "metadata": {}, "output_type": "display_data" }, { "name": "stdout", "output_type": "stream", "text": [ "\n", "Upload complete\n" ] } ], "source": [ "spup.upload_file(file_path='data/alertlist.csv', index_name='upload_demo', table_name='upload_demo')" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "### Upload a folder\n", "We can now upload a file to our Workspace using the same Uploader. We simply pass the the path to the folder we want to upload file from. In this case we aren't going to pass a table name, in which case the name will be generated automatically for each file from the file's name however we still need to pass and index name." ] }, { "cell_type": "code", "execution_count": 7, "metadata": {}, "outputs": [ { "data": { "application/vnd.jupyter.widget-view+json": { "model_id": "d357e8d4f1804e1783055190e70eed65", "version_major": 2, "version_minor": 0 }, "text/plain": [ "HBox(children=(FloatProgress(value=0.0, description='Files', max=10.0, style=ProgressStyle(description_width='…" ] }, "metadata": {}, "output_type": "display_data" }, { "data": { "application/vnd.jupyter.widget-view+json": { "model_id": "76a8427b7919484199701613f624d648", "version_major": 2, "version_minor": 0 }, "text/plain": [ "HBox(children=(FloatProgress(value=0.0, description='Rows', max=189.0, style=ProgressStyle(description_width='…" ] }, "metadata": {}, "output_type": "display_data" }, { "name": "stdout", "output_type": "stream", "text": [ "\n", "Upload complete\n", "data\\alertlist.csv uploaded to alertlist\n" ] }, { "data": { "application/vnd.jupyter.widget-view+json": { "model_id": "7b6bec5403424259b23791440925ac96", "version_major": 2, "version_minor": 0 }, "text/plain": [ "HBox(children=(FloatProgress(value=0.0, description='Rows', max=460.0, style=ProgressStyle(description_width='…" ] }, "metadata": {}, "output_type": "display_data" }, { "name": "stdout", "output_type": "stream", "text": [ "\n", "Upload complete\n", "data\\az_net_flows.csv uploaded to az_net_flows\n" ] }, { "data": { "application/vnd.jupyter.widget-view+json": { "model_id": "b13fbc6393a4412fa2db7ba1382178e0", "version_major": 2, "version_minor": 0 }, "text/plain": [ "HBox(children=(FloatProgress(value=0.0, description='Rows', max=1334.0, style=ProgressStyle(description_width=…" ] }, "metadata": {}, "output_type": "display_data" }, { "name": "stdout", "output_type": "stream", "text": [ "\n", "Upload complete\n", "data\\demo_exchange_data.csv uploaded to demo_exchange_data\n" ] }, { "data": { "application/vnd.jupyter.widget-view+json": { "model_id": "ba139fa3842a41e9a569481e4b539ae2", "version_major": 2, "version_minor": 0 }, "text/plain": [ "HBox(children=(FloatProgress(value=0.0, description='Rows', max=14.0, style=ProgressStyle(description_width='i…" ] }, "metadata": {}, "output_type": "display_data" }, { "name": "stdout", "output_type": "stream", "text": [ "\n", "Upload complete\n", "data\\host_logons.csv uploaded to host_logons\n" ] }, { "data": { "application/vnd.jupyter.widget-view+json": { "model_id": "56c8a58f26234be7b034153b114bf62e", "version_major": 2, "version_minor": 0 }, "text/plain": [ "HBox(children=(FloatProgress(value=0.0, description='Rows', max=87.0, style=ProgressStyle(description_width='i…" ] }, "metadata": {}, "output_type": "display_data" }, { "name": "stdout", "output_type": "stream", "text": [ "\n", "Upload complete\n", "data\\ip_locs.csv uploaded to ip_locs\n" ] }, { "data": { "application/vnd.jupyter.widget-view+json": { "model_id": "1ea09392e762426e98ceeb8596c5bc8e", "version_major": 2, "version_minor": 0 }, "text/plain": [ "HBox(children=(FloatProgress(value=0.0, description='Rows', max=363.0, style=ProgressStyle(description_width='…" ] }, "metadata": {}, "output_type": "display_data" }, { "name": "stdout", "output_type": "stream", "text": [ "\n", "Upload complete\n", "data\\processes_on_host.csv uploaded to processes_on_host\n" ] }, { "data": { "application/vnd.jupyter.widget-view+json": { "model_id": "14d86e700fad482da55d6c9f32db9d49", "version_major": 2, "version_minor": 0 }, "text/plain": [ "HBox(children=(FloatProgress(value=0.0, description='Rows', max=32.0, style=ProgressStyle(description_width='i…" ] }, "metadata": {}, "output_type": "display_data" }, { "name": "stdout", "output_type": "stream", "text": [ "\n", "Upload complete\n", "data\\raw_network.csv uploaded to raw_network\n" ] }, { "data": { "application/vnd.jupyter.widget-view+json": { "model_id": "ff7221ded09a4b698567e9335ca12de2", "version_major": 2, "version_minor": 0 }, "text/plain": [ "HBox(children=(FloatProgress(value=0.0, description='Rows', max=78.0, style=ProgressStyle(description_width='i…" ] }, "metadata": {}, "output_type": "display_data" }, { "name": "stdout", "output_type": "stream", "text": [ "\n", "Upload complete\n", "data\\sample_alerts.csv uploaded to sample_alerts\n" ] }, { "data": { "application/vnd.jupyter.widget-view+json": { "model_id": "d5fe6b46a1cd4469b6e2a5a722903b3f", "version_major": 2, "version_minor": 0 }, "text/plain": [ "HBox(children=(FloatProgress(value=0.0, description='Rows', max=840.0, style=ProgressStyle(description_width='…" ] }, "metadata": {}, "output_type": "display_data" }, { "name": "stdout", "output_type": "stream", "text": [ "\n", "Upload complete\n", "data\\TimeSeriesDemo.csv uploaded to TimeSeriesDemo\n" ] }, { "data": { "application/vnd.jupyter.widget-view+json": { "model_id": "b5ae2edc49eb448fa97ce4ef9ce15d5a", "version_major": 2, "version_minor": 0 }, "text/plain": [ "HBox(children=(FloatProgress(value=0.0, description='Rows', max=80.0, style=ProgressStyle(description_width='i…" ] }, "metadata": {}, "output_type": "display_data" }, { "name": "stdout", "output_type": "stream", "text": [ "\n", "Upload complete\n", "data\\ti_data.csv uploaded to ti_data\n", "\n" ] } ], "source": [ "spup.upload_folder(folder_path='data/', index_name='upload_demo') " ] }, { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [], "source": [] } ], "metadata": { "kernelspec": { "display_name": "Python 3", "language": "python", "name": "python3" }, "language_info": { "codemirror_mode": { "name": "ipython", "version": 3 }, "file_extension": ".py", "mimetype": "text/x-python", "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", "version": "3.7.9" }, "widgets": { "application/vnd.jupyter.widget-state+json": { "state": {}, "version_major": 2, "version_minor": 0 } } }, "nbformat": 4, "nbformat_minor": 4 }