Skip to content

Commit

Permalink
Notebook changes for testing
Browse files Browse the repository at this point in the history
  • Loading branch information
@ianhelle
ianhelle committed Apr 1, 2021
1 parent 0461b82 commit 83b7255
Show file tree
Hide file tree
Showing 22 changed files with 2,252 additions and 2,446 deletions.
995 changes: 6 additions & 989 deletions A Getting Started Guide For Azure Sentinel ML Notebooks.ipynb
View file

Large diffs are not rendered by default.

518 changes: 1 addition & 517 deletions ConfiguringNotebookEnvironment.ipynb
View file

Large diffs are not rendered by default.

73 changes: 21 additions & 52 deletions Entity Explorer - Domain and URL.ipynb
View file
Original file line number Diff line number Diff line change
Expand Up @@ -88,32 +88,19 @@
"outputs": [],
"source": [
"from pathlib import Path\n",
"import os\n",
"import sys\n",
"import warnings\n",
"from IPython.display import display, HTML, Markdown, Image\n",
"from IPython.display import display, HTML\n",
"\n",
"REQ_PYTHON_VER=(3, 6)\n",
"REQ_MSTICPY_VER=(0, 6, 0)\n",
"REQ_MSTICPY_VER=(1, 0, 0)\n",
"\n",
"display(HTML(\"<h3>Starting Notebook setup...</h3>\"))\n",
"if Path(\"./utils/nb_check.py\").is_file():\n",
" from utils.nb_check import check_python_ver, check_mp_ver\n",
"\n",
" check_python_ver(min_py_ver=REQ_PYTHON_VER)\n",
" try:\n",
" check_mp_ver(min_msticpy_ver=REQ_MSTICPY_VER)\n",
" except ImportError:\n",
" !pip install --upgrade msticpy\n",
" if \"msticpy\" in sys.modules:\n",
" importlib.reload(sys.modules[\"msticpy\"])\n",
" else:\n",
" import msticpy\n",
" check_mp_ver(REQ_MSTICPY_VER)\n",
" \n",
" from utils.nb_check import check_versions\n",
" check_versions(REQ_PYTHON_VER, REQ_MSTICPY_VER)\n",
"\n",
"# If not using Azure Notebooks, install msticpy with\n",
"# !pip install msticpy\n",
"\n",
"from msticpy.nbtools import nbinit\n",
"extra_imports = [\n",
" \"msticpy.nbtools, observationlist\",\n",
Expand All @@ -124,12 +111,7 @@
" namespace=globals(),\n",
" additional_packages=[\"oauthlib\", \"pyvis\", \"python-whois\"],\n",
" extra_imports=extra_imports,\n",
");\n",
"\n",
"WIDGET_DEFAULTS = {\n",
" \"layout\": widgets.Layout(width=\"95%\"),\n",
" \"style\": {\"description_width\": \"initial\"},\n",
"}"
");"
]
},
{
Expand Down Expand Up @@ -176,22 +158,15 @@
},
"outputs": [],
"source": [
"#See if we have an Azure Sentinel Workspace defined in our config file, if not let the user specify Workspace and Tenant IDs\n",
"from msticpy.nbtools.wsconfig import WorkspaceConfig\n",
"# See if we have an Azure Sentinel Workspace defined in our config file.\n",
"# If not, let the user specify Workspace and Tenant IDs\n",
"\n",
"ws_config = WorkspaceConfig()\n",
"try:\n",
" ws_id = ws_config['workspace_id']\n",
" ten_id = ws_config['tenant_id']\n",
" config = True\n",
" md(\"Workspace details collected from config file\")\n",
"except KeyError:\n",
" md(('Please go to your Log Analytics workspace, copy the workspace ID'\n",
" ' and/or tenant Id and paste here to enable connection to the workspace and querying of it..<br> '))\n",
" ws_id_wgt = nbwidgets.GetEnvironmentKey(env_var='WORKSPACE_ID',\n",
" prompt='Please enter your Log Analytics Workspace Id:', auto_display=True)\n",
" ten_id_wgt = nbwidgets.GetEnvironmentKey(env_var='TENANT_ID',\n",
" prompt='Please enter your Log Analytics Tenant Id:', auto_display=True)\n",
" config = False\n"
"if not ws_config.config_loaded:\n",
" ws_config.prompt_for_ws()\n",
" \n",
"qry_prov = QueryProvider(data_environment=\"AzureSentinel\")\n",
"print(\"done\")"
]
},
{
Expand All @@ -205,13 +180,9 @@
},
"outputs": [],
"source": [
"if config is False:\n",
" ws_id = ws_id_wgt.value\n",
" ten_id = ten_id_wgt.value\n",
"# Establish a query provider for Azure Sentinel and connect to it\n",
"qry_prov = QueryProvider('LogAnalytics')\n",
"la_connection_string = f'loganalytics://code().tenant(\"{ten_id}\").workspace(\"{ws_id}\")'\n",
"qry_prov.connect(connection_str=f'{la_connection_string}')\n",
"# Authenticate to Azure Sentinel workspace\n",
"qry_prov.connect(ws_config)\n",
"# Load TI Providers\n",
"tilookup = TILookup()\n",
"tilookup.reload_providers()\n",
"tilookup.provider_status"
Expand Down Expand Up @@ -1202,8 +1173,6 @@
"outputs": [],
"source": [
"if az_net_comms_df is not None and not az_net_comms_df.empty:\n",
" cm = sns.light_palette(\"green\", as_cmap=True)\n",
"\n",
" cols = [\n",
" \"VMName\",\n",
" \"VMIPAddress\",\n",
Expand Down Expand Up @@ -1344,9 +1313,9 @@
"celltoolbar": "Tags",
"hide_input": false,
"kernelspec": {
"display_name": "Python 3.6",
"display_name": "Python 3.6.7 64-bit ('condadev': conda)",
"language": "python",
"name": "python36"
"name": "python36764bitcondadevconda6cccf545f08246a1a5c093078bc87e5f"
},
"language_info": {
"codemirror_mode": {
Expand All @@ -1358,7 +1327,7 @@
"name": "python",
"nbconvert_exporter": "python",
"pygments_lexer": "ipython3",
"version": "3.6.7"
"version": "3.7.9"
},
"latex_envs": {
"LaTeX_envs_menu_present": true,
Expand Down Expand Up @@ -1435,4 +1404,4 @@
},
"nbformat": 4,
"nbformat_minor": 4
}
}
111 changes: 38 additions & 73 deletions Entity Explorer - Linux Host.ipynb
View file
Original file line number Diff line number Diff line change
Expand Up @@ -80,74 +80,50 @@
"outputs": [],
"source": [
"from pathlib import Path\n",
"import os\n",
"import sys\n",
"import warnings\n",
"from IPython.display import display, HTML, Markdown\n",
"from IPython.display import display, HTML\n",
"\n",
"REQ_PYTHON_VER=(3, 6)\n",
"REQ_MSTICPY_VER=(0, 6, 0)\n",
"REQ_MSTICPY_VER=(1, 0, 0)\n",
"REQ_MP_EXTRAS = [\"ml\"]\n",
"\n",
"display(HTML(\"<h3>Starting Notebook setup...</h3>\"))\n",
"if Path(\"./utils/nb_check.py\").is_file():\n",
" from utils.nb_check import check_python_ver, check_mp_ver\n",
" from utils.nb_check import check_versions\n",
" check_versions(REQ_PYTHON_VER, REQ_MSTICPY_VER, REQ_MP_EXTRAS)\n",
"\n",
" check_python_ver(min_py_ver=REQ_PYTHON_VER)\n",
" try:\n",
" check_mp_ver(min_msticpy_ver=REQ_MSTICPY_VER)\n",
" except ImportError:\n",
" !pip install --upgrade msticpy\n",
" if \"msticpy\" in sys.modules:\n",
" importlib.reload(sys.modules[\"msticpy\"])\n",
" else:\n",
" import msticpy\n",
" check_mp_ver(REQ_MSTICPY_VER)\n",
" \n",
"# If the installation fails try to manually install using\n",
"# !pip install --upgrade msticpy\n",
"\n",
"# If not using Azure Notebooks, install msticpy with\n",
"# !pip install msticpy\n",
"from msticpy.nbtools import nbinit\n",
"extra_imports = [\n",
" \"msticpy.nbtools, observationlist\",\n",
" \"msticpy.nbtools.foliummap, get_map_center\",\n",
" \"msticpy.common.exceptions, MsticpyException\",\n",
" \"msticpy.sectools.syslog_utils, create_host_record\",\n",
" \"msticpy.sectools.syslog_utils, cluster_syslog_logons_df\",\n",
" \"msticpy.sectools.syslog_utils, risky_sudo_sessions\",\n",
" \"msticpy.sectools.ip_utils, convert_to_ip_entities\",\n",
" \"msticpy.sectools, auditdextract\",\n",
" \"msticpy.sectools.cmd_line, risky_cmd_line\",\n",
" \"pyvis.network, Network\",\n",
" \"re\",\n",
" \"math, pi\",\n",
" \"ipwhois, IPWhois\",\n",
" \"bokeh.plotting, show\",\n",
" \"bokeh.plotting, Row\",\n",
" \"bokeh.models, ColumnDataSource\",\n",
" \"bokeh.models, FactorRange\",\n",
" \"bokeh.transform, factor_cmap\",\n",
" \"bokeh.transform, cumsum\",\n",
" \"bokeh.palettes, viridis\",\n",
" \"dns, reversename\",\n",
" \"dns, resolver\",\n",
" \"ipaddress, ip_address\",\n",
" \"functools, lru_cache\",\n",
" \"datetime,,dt\"\n",
"]\n",
"additional_packages = [\n",
" \"oauthlib\", \"pyvis\", \"python-whois\"\n",
" \"oauthlib\", \"pyvis\", \"python-whois\", \"seaborn\"\n",
"]\n",
"nbinit.init_notebook(\n",
" namespace=globals(),\n",
" additional_packages=additional_packages,\n",
" extra_imports=extra_imports,\n",
");\n",
"\n",
"WIDGET_DEFAULTS = {\n",
" \"layout\": widgets.Layout(width=\"95%\"),\n",
" \"style\": {\"description_width\": \"initial\"},\n",
"}\n",
"from bokeh.plotting import figure"
"\n",
"from bokeh.models import ColumnDataSource, FactorRange\n",
"from bokeh.palettes import viridis\n",
"from bokeh.plotting import show, Row, figure\n",
"from bokeh.transform import factor_cmap, cumsum\n",
"from dns import reversename, resolver\n",
"from functools import lru_cache\n",
"from ipaddress import ip_address\n",
"from ipwhois import IPWhois\n",
"from math import pi\n",
"from msticpy.common.exceptions import MsticpyException\n",
"from msticpy.nbtools import observationlist\n",
"from msticpy.nbtools.foliummap import get_map_center\n",
"from msticpy.sectools import auditdextract\n",
"from msticpy.sectools.cmd_line import risky_cmd_line\n",
"from msticpy.sectools.ip_utils import convert_to_ip_entities\n",
"from msticpy.sectools.syslog_utils import create_host_record, cluster_syslog_logons_df, risky_sudo_sessions\n",
"from pyvis.network import Network\n",
"import datetime as dt\n",
"import re\n"
]
},
{
Expand Down Expand Up @@ -193,22 +169,15 @@
},
"outputs": [],
"source": [
"#See if we have an Azure Sentinel Workspace defined in our config file, if not let the user specify Workspace and Tenant IDs\n",
"from msticpy.nbtools.wsconfig import WorkspaceConfig\n",
"# See if we have an Azure Sentinel Workspace defined in our config file.\n",
"# If not, let the user specify Workspace and Tenant IDs\n",
"\n",
"ws_config = WorkspaceConfig()\n",
"try:\n",
" ws_id = ws_config['workspace_id']\n",
" ten_id = ws_config['tenant_id']\n",
" md(\"Workspace details collected from config file\")\n",
" config = True\n",
"except:\n",
" md('Please go to your Log Analytics workspace, copy the workspace ID'\n",
" ' and/or tenant Id and paste here to enable connection to the workspace and querying of it..<br> ')\n",
" ws_id = nbwidgets.GetEnvironmentKey(env_var='WORKSPACE_ID',\n",
" prompt='Please enter your Log Analytics Workspace Id:', auto_display=True)\n",
" ten_id = nbwidgets.GetEnvironmentKey(env_var='TENANT_ID',\n",
" prompt='Please enter your Log Analytics Tenant Id:', auto_display=True)\n",
" config = False\n"
"if not ws_config.config_loaded:\n",
" ws_config.prompt_for_ws()\n",
" \n",
"qry_prov = QueryProvider(data_environment=\"AzureSentinel\")\n",
"print(\"done\")"
]
},
{
Expand All @@ -222,12 +191,8 @@
},
"outputs": [],
"source": [
"# Establish a query provider for Azure Sentinel and connect to it\n",
"if config is False:\n",
" ws_id = ws_id.value\n",
" ten_id = ten_id.value\n",
"qry_prov = QueryProvider('LogAnalytics')\n",
"qry_prov.connect(connection_str=ws_config.code_connect_str)"
"# Authenticate to Azure Sentinel workspace\n",
"qry_prov.connect(ws_config)"
]
},
{
Expand Down
63 changes: 17 additions & 46 deletions Entity Explorer - Windows Host.ipynb
View file
Original file line number Diff line number Diff line change
Expand Up @@ -67,32 +67,20 @@
"outputs": [],
"source": [
"from pathlib import Path\n",
"import os\n",
"import sys\n",
"import warnings\n",
"from IPython.display import display, HTML, Markdown\n",
"from IPython.display import display, HTML\n",
"\n",
"REQ_PYTHON_VER=(3, 6)\n",
"REQ_MSTICPY_VER=(0, 6, 0)\n",
"REQ_MSTICPY_VER=(1, 0, 0)\n",
"REQ_MP_EXTRAS = [\"ml\"]\n",
"\n",
"display(HTML(\"<h3>Starting Notebook setup...</h3>\"))\n",
"if Path(\"./utils/nb_check.py\").is_file():\n",
" from utils.nb_check import check_python_ver, check_mp_ver\n",
"\n",
" check_python_ver(min_py_ver=REQ_PYTHON_VER)\n",
" try:\n",
" check_mp_ver(min_msticpy_ver=REQ_MSTICPY_VER)\n",
" except ImportError:\n",
" !pip install --upgrade msticpy\n",
" if \"msticpy\" in sys.modules:\n",
" importlib.reload(sys.modules[\"msticpy\"])\n",
" else:\n",
" import msticpy\n",
" check_mp_ver(REQ_MSTICPY_VER)\n",
" \n",
" from utils.nb_check import check_versions\n",
" check_versions(REQ_PYTHON_VER, REQ_MSTICPY_VER, REQ_MP_EXTRAS)\n",
"\n",
"# If not using Azure Notebooks, install msticpy with\n",
"# !pip install msticpy\n",
"\n",
"from msticpy.nbtools import nbinit\n",
"nbinit.init_notebook(\n",
" namespace=globals(),\n",
Expand Down Expand Up @@ -135,26 +123,15 @@
"metadata": {},
"outputs": [],
"source": [
"#See if we have an Azure Sentinel Workspace defined in our config file, if not let the user specify Workspace and Tenant IDs\n",
"from msticpy.nbtools.wsconfig import WorkspaceConfig\n",
"# WorkspaceConfig.list_workspaces()\n",
"# ws_config = WorkspaceConfig(workspace=\"My_Workspace_Name\")\n",
"# calling WorkspaceConfig with no parameters will load the default workspace from msticpyconfig.yaml\n",
"# or fall back on a config.json file.\n",
"# See if we have an Azure Sentinel Workspace defined in our config file.\n",
"# If not, let the user specify Workspace and Tenant IDs\n",
"\n",
"ws_config = WorkspaceConfig()\n",
"try:\n",
" ws_id = ws_config['workspace_id']\n",
" ten_id = ws_config['tenant_id']\n",
" config = True\n",
" md(\"Workspace details collected from config file\")\n",
"except KeyError:\n",
" md(('Please go to your Log Analytics workspace, copy the workspace ID'\n",
" ' and/or tenant Id and paste here to enable connection to the workspace and querying of it..<br> '))\n",
" ws_id_wgt = nbwidgets.GetEnvironmentKey(env_var='WORKSPACE_ID',\n",
" prompt='Please enter your Log Analytics Workspace Id:', auto_display=True)\n",
" ten_id_wgt = nbwidgets.GetEnvironmentKey(env_var='TENANT_ID',\n",
" prompt='Please enter your Log Analytics Tenant Id:', auto_display=True)\n",
" config = False\n"
"if not ws_config.config_loaded:\n",
" ws_config.prompt_for_ws()\n",
" \n",
"qry_prov = QueryProvider(data_environment=\"AzureSentinel\")\n",
"print(\"done\")"
]
},
{
Expand All @@ -163,12 +140,8 @@
"metadata": {},
"outputs": [],
"source": [
"if config is False:\n",
" ws_id = ws_id_wgt.value\n",
" ten_id = ten_id_wgt.value\n",
"# Establish a query provider for Azure Sentinel and connect to it\n",
"qry_prov = QueryProvider('LogAnalytics')\n",
"qry_prov.connect(connection_str=ws_config.code_connect_str)\n",
"# Authenticate to Azure Sentinel workspace\n",
"qry_prov.connect(ws_config)\n",
"table_index = qry_prov.schema_tables"
]
},
Expand Down Expand Up @@ -1619,8 +1592,6 @@
"outputs": [],
"source": [
"if az_net_comms_df is not None and not az_net_comms_df.empty:\n",
" cm = sns.light_palette(\"green\", as_cmap=True)\n",
"\n",
" cols = [\n",
" \"VMName\",\n",
" \"VMIPAddress\",\n",
Expand Down Expand Up @@ -2019,4 +1990,4 @@
},
"nbformat": 4,
"nbformat_minor": 4
}
}
Loading

0 comments on commit 83b7255

Please sign in to comment.