Skip to content

Commit

Permalink
Covid Notebook Updates
Browse files Browse the repository at this point in the history
  • Loading branch information
@petebryan
petebryan committed Apr 8, 2021
1 parent 46f9c99 commit a8fe14b
Show file tree
Hide file tree
Showing 2 changed files with 100 additions and 116 deletions.
18 changes: 5 additions & 13 deletions A Getting Started Guide For Azure Sentinel ML Notebooks.ipynb
View file
Original file line number Diff line number Diff line change
Expand Up @@ -138,11 +138,12 @@
"\r\n",
"REQ_PYTHON_VER=(3, 6)\r\n",
"REQ_MSTICPY_VER=(1, 0, 0)\r\n",
"REQ_MP_EXTRAS=[\"Azure\"]\r\n",
"\r\n",
"display(HTML(\"<h3>Starting Notebook setup...</h3>\"))\r\n",
"if Path(\"./utils/nb_check.py\").is_file():\r\n",
" from utils.nb_check import check_versions\r\n",
" check_versions(REQ_PYTHON_VER, REQ_MSTICPY_VER, extras=[\"Azure\"])\r\n",
" check_versions(REQ_PYTHON_VER, REQ_MSTICPY_VER, REQ_MP_EXTRAS)\r\n",
" \r\n",
"from msticpy.nbtools import nbinit\r\n",
"nbinit.init_notebook(\r\n",
Expand Down Expand Up @@ -967,21 +968,12 @@
"metadata": {
"hide_input": false,
"kernelspec": {
"display_name": "Python (condadev)",
"language": "python",
"name": "condadev"
"display_name": "Python 3.6.10 64-bit ('Dev36': conda)",
"name": "python3610jvsc74a57bd0eec69deb3bab9f02a3e61673e87704ade89b113db30d8e00ac867bc872c8dbc5"
},
"language_info": {
"codemirror_mode": {
"name": "ipython",
"version": 3
},
"file_extension": ".py",
"mimetype": "text/x-python",
"name": "python",
"nbconvert_exporter": "python",
"pygments_lexer": "ipython3",
"version": "3.7.9"
"version": ""
},
"toc": {
"base_numbering": 1,
Expand Down
198 changes: 95 additions & 103 deletions Guided Hunting - Covid-19 Themed Threats.ipynb
View file
Original file line number Diff line number Diff line change
Expand Up @@ -39,42 +39,43 @@
"metadata": {},
"outputs": [],
"source": [
"from pathlib import Path\n",
"from IPython.display import display, HTML\n",
"\n",
"REQ_PYTHON_VER=(3, 6)\n",
"REQ_MSTICPY_VER=(1, 0, 0)\n",
"REQ_MP_EXTRAS = [\"ml\"]\n",
"\n",
"display(HTML(\"<h3>Starting Notebook setup...</h3>\"))\n",
"if Path(\"./utils/nb_check.py\").is_file():\n",
" from utils.nb_check import check_versions\n",
" check_versions(REQ_PYTHON_VER, REQ_MSTICPY_VER, REQ_MP_EXTRAS)\n",
"\n",
"\n",
"# If not using Azure Notebooks, install msticpy with\n",
"# !pip install msticpy\n",
"from msticpy.nbtools import nbinit\n",
"extra_imports = [\n",
" \"tqdm.notebook, tqdm\",\n",
" \"whois\",\n",
" \"dns\",\n",
" \"tldextract\",\n",
" \"datetime\",\n",
" \"msticpy.nbtools.foliummap, get_map_center\",\n",
" \"msticpy.nbtools.foliummap, get_center_ip_entities\",\n",
" \"functools, lru_cache\",\n",
"]\n",
"\n",
"additional_packages = [\n",
" \"tldextract\", \"IPWhois\", \"python-whois\"\n",
"]\n",
"nbinit.init_notebook(\n",
" namespace=globals(),\n",
" additional_packages=additional_packages,\n",
" extra_imports=extra_imports,\n",
");\n",
"\n",
"from pathlib import Path\r\n",
"from IPython.display import display, HTML\r\n",
"\r\n",
"REQ_PYTHON_VER=(3, 6)\r\n",
"REQ_MSTICPY_VER=(1, 0, 0)\r\n",
"REQ_MP_EXTRAS = [\"ml\"]\r\n",
"\r\n",
"display(HTML(\"<h3>Starting Notebook setup...</h3>\"))\r\n",
"if Path(\"./utils/nb_check.py\").is_file():\r\n",
" from utils.nb_check import check_versions\r\n",
" check_versions(REQ_PYTHON_VER, REQ_MSTICPY_VER, REQ_MP_EXTRAS)\r\n",
"\r\n",
"\r\n",
"# If not using Azure Notebooks, install msticpy with\r\n",
"# !pip install msticpy\r\n",
"from msticpy.nbtools import nbinit\r\n",
"extra_imports = [\r\n",
" \"tqdm.notebook, tqdm\",\r\n",
" \"whois\",\r\n",
" \"dns\",\r\n",
" \"tldextract\",\r\n",
" \"datetime\",\r\n",
" \"msticpy.nbtools.foliummap, get_map_center\",\r\n",
" \"msticpy.nbtools.foliummap, get_center_ip_entities\",\r\n",
" \"msticpy.sectools.ip_utils, convert_to_ip_entities\",\r\n",
" \"functools, lru_cache\",\r\n",
"]\r\n",
"\r\n",
"additional_packages = [\r\n",
" \"tldextract\", \"IPWhois\", \"python-whois\"\r\n",
"]\r\n",
"nbinit.init_notebook(\r\n",
" namespace=globals(),\r\n",
" additional_packages=additional_packages,\r\n",
" extra_imports=extra_imports,\r\n",
");\r\n",
"\r\n",
"from bokeh.plotting import figure"
]
},
Expand Down Expand Up @@ -134,27 +135,19 @@
"metadata": {},
"outputs": [],
"source": [
"start = query_times.start\n",
"end = query_times.end"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": [
"# Get Covid-19 related URLs from Network Logs\n",
"url_q = f\"\"\"\n",
"CommonSecurityLog\n",
"| where TimeGenerated between (datetime({start})..datetime({end}))\n",
"| extend Url = iif(isnotempty(RequestURL), RequestURL , iif(isnotempty(DestinationHostName), DestinationHostName, \"None\"))\n",
"| where Url != \"None\" \n",
"| distinct Url\n",
"| where tolower(Url) matches regex(\"(?i)(covid|corona.*virus)\")\n",
"\"\"\"\n",
"print(\"Collecting data...\")\n",
"url_data = qry_prov.exec_query(url_q)\n",
"start = query_times.start\r\n",
"end = query_times.end\r\n",
"# Get Covid-19 related URLs from Network Logs\r\n",
"url_q = f\"\"\"\r\n",
"CommonSecurityLog\r\n",
"| where TimeGenerated between (datetime({start})..datetime({end}))\r\n",
"| extend Url = iif(isnotempty(RequestURL), RequestURL , iif(isnotempty(DestinationHostName), DestinationHostName, \"None\"))\r\n",
"| where Url != \"None\" \r\n",
"| distinct Url\r\n",
"| where tolower(Url) matches regex(\"(?i)(covid|corona.*virus)\")\r\n",
"\"\"\"\r\n",
"print(\"Collecting data...\")\r\n",
"url_data = qry_prov.exec_query(url_q)\r\n",
"print(\"Done\")"
]
},
Expand All @@ -164,32 +157,32 @@
"metadata": {},
"outputs": [],
"source": [
"@lru_cache(maxsize=5000)\n",
"def get_domain(url):\n",
" try:\n",
" _, domain,tld = tldextract.extract(url)\n",
" return f\"{domain}.{tld}\"\n",
" except:\n",
" return None\n",
"\n",
"@lru_cache(maxsize=5000)\n",
"def whois_url(dom):\n",
" try:\n",
" wis = whois.whois(dom)\n",
" return wis['creation_date']\n",
" except:\n",
" return None\n",
"\n",
"tqdm.pandas(desc=\"Lookup progress\")\n",
"\n",
"if isinstance(url_data, pd.DataFrame) and not url_data.empty:\n",
" md(\"Extracting domains\")\n",
" url_data['domain'] = url_data['Url'].progress_apply(get_domain)\n",
" url_data = url_data['domain'].drop_duplicates().reset_index().drop(['index'], axis=1)\n",
" md(\"Getting domain registration dates for {len(url_data)} unique domains\")\n",
" md(\"This can take a while for large numbers of domains ~ 100 domains/min\")\n",
" url_data['creation_date'] = url_data['domain'].progress_apply(whois_url)\n",
"else:\n",
"@lru_cache(maxsize=5000)\r\n",
"def get_domain(url):\r\n",
" try:\r\n",
" _, domain,tld = tldextract.extract(url)\r\n",
" return f\"{domain}.{tld}\"\r\n",
" except:\r\n",
" return None\r\n",
"\r\n",
"@lru_cache(maxsize=5000)\r\n",
"def whois_url(dom):\r\n",
" try:\r\n",
" wis = whois.whois(dom)\r\n",
" return wis['creation_date']\r\n",
" except:\r\n",
" return None\r\n",
"\r\n",
"tqdm.pandas(desc=\"Lookup progress\")\r\n",
"\r\n",
"if isinstance(url_data, pd.DataFrame) and not url_data.empty:\r\n",
" md(\"Extracting domains\")\r\n",
" url_data['domain'] = url_data['Url'].progress_apply(get_domain)\r\n",
" url_data = url_data['domain'].drop_duplicates().reset_index().drop(['index'], axis=1)\r\n",
" md(f\"Getting domain registration dates for {len(url_data)} unique domains\")\r\n",
" md(\"This can take a while for large numbers of domains ~ 100 domains/min\")\r\n",
" url_data['creation_date'] = url_data['domain'].progress_apply(whois_url)\r\n",
"else:\r\n",
" md(\"No matches found.\")"
]
},
Expand Down Expand Up @@ -604,22 +597,22 @@
"metadata": {},
"outputs": [],
"source": [
"if cmd_lines:\n",
" cmd_line_clean = cmd_line.value.replace('\\\\', \"\\\\\\\\\")\n",
" cmd_line_q = f\"\"\"\n",
" SecurityEvent\n",
" | where TimeGenerated between (datetime({start})..datetime({end}))\n",
" | where EventID == 4688\n",
" | where CommandLine == '{cmd_line_clean}'\n",
" \"\"\"\n",
"\n",
" cmd_line_events = qry_prov.exec_query(cmd_line_q)\n",
"\n",
" if isinstance(cmd_line_events, pd.DataFrame) and not cmd_line_events.empty:\n",
" display(cmd_line_events)\n",
" else:\n",
" md(\"No events found\")\n",
"else:\n",
"if cmd_lines:\r\n",
" cmd_line_clean = cmd_line.value.replace('\\\\', \"\\\\\\\\\")\r\n",
" cmd_line_q = f\"\"\"\r\n",
" SecurityEvent\r\n",
" | where TimeGenerated between (datetime({start})..datetime({end}))\r\n",
" | where EventID == 4688\r\n",
" | where CommandLine == \"{cmd_line_clean}\"\r\n",
" \"\"\"\r\n",
"\r\n",
" cmd_line_events = qry_prov.exec_query(cmd_line_q)\r\n",
"\r\n",
" if isinstance(cmd_line_events, pd.DataFrame) and not cmd_line_events.empty:\r\n",
" display(cmd_line_events)\r\n",
" else:\r\n",
" md(\"No events found\")\r\n",
"else:\r\n",
" md(\"No Covid related process data found\")"
]
},
Expand Down Expand Up @@ -674,9 +667,8 @@
"metadata": {
"hide_input": false,
"kernelspec": {
"display_name": "Python 3.6",
"language": "python",
"name": "python36"
"display_name": "Python 3.6.10 64-bit ('Dev36': conda)",
"name": "python3610jvsc74a57bd0eec69deb3bab9f02a3e61673e87704ade89b113db30d8e00ac867bc872c8dbc5"
},
"language_info": {
"codemirror_mode": {
Expand All @@ -688,7 +680,7 @@
"name": "python",
"nbconvert_exporter": "python",
"pygments_lexer": "ipython3",
"version": "3.6.7"
"version": "3.6.10"
},
"latex_envs": {
"LaTeX_envs_menu_present": true,
Expand Down

0 comments on commit a8fe14b

Please sign in to comment.