From 5c10f61d827b28185d9592afc124eda4e9af2ef5 Mon Sep 17 00:00:00 2001 From: Fabian Bader Date: Fri, 24 May 2024 14:28:57 +0200 Subject: [PATCH 1/5] feat: Add support for scheduled YAML conversion with parameter file --- tests/Convert-SentinelARYamlToArm.tests.ps1 | 37 ++++++++++++++++++ tests/examples/ScheduledParam.params.yaml | 16 ++++++++ tests/examples/ScheduledParam.yaml | 42 +++++++++++++++++++++ 3 files changed, 95 insertions(+) create mode 100644 tests/examples/ScheduledParam.params.yaml create mode 100644 tests/examples/ScheduledParam.yaml diff --git a/tests/Convert-SentinelARYamlToArm.tests.ps1 b/tests/Convert-SentinelARYamlToArm.tests.ps1 index b17da57..13bc97e 100644 --- a/tests/Convert-SentinelARYamlToArm.tests.ps1 +++ b/tests/Convert-SentinelARYamlToArm.tests.ps1 @@ -12,6 +12,12 @@ param( [String] $exampleScheduledTTPFilePath = "./tests/examples/TTPWithTacticsNTechniques.yaml", [Parameter()] + [String] + $exampleScheduledWithVariablesFilePath = "./tests/examples/ScheduledParam.yaml", + [Parameter()] + [String] + $exampleScheduledParameterFilePath = "./tests/examples/ScheduledParam.params.yaml", + [Parameter()] [Switch] $RetainTestFiles = $false ) @@ -393,6 +399,37 @@ Describe "Convert-SentinelARYamlToArm" { } } + Context "Scheduled with parameter file provided" { + BeforeAll { + Copy-Item -Path $exampleScheduledWithVariablesFilePath -Destination "TestDrive:/Scheduled.yaml" -Force + Copy-Item -Path $exampleScheduledParameterFilePath -Destination "TestDrive:/ScheduledParam.params.yaml" -Force + Convert-SentinelARYamlToArm -Filename "TestDrive:/Scheduled.yaml" -OutFile "TestDrive:/Scheduled.json" -ParameterFile "TestDrive:/ScheduledParam.params.yaml" + $armTemplate = Get-Content -Path "TestDrive:/Scheduled.json" -Raw | ConvertFrom-Json + } + + AfterEach { + if ( -not $RetainTestFiles) { + Remove-Item -Path "TestDrive:/*" -Include *.json -Force + } + } + + It "Should have the correct replaced parameter values" { + $armTemplate.resources.properties.queryPeriod | Should -Be "PT1H" + $armTemplate.resources.properties.queryFrequency | Should -Be "PT1H" + $armTemplate.resources.properties.description | Should -Match "'403' or '404'" + } + + It "Should have the correct added query values" { + $armTemplate.resources.properties.query | Should -Match "// Example description. Will be added to the beginning of the query." + $armTemplate.resources.properties.query | Should -Match "// Example text that will be added to the end of the query." + } + + It "Should have the correct variables replaced" { + $armTemplate.resources.properties.query | Should -Match 'where Message in \("403","404"\)' + $armTemplate.resources.properties.query | Should -Match 'where NumberOfErrors > 200' + } + } + AfterAll { Remove-Module SentinelARConverter -Force } diff --git a/tests/examples/ScheduledParam.params.yaml b/tests/examples/ScheduledParam.params.yaml new file mode 100644 index 0000000..420e875 --- /dev/null +++ b/tests/examples/ScheduledParam.params.yaml @@ -0,0 +1,16 @@ +OverwriteProperties: + description: |- + Identifies instances where Wazuh logged over 200 '403' or '404' Web Errors from one IP Address. + To onboard Wazuh data into Sentinel please view: https://github.com/wazuh/wazuh-documentation/blob/master/source/azure/monitoring%20activity.rst + queryFrequency: 1h + queryPeriod: 1h +PrependQuery: | + // Example description. Will be added to the beginning of the query. +AppendQuery: | + // Example text that will be added to the end of the query. + | extend TimeGenerated = StartTime +ReplaceQueryVariables: + NumberOfErrors: 200 + ErrorCodes: + - "403" + - "404" \ No newline at end of file diff --git a/tests/examples/ScheduledParam.yaml b/tests/examples/ScheduledParam.yaml new file mode 100644 index 0000000..667ca67 --- /dev/null +++ b/tests/examples/ScheduledParam.yaml @@ -0,0 +1,42 @@ +id: 2790795b-7dba-483e-853f-44aa0bc9c985 +name: Wazuh - Large Number of Web errors from an IP +description: | + 'Identifies instances where Wazuh logged over 400 '403' Web Errors from one IP Address. To onboard Wazuh data into Sentinel please view: https://github.com/wazuh/wazuh-documentation/blob/master/source/azure/monitoring%20activity.rst' +severity: Low +requiredDataConnectors: [] +queryFrequency: 1d +queryPeriod: 1d +triggerOperator: gt +triggerThreshold: 0 +tactics: + - Persistence +query: | + CommonSecurityLog + | where DeviceProduct =~ "Wazuh" + | where Activity has "Web server 400 error code." + | where Message in (%%ErrorCodes%%) + | extend HostName=substring(split(DeviceCustomString1,")")[0],1) + | summarize StartTime = min(TimeGenerated), EndTime = max(TimeGenerated), NumberOfErrors = dcount(SourceIP) by HostName, SourceIP + | where NumberOfErrors > %%NumberOfErrors%% + | sort by NumberOfErrors desc + | extend timestamp = StartTime +entityMappings: + - entityType: Host + fieldMappings: + - identifier: HostName + columnName: HostName + - entityType: IP + fieldMappings: + - identifier: Address + columnName: SourceIP +version: 1.0.3 +kind: Scheduled +metadata: + source: + kind: Community + author: + name: Jordan Ross + support: + tier: Community + categories: + domains: ["Security - Others", "Networking"] From 2864919e20c6322f786f88c440460d76b72f47f8 Mon Sep 17 00:00:00 2001 From: Fabian Bader Date: Fri, 24 May 2024 14:29:08 +0200 Subject: [PATCH 2/5] feat: Add support for scheduled YAML conversion with parameter file --- src/public/Convert-SentinelARYamlToArm.ps1 | 78 ++++++++++++++++++++++ 1 file changed, 78 insertions(+) diff --git a/src/public/Convert-SentinelARYamlToArm.ps1 b/src/public/Convert-SentinelARYamlToArm.ps1 index c69346e..f39d59c 100644 --- a/src/public/Convert-SentinelARYamlToArm.ps1 +++ b/src/public/Convert-SentinelARYamlToArm.ps1 @@ -103,6 +103,9 @@ function Convert-SentinelARYamlToArm { [Parameter()] [string]$Severity, + [Parameter()] + [string]$ParameterFile, + [Parameter()] [datetime]$StartRunningAt, @@ -120,6 +123,16 @@ function Convert-SentinelARYamlToArm { throw "File not found" } } + + if ($ParameterFile) { + try { + if (-not (Test-Path $ParameterFile)) { + Write-Error -Exception + } + } catch { + throw "Parameters file not found" + } + } } process { @@ -143,6 +156,71 @@ function Convert-SentinelARYamlToArm { throw "Could not convert source file. YAML might be corrupted" } + try { + if ($ParameterFile) { + Write-Verbose "Read parameters file `"$ParameterFile`"" + $Parameters = Get-Content $ParameterFile | ConvertFrom-Yaml + } else { + Write-Verbose "No parameters file provided" + } + } catch { + throw "Could not convert parameters file. YAML might be corrupted" + } + + #region Parameter file handling + if ($Parameters) { + #region Overwrite values from parameters file + if ($Parameters.OverwriteProperties) { + foreach ($Key in $Parameters.OverwriteProperties.Keys) { + if ($analyticRule.ContainsKey($Key)) { + Write-Verbose "Overwriting property $Key with $($Parameters.OverwriteProperties[$Key])" + $analyticRule[$Key] = $Parameters.OverwriteProperties[$Key] + } + } + } else { + Write-Verbose "No properties to overwrite in provided parameters file" + } + #endregion Overwrite values from parameters file + + #region Prepend KQL query with data from parameters file + if ($Parameters.PrependQuery) { + $analyticRule.query = $Parameters.PrependQuery + $analyticRule.query + } else { + Write-Verbose "No query to prepend in provided parameters file" + } + #endregion Prepend KQL query with data from parameters file + + #region Append KQL query with data from parameters file + if ($Parameters.AppendQuery) { + $analyticRule.query = $analyticRule.query + $Parameters.AppendQuery + } else { + Write-Verbose "No query to append in provided parameters file" + } + #endregion Append KQL query with data from parameters file + + #region Replace variables in KQL query with data from parameters file + if ($Parameters.ReplaceQueryVariables) { + foreach ($Key in $Parameters.ReplaceQueryVariables.Keys) { + if ($Parameters.ReplaceQueryVariables[$Key].Count -gt 1) { + # Join array values with comma and wrap in quotes + $ReplaceValue = $Parameters.ReplaceQueryVariables[$Key] -join '","' + $ReplaceValue = '"' + $ReplaceValue + '"' + } else { + # Use single value + $ReplaceValue = $Parameters.ReplaceQueryVariables[$Key] + } + Write-Verbose "Replacing variable %%$Key%% with $($ReplaceValue)" + $analyticRule.query = $analyticRule.query -replace "%%$($Key)%%", $ReplaceValue + } + } else { + Write-Verbose "No variables to replace in provided parameters file" + } + #endregion Replace variables in KQL query with data from parameters file + + Write-Verbose "$($analyticRule | ConvertTo-Json -Depth 99)" + } + #endregion Parameter file handling + if ( [string]::IsNullOrWhiteSpace($analyticRule.name) -or [string]::IsNullOrWhiteSpace($analyticRule.id) ) { throw "Analytics Rule name or id is empty. YAML might be corrupted" } From 0b0cf7acf7542b7219cf000358c9e6d31fbc7481 Mon Sep 17 00:00:00 2001 From: Fabian Bader Date: Fri, 24 May 2024 14:29:31 +0200 Subject: [PATCH 3/5] Update module version to 2.3.0 --- src/SentinelARConverter.psd1 | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/SentinelARConverter.psd1 b/src/SentinelARConverter.psd1 index def0297..c09e3e8 100644 --- a/src/SentinelARConverter.psd1 +++ b/src/SentinelARConverter.psd1 @@ -12,7 +12,7 @@ RootModule = 'SentinelARConverter.psm1' # Version number of this module. - ModuleVersion = '2.2.4' + ModuleVersion = '2.3.0' # Supported PSEditions # CompatiblePSEditions = @() From 0778b7f502e1cab767129c80598c4bcc68b8fe84 Mon Sep 17 00:00:00 2001 From: Fabian Bader Date: Fri, 24 May 2024 15:01:07 +0200 Subject: [PATCH 4/5] feat: Add new properties from parameter file to Convert-SentinelARYamlToArm.ps1 --- src/public/Convert-SentinelARYamlToArm.ps1 | 3 +++ tests/Convert-SentinelARYamlToArm.tests.ps1 | 4 ++++ tests/examples/ScheduledParam.params.yaml | 4 +++- 3 files changed, 10 insertions(+), 1 deletion(-) diff --git a/src/public/Convert-SentinelARYamlToArm.ps1 b/src/public/Convert-SentinelARYamlToArm.ps1 index f39d59c..c64bc16 100644 --- a/src/public/Convert-SentinelARYamlToArm.ps1 +++ b/src/public/Convert-SentinelARYamlToArm.ps1 @@ -175,6 +175,9 @@ function Convert-SentinelARYamlToArm { if ($analyticRule.ContainsKey($Key)) { Write-Verbose "Overwriting property $Key with $($Parameters.OverwriteProperties[$Key])" $analyticRule[$Key] = $Parameters.OverwriteProperties[$Key] + } else { + Write-Verbose "Add new property $Key with $($Parameters.OverwriteProperties[$Key])" + $analyticRule.Add($Key, $Parameters.OverwriteProperties[$Key]) } } } else { diff --git a/tests/Convert-SentinelARYamlToArm.tests.ps1 b/tests/Convert-SentinelARYamlToArm.tests.ps1 index 13bc97e..c35814f 100644 --- a/tests/Convert-SentinelARYamlToArm.tests.ps1 +++ b/tests/Convert-SentinelARYamlToArm.tests.ps1 @@ -419,6 +419,10 @@ Describe "Convert-SentinelARYamlToArm" { $armTemplate.resources.properties.description | Should -Match "'403' or '404'" } + It "Should have all new properties from parameter file" { + $armTemplate.resources.properties.customDetails | Should -Not -BeNullOrEmpty + } + It "Should have the correct added query values" { $armTemplate.resources.properties.query | Should -Match "// Example description. Will be added to the beginning of the query." $armTemplate.resources.properties.query | Should -Match "// Example text that will be added to the end of the query." diff --git a/tests/examples/ScheduledParam.params.yaml b/tests/examples/ScheduledParam.params.yaml index 420e875..be5503f 100644 --- a/tests/examples/ScheduledParam.params.yaml +++ b/tests/examples/ScheduledParam.params.yaml @@ -4,6 +4,8 @@ OverwriteProperties: To onboard Wazuh data into Sentinel please view: https://github.com/wazuh/wazuh-documentation/blob/master/source/azure/monitoring%20activity.rst queryFrequency: 1h queryPeriod: 1h + customDetails: + HostName: HostName PrependQuery: | // Example description. Will be added to the beginning of the query. AppendQuery: | @@ -12,5 +14,5 @@ AppendQuery: | ReplaceQueryVariables: NumberOfErrors: 200 ErrorCodes: - - "403" + - 403 - "404" \ No newline at end of file From 638bd6d1f72817ed4c6eb6eb00bcc5d737ccc944 Mon Sep 17 00:00:00 2001 From: Fabian Bader Date: Fri, 24 May 2024 15:03:34 +0200 Subject: [PATCH 5/5] docs: Add support for scheduled YAML conversion with parameter file --- README.md | 103 ++++++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 103 insertions(+) diff --git a/README.md b/README.md index 529a789..9279bf3 100644 --- a/README.md +++ b/README.md @@ -2,6 +2,8 @@ # Sentinel Analytics Rule converter +[![PSGallery Version](https://img.shields.io/powershellgallery/v/SentinelARConverter.svg?style=flat&logo=powershell&label=PSGallery%20Version)](https://www.powershellgallery.com/packages/SentinelARConverter) [![PSGallery Downloads](https://img.shields.io/powershellgallery/dt/SentinelARConverter.svg?style=flat&logo=powershell&label=PSGallery%20Downloads)](https://www.powershellgallery.com/packages/SentinelARConverter) + ## Installation ```PowerShell @@ -72,8 +74,109 @@ Get-Content "C:\Users\User\Downloads\Azure_Sentinel_analytic_rule.yaml" | Conver If no output file path is given, the output will be send to `stdout` +```PowerShell +Convert-SentinelARYamlToArm -Filename "C:\Users\User\Downloads\Azure_Sentinel_analytic_rule.yaml" -ParameterFile "C:\Users\User\Downloads\Azure_Sentinel_analytic_rule.params.yaml" -UseOriginalFilename +``` + +In this case the yaml file is converted and saved with the original file name (`Azure_Sentinel_analytic_rule.json`) but in the process of converting the file additional changes, according to the parameter file are applied. + +## Parameter file + +There are four different types of parametrization you can use. Each must be defined in it's own subsection. + +There is no validation of the values provided, which can result in invalid arm templates. + +Only [valid properties](https://learn.microsoft.com/en-us/azure/templates/microsoft.securityinsights/alertrules?pivots=deployment-language-arm-template#scheduledalertruleproperties-1) should be added. + +```yaml +OverwriteProperties: + queryFrequency: 1h + queryPeriod: 1h +PrependQuery: | + // Example description. Will be added to the beginning of the query. +AppendQuery: | + // Example text that will be added to the end of the query. +ReplaceQueryVariables: + NumberOfErrors: 200 + ErrorCodes: + - "403" + - "404" +``` + +### OverwriteProperties + +Every key found in this section is used to either replace the existing key or is added as a new key to the resulting ARM template. Make sure to use the correct spacing to ensure that the correct keys are overwritten. + +```yaml +OverwriteProperties: + queryFrequency: 1h + queryPeriod: 1h +``` + +In this example the `queryFrequency` and the `queryPeriod` are adjusted. + +### PrependQuery + +This text will be added to the beginning of the KQL query. If `PrependQuery: |` is used, a newline will be added automatically. If you use `PrependQuery: |-` no newline will be written. + +```yaml +PrependQuery: | + // Example description. Will be added to the beginning of the query. +``` + +This example adds a description at the top of the KQL query and adds a newline. + +### AppendQuery + +Add text at the end of the KQL query. This ways you can extend the query, add additional filters or rename certain fields. + +```yaml +AppendQuery: | + | extend TimeGenerated = StartTime +``` + +Adds the line to the end of the query and adds a new column named `TimeGenerated` based on value of the `StartTime` column. + +### ReplaceQueryVariables + +This section allows you to use variable names in your original YAML file. They will be replaced by the value provided in the parameter file. There is support for simple string replacement and arrays. + +All variables must be named using two percent sign at the beginning and the end e.g. `%%VARIABLENAME%%`. + +* String values are replaced as is. +* Array values are joined together using `","` and a single `"` is added at the start and end. The resulting string is used to replace the variable. + +```yaml +ReplaceQueryVariables: + NumberOfErrors: 200 + ErrorCodes: + - 403 + - 404 +``` + +* The variable `%%NumberOfErrors%%` will be replaced by the string value `200` +* Before the variable `%%ErrorCodes%%` will be replaced, the `ErrorCodes` array will be converted into a single string `"403","404"` + +This way the following KQL query will be converted... + +```kql +| where Message in (%%ErrorCodes%%) +| summarize StartTime = min(TimeGenerated), EndTime = max(TimeGenerated), NumberOfErrors = dcount(SourceIP) by HostName, SourceIP +| where NumberOfErrors > %%NumberOfErrors%% +``` +...to this result: + +```kql +| where Message in ("403","404") +| summarize StartTime = min(TimeGenerated), EndTime = max(TimeGenerated), NumberOfErrors = dcount(SourceIP) by HostName, SourceIP +| where NumberOfErrors > 200 +``` + ## Changelog +### 2.3.0 + * FEATURE: Add the option to specify a parameter file. This gives a maximum of flexbility to manipulate existing YAML files. + ### v2.2.3 * FEATURE: Add validation and auto-correction of invalid MITRE ATT&CK® tactics and techniques when converting YAML files to ARM templates