From 397038b04dfad1110d8f1a888a17fd467b9d346e Mon Sep 17 00:00:00 2001 From: ParthaI Date: Mon, 15 Jan 2024 18:46:42 +0530 Subject: [PATCH 1/2] Add table azure_monitor_log_profile Closes #712 --- .../test-get-expected.json | 7 + .../test-get-query.sql | 8 + .../test-list-expected.json | 6 + .../test-list-query.sql | 7 + .../test-not-found-expected.json | 1 + .../test-not-found-query.sql | 7 + .../test-turbot-expected.json | 7 + .../test-turbot-query.sql | 8 + .../azure_monitor_log_profile/variables.json | 1 + .../azure_monitor_log_profile/vatiables.tf | 106 +++++++++++ azure/plugin.go | 1 + azure/table_azure_monitor_log_profile.go | 168 ++++++++++++++++++ docs/tables/azure_monitor_log_profile.md | 155 ++++++++++++++++ 13 files changed, 482 insertions(+) create mode 100644 azure-test/tests/azure_monitor_log_profile/test-get-expected.json create mode 100644 azure-test/tests/azure_monitor_log_profile/test-get-query.sql create mode 100644 azure-test/tests/azure_monitor_log_profile/test-list-expected.json create mode 100644 azure-test/tests/azure_monitor_log_profile/test-list-query.sql create mode 100644 azure-test/tests/azure_monitor_log_profile/test-not-found-expected.json create mode 100644 azure-test/tests/azure_monitor_log_profile/test-not-found-query.sql create mode 100644 azure-test/tests/azure_monitor_log_profile/test-turbot-expected.json create mode 100644 azure-test/tests/azure_monitor_log_profile/test-turbot-query.sql create mode 100644 azure-test/tests/azure_monitor_log_profile/variables.json create mode 100644 azure-test/tests/azure_monitor_log_profile/vatiables.tf create mode 100644 azure/table_azure_monitor_log_profile.go create mode 100644 docs/tables/azure_monitor_log_profile.md diff --git a/azure-test/tests/azure_monitor_log_profile/test-get-expected.json b/azure-test/tests/azure_monitor_log_profile/test-get-expected.json new file mode 100644 index 00000000..8ef02d73 --- /dev/null +++ b/azure-test/tests/azure_monitor_log_profile/test-get-expected.json @@ -0,0 +1,7 @@ +[ + { + "id": "{{ output.resource_id_lower.value }}", + "name": "{{ resourceName }}", + "storage_account_id": "{{ output.storage_account_id.value }}" + } +] diff --git a/azure-test/tests/azure_monitor_log_profile/test-get-query.sql b/azure-test/tests/azure_monitor_log_profile/test-get-query.sql new file mode 100644 index 00000000..725aaf8d --- /dev/null +++ b/azure-test/tests/azure_monitor_log_profile/test-get-query.sql @@ -0,0 +1,8 @@ +select + name, + id, + storage_account_id +from + azure.azure_monitor_log_profile +where + name = '{{ resourceName }}' \ No newline at end of file diff --git a/azure-test/tests/azure_monitor_log_profile/test-list-expected.json b/azure-test/tests/azure_monitor_log_profile/test-list-expected.json new file mode 100644 index 00000000..8699da92 --- /dev/null +++ b/azure-test/tests/azure_monitor_log_profile/test-list-expected.json @@ -0,0 +1,6 @@ +[ + { + "id": "{{ output.resource_id_lower.value }}", + "name": "{{ resourceName }}" + } +] diff --git a/azure-test/tests/azure_monitor_log_profile/test-list-query.sql b/azure-test/tests/azure_monitor_log_profile/test-list-query.sql new file mode 100644 index 00000000..1e00eb06 --- /dev/null +++ b/azure-test/tests/azure_monitor_log_profile/test-list-query.sql @@ -0,0 +1,7 @@ +select + name, + id +from + azure.azure_monitor_log_profile +where + id = '{{ output.resource_id_lower.value }}'; \ No newline at end of file diff --git a/azure-test/tests/azure_monitor_log_profile/test-not-found-expected.json b/azure-test/tests/azure_monitor_log_profile/test-not-found-expected.json new file mode 100644 index 00000000..fe51488c --- /dev/null +++ b/azure-test/tests/azure_monitor_log_profile/test-not-found-expected.json @@ -0,0 +1 @@ +[] diff --git a/azure-test/tests/azure_monitor_log_profile/test-not-found-query.sql b/azure-test/tests/azure_monitor_log_profile/test-not-found-query.sql new file mode 100644 index 00000000..06194806 --- /dev/null +++ b/azure-test/tests/azure_monitor_log_profile/test-not-found-query.sql @@ -0,0 +1,7 @@ +select + name, + id +from + azure.azure_monitor_log_profile +where + name = 'dummy-{{ resourceName }}'; \ No newline at end of file diff --git a/azure-test/tests/azure_monitor_log_profile/test-turbot-expected.json b/azure-test/tests/azure_monitor_log_profile/test-turbot-expected.json new file mode 100644 index 00000000..976e8e84 --- /dev/null +++ b/azure-test/tests/azure_monitor_log_profile/test-turbot-expected.json @@ -0,0 +1,7 @@ +[ + { + "name": "{{ resourceName }}", + "subscription_id": "{{ output.subscription_id.value }}", + "title": "{{ resourceName }}" + } +] diff --git a/azure-test/tests/azure_monitor_log_profile/test-turbot-query.sql b/azure-test/tests/azure_monitor_log_profile/test-turbot-query.sql new file mode 100644 index 00000000..cbd30225 --- /dev/null +++ b/azure-test/tests/azure_monitor_log_profile/test-turbot-query.sql @@ -0,0 +1,8 @@ +select + name, + subscription_id, + title +from + azure.azure_monitor_log_profile +where + name = '{{ resourceName }}'; \ No newline at end of file diff --git a/azure-test/tests/azure_monitor_log_profile/variables.json b/azure-test/tests/azure_monitor_log_profile/variables.json new file mode 100644 index 00000000..0967ef42 --- /dev/null +++ b/azure-test/tests/azure_monitor_log_profile/variables.json @@ -0,0 +1 @@ +{} diff --git a/azure-test/tests/azure_monitor_log_profile/vatiables.tf b/azure-test/tests/azure_monitor_log_profile/vatiables.tf new file mode 100644 index 00000000..a2351d6e --- /dev/null +++ b/azure-test/tests/azure_monitor_log_profile/vatiables.tf @@ -0,0 +1,106 @@ +variable "resource_name" { + type = string + default = "turbot-test-20200125-create-update" + description = "Name of the resource used throughout the test." +} + +variable "azure_environment" { + type = string + default = "public" + description = "Azure environment used for the test." +} + +variable "azure_subscription" { + type = string + default = "3510ae4d-530b-497d-8f30-53b9616fc6c1" + description = "Azure subscription used for the test." +} + +provider "azurerm" { + # Cannot be passed as a variable + environment = var.azure_environment + subscription_id = var.azure_subscription + features {} +} + +data "azurerm_client_config" "current" {} + +data "null_data_source" "resource" { + inputs = { + scope = "azure:///subscriptions/${data.azurerm_client_config.current.subscription_id}" + } +} + +resource "azurerm_resource_group" "named_test_resource" { + name = var.resource_name + location = "East US" +} + +resource "azurerm_storage_account" "named_test_resource" { + name = var.resource_name + resource_group_name = azurerm_resource_group.named_test_resource.name + location = azurerm_resource_group.named_test_resource.location + account_tier = "Standard" + account_replication_type = "GRS" +} + +resource "azurerm_eventhub_namespace" "named_test_resource" { + name = var.resource_name + location = azurerm_resource_group.named_test_resource.location + resource_group_name = azurerm_resource_group.named_test_resource.name + sku = "Standard" + capacity = 2 +} + +resource "azurerm_monitor_log_profile" "named_test_resource" { + name = var.resource_name + + categories = [ + "Action", + "Delete", + "Write", + ] + + locations = [ + "eastus", + "global", + ] + + # RootManageSharedAccessKey is created by default with listen, send, manage permissions + servicebus_rule_id = "${azurerm_eventhub_namespace.named_test_resource.id}/authorizationrules/RootManageSharedAccessKey" + storage_account_id = azurerm_storage_account.named_test_resource.id + + retention_policy { + enabled = true + days = 7 + } +} + +output "resource_aka" { + value = "azure://${azurerm_monitor_log_profile.named_test_resource.id}" +} + +output "resource_aka_lower" { + value = "azure://${lower(azurerm_monitor_log_profile.named_test_resource.id)}" +} + +output "resource_name" { + value = var.resource_name +} + +output "resource_id_lower" { + value = lower(azurerm_monitor_log_profile.named_test_resource.id) +} + +output "storage_account_id" { + value = azurerm_storage_account.named_test_resource.id +} + +output "subscription_id" { + value = var.azure_subscription +} + +output "location" { + value = azurerm_resource_group.named_test_resource.location +} + diff --git a/azure/plugin.go b/azure/plugin.go index 15c2d62d..714143bd 100644 --- a/azure/plugin.go +++ b/azure/plugin.go @@ -121,6 +121,7 @@ func Plugin(ctx context.Context) *plugin.Plugin { "azure_management_lock": tableAzureManagementLock(ctx), "azure_mariadb_server": tableAzureMariaDBServer(ctx), "azure_monitor_activity_log_event": tableAzureMonitorActivityLogEvent(ctx), + "azure_monitor_log_profile": tableAzureMonitorLogProfile(ctx), "azure_mssql_elasticpool": tableAzureMSSQLElasticPool(ctx), "azure_mssql_managed_instance": tableAzureMSSQLManagedInstance(ctx), "azure_mssql_virtual_machine": tableAzureMSSQLVirtualMachine(ctx), diff --git a/azure/table_azure_monitor_log_profile.go b/azure/table_azure_monitor_log_profile.go new file mode 100644 index 00000000..9588ecf7 --- /dev/null +++ b/azure/table_azure_monitor_log_profile.go @@ -0,0 +1,168 @@ +package azure + +import ( + "context" + + "github.com/turbot/steampipe-plugin-sdk/v5/grpc/proto" + "github.com/turbot/steampipe-plugin-sdk/v5/plugin" + "github.com/turbot/steampipe-plugin-sdk/v5/plugin/transform" + + "github.com/Azure/azure-sdk-for-go/profiles/2020-09-01/monitor/mgmt/insights" +) + +//// TABLE DEFINITION + +func tableAzureMonitorLogProfile(_ context.Context) *plugin.Table { + return &plugin.Table{ + Name: "azure_monitor_log_profile", + Description: "Azure Monitor Log Profile", + Get: &plugin.GetConfig{ + KeyColumns: plugin.AllColumns([]string{"name"}), + Hydrate: getMonitorLogProfile, + IgnoreConfig: &plugin.IgnoreConfig{ + ShouldIgnoreErrorFunc: isNotFoundError([]string{"ResourceNotFound", "404"}), + }, + }, + List: &plugin.ListConfig{ + Hydrate: listMonitorLogProfiles, + }, + Columns: azureColumns([]*plugin.Column{ + { + Name: "id", + Description: "Azure resource Id.", + Type: proto.ColumnType_STRING, + Transform: transform.FromGo(), + }, + { + Name: "name", + Description: "Azure resource name.", + Type: proto.ColumnType_STRING, + }, + { + Name: "type", + Description: "Azure resource type.", + Type: proto.ColumnType_STRING, + }, + { + Name: "location", + Description: "The resource location.", + Type: proto.ColumnType_STRING, + }, + { + Name: "storage_account_id", + Description: "The resource id of the storage account to which you would like to send the Activity Log.", + Type: proto.ColumnType_STRING, + Transform: transform.FromField("LogProfileProperties.StorageAccountID"), + }, + { + Name: "service_bus_rule_id", + Description: "The service bus rule ID of the service bus namespace in which you would like to have Event Hubs created for streaming the Activity Log.", + Type: proto.ColumnType_STRING, + Transform: transform.FromField("LogProfileProperties.ServiceBusRuleID"), + }, + { + Name: "locations", + Description: "List of regions for which Activity Log events should be stored or streamed. It is a comma separated list of valid ARM locations including the 'global' location.", + Type: proto.ColumnType_JSON, + Transform: transform.FromField("LogProfileProperties.Locations"), + }, + { + Name: "categories", + Description: "The categories of the logs. These categories are created as is convenient to the user.", + Type: proto.ColumnType_JSON, + Transform: transform.FromField("LogProfileProperties.Categories"), + }, + { + Name: "retention_policy", + Description: "The retention policy for the events in the log.", + Type: proto.ColumnType_JSON, + Transform: transform.FromField("LogProfileProperties.RetentionPolicy"), + }, + + // Steampipe standard columns + { + Name: "title", + Description: ColumnDescriptionTitle, + Type: proto.ColumnType_STRING, + Transform: transform.FromField("Name"), + }, + { + Name: "tags", + Description: ColumnDescriptionTags, + Type: proto.ColumnType_JSON, + }, + + // Azure standard columns + { + Name: "akas", + Description: ColumnDescriptionAkas, + Type: proto.ColumnType_STRING, + Transform: transform.FromField("ID").Transform(idToAkas), + }, + }), + } +} + +//// LIST FUNCTION + +func listMonitorLogProfiles(ctx context.Context, d *plugin.QueryData, _ *plugin.HydrateData) (interface{}, error) { + session, err := GetNewSession(ctx, d, "MANAGEMENT") + if err != nil { + plugin.Logger(ctx).Error("azure_monitor_log_profile.listMonitorLogProfiles", "session_error", err) + return nil, err + } + subscriptionID := session.SubscriptionID + + client := insights.NewLogProfilesClientWithBaseURI(session.ResourceManagerEndpoint, subscriptionID) + client.Authorizer = session.Authorizer + + // API doesn't support pagination + result, err := client.List(ctx) + if err != nil { + plugin.Logger(ctx).Error("azure_monitor_log_profile.listMonitorLogProfiles", "api_error", err) + return nil, err + } + + for _, profile := range *result.Value { + d.StreamListItem(ctx, profile) + + // Check if context has been cancelled or if the limit has been hit (if specified) + // if there is a limit, it will return the number of rows required to reach this limit + if d.RowsRemaining(ctx) == 0 { + return nil, nil + } + } + + return nil, err +} + +//// HYDRATE FUNCTIONS + +func getMonitorLogProfile(ctx context.Context, d *plugin.QueryData, h *plugin.HydrateData) (interface{}, error) { + + name := d.EqualsQuals["name"].GetStringValue() + + session, err := GetNewSession(ctx, d, "MANAGEMENT") + if err != nil { + plugin.Logger(ctx).Error("azure_monitor_log_profile.getMonitorLogProfile", "session_error", err) + return nil, err + } + subscriptionID := session.SubscriptionID + + client := insights.NewLogProfilesClientWithBaseURI(session.ResourceManagerEndpoint, subscriptionID) + client.Authorizer = session.Authorizer + + op, err := client.Get(ctx, name) + if err != nil { + plugin.Logger(ctx).Error("azure_monitor_log_profile.getMonitorLogProfile", "api_error", err) + return nil, err + } + + // In some cases resource does not give any notFound error + // instead of notFound error, it returns empty data + if op.ID != nil { + return op, nil + } + + return nil, nil +} diff --git a/docs/tables/azure_monitor_log_profile.md b/docs/tables/azure_monitor_log_profile.md new file mode 100644 index 00000000..003ea1e1 --- /dev/null +++ b/docs/tables/azure_monitor_log_profile.md @@ -0,0 +1,155 @@ +--- +title: "Steampipe Table: azure_monitor_log_profile - Query Azure Monitor Log Profiles using SQL" +description: "Allows users to query Monitor Log Profiles in Azure Monitor, providing insights into the log profile." +--- + +# Table: azure_monitor_log_profile - Query Azure Monitor Log Profiles using SQL + +Azure Monitor Log Profile is a configuration in Azure Monitor that specifies how activity logs are collected and retained. These profiles are essential for managing and controlling the export of Azure activity logs, which include logs related to resource usage, service health, and operations within a Azure subscription. By setting up a Log Profile, administrators can define where these logs are stored, how long they are retained, and can ensure that they have access to historical data for compliance, auditing, and troubleshooting purposes. + +## Table Usage Guide + +The `azure_monitor_log_profile` table provides insights into logs related to resource usage, service health, and operations within a Azure subscription. By setting up a Log Profile, administrators can define where these logs are stored, how long they are retained, and can ensure that they have access to historical data for compliance, auditing, and troubleshooting purposes. + +## Examples + +### Basic info +Explore the quite useful for managing and understanding Azure Monitor Log Profiles. It selects key attributes of log profiles, which are crucial for monitoring and auditing purposes in Azure environments. + +```sql+postgres +select + id, + name, + storage_account_id, + service_bus_rule_id, + locations, + retention_policy +from + azure_monitor_log_profile; +``` + +```sql+sqlite +select + id, + name, + storage_account_id, + service_bus_rule_id, + locations, + retention_policy +from + azure_monitor_log_profile; +``` + +### List events with event-level critical +This example helps identify critical events in your Azure activity log. By doing so, it allows you to promptly respond to potential issues or security threats. + +```sql+postgres +select + event_name, + id, + operation_name, + event_timestamp, + level, + caller +from + azure_monitor_log_profile +where + level = 'EventLevelCritical'; +``` + +```sql+sqlite +select + event_name, + id, + operation_name, + event_timestamp, + level, + caller +from + azure_monitor_log_profile +where + level = 'EventLevelCritical'; +``` + +### Get retention policy details of log profiles + The query helps in efficiently tracking and managing log retention settings, ensuring that data retention complies with organizational policies and regulatory requirements. + +```sql+postgres +select + id, + name, + retention_policy -> 'Enabled' as retention_policy_enabled, + retention_policy -> 'Days' as retention_policy_days +from + azure_monitor_log_profile; +``` + +```sql+sqlite +select + id, + name, + json_extract(retention_policy, '$.Enabled') as retention_policy_enabled, + json_extract(retention_policy, '$.Days') as retention_policy_days +from + azure_monitor_log_profile; +``` + +### Get the location for which Activity Log events should be stored +Retrieve the specific locations associated with each log profile to understand where log data is being accumulated. + +```sql+postgres +select + p.name, + p.id, + p.storage_account_id, + l as location +from + azure_monitor_log_profile as p, + jsonb_array_elements_text(locations) as l; +``` + +```sql+sqlite +select + p.name, + p.id, + p.storage_account_id, + json_each.value as location +from + azure_monitor_log_profile as p, + json_each(p.locations); +``` + +### Get storage account details associated with the log profile +Highly beneficial for organizations using Azure services, as it helps in assessing the configuration and security aspects of their storage solutions linked with log profiles. By retrieving data such as the storage account's name, type, access tier, and various security and feature enablements like HTTPS traffic only, blob change feed, container soft delete, and encryption key sources, administrators + +```sql+postgres +select + l.name, + l.type, + s.access_tier, + s.kind, + s.blob_change_feed_enabled, + s.blob_container_soft_delete_enabled, + s.enable_https_traffic_only, + s.encryption_key_source +from + azure_monitor_log_profile as l, + azure_storage_account as s +where + l.storage_account_id = s.id +``` + +```sql+sqlite +select + l.name, + l.type, + s.access_tier, + s.kind, + s.blob_change_feed_enabled, + s.blob_container_soft_delete_enabled, + s.enable_https_traffic_only, + s.encryption_key_source +from + azure_monitor_log_profile as l + join azure_storage_account as s on l.storage_account_id = s.id; +``` \ No newline at end of file From 8fa5c2038bf9a823811a2626c7340e4749dab13e Mon Sep 17 00:00:00 2001 From: ParthaI Date: Mon, 15 Jan 2024 18:51:06 +0530 Subject: [PATCH 2/2] Re-arrange the columns --- azure/table_azure_monitor_log_profile.go | 2 -- 1 file changed, 2 deletions(-) diff --git a/azure/table_azure_monitor_log_profile.go b/azure/table_azure_monitor_log_profile.go index 9588ecf7..9171bb0f 100644 --- a/azure/table_azure_monitor_log_profile.go +++ b/azure/table_azure_monitor_log_profile.go @@ -91,8 +91,6 @@ func tableAzureMonitorLogProfile(_ context.Context) *plugin.Table { Description: ColumnDescriptionTags, Type: proto.ColumnType_JSON, }, - - // Azure standard columns { Name: "akas", Description: ColumnDescriptionAkas,