feat: Added support for Log filtering (using new input log_filters) and support for enabling annotation filtering (using new input enable_annotations) (#152)

* feat: added option to apply filters on logs

* Update variables.tf

* Update variables.tf

* Update variables.tf

* fixed precommit

* added documentation for log_filters variable

* added variable in DA

* precommit fix

* exposed enable_annotations variable in DA

* Update variables.tf

* Update variables.tf

* resolved comments

* fixed precommit error

Author: vkuma17

README.md

@@ -117,8 +117,10 @@ No modules.
 | <a name="input_cluster_config_endpoint_type"></a> [cluster\_config\_endpoint\_type](#input\_cluster\_config\_endpoint\_type) | The type of endpoint to use for the cluster config access: `default`, `private`, `vpe`, or `link`. The `default` value uses the default endpoint of the cluster. | `string` | `"default"` | no |
 | <a name="input_cluster_id"></a> [cluster\_id](#input\_cluster\_id) | The ID of the cluster to deploy the agent. | `string` | n/a | yes |
 | <a name="input_cluster_resource_group_id"></a> [cluster\_resource\_group\_id](#input\_cluster\_resource\_group\_id) | The resource group ID of the cluster. | `string` | n/a | yes |
+| <a name="input_enable_annotations"></a> [enable\_annotations](#input\_enable\_annotations) | Set to true to include pod annotations in log records. Default annotations such as pod IP address and container ID, along with any custom annotations on the pod, will be included. This can help filter logs based on pod annotations in Cloud Logs. | `bool` | `false` | no |
 | <a name="input_enable_multiline"></a> [enable\_multiline](#input\_enable\_multiline) | Enable or disable multiline log support. [Learn more](https://cloud.ibm.com/docs/cloud-logs?topic=cloud-logs-agent-multiline) | `bool` | `false` | no |
 | <a name="input_is_vpc_cluster"></a> [is\_vpc\_cluster](#input\_is\_vpc\_cluster) | Specify true if the target cluster for the agent is a VPC cluster, false if it is a classic cluster. | `bool` | `true` | no |
+| <a name="input_log_filters"></a> [log\_filters](#input\_log\_filters) | List of additional filters to be applied on logs. [Learn more](https://github.com/terraform-ibm-modules/terraform-ibm-logs-agent/blob/main/solutions/fully-configurable/DA-types.md#configuring-log-filters). | `any` | `[]` | no |
 | <a name="input_logs_agent_additional_metadata"></a> [logs\_agent\_additional\_metadata](#input\_logs\_agent\_additional\_metadata) | The list of additional metadata fields to add to the routed logs. | <pre>list(object({<br/>    key   = optional(string)<br/>    value = optional(string)<br/>  }))</pre> | `[]` | no |
 | <a name="input_logs_agent_chart"></a> [logs\_agent\_chart](#input\_logs\_agent\_chart) | The name of the Helm chart to deploy. | `string` | `"logs-agent-helm"` | no |
 | <a name="input_logs_agent_chart_location"></a> [logs\_agent\_chart\_location](#input\_logs\_agent\_chart\_location) | The location of the Logs agent helm chart. | `string` | `"oci://icr.io/ibm/observe"` | no |

examples/logs-agent-iks/main.tf

@@ -153,4 +153,27 @@ module "logs_agent" {
   cloud_logs_ingress_endpoint = module.cloud_logs.ingress_private_endpoint
   cloud_logs_ingress_port     = 3443
   logs_agent_enable_scc       = false # only true for Openshift
+  log_filters = [
+    {
+      name  = "lua"
+      match = "*"
+      call  = "add_crn_field"
+      code  = <<EOL
+      -- Enrich records with CRN field
+      -- This is just a sample code for showing usage of lua filter
+      function add_crn_field(tag, timestamp, record)
+
+          record["logSourceCRN"] = "crn:v1:bluemix:public:postgres:us-south:a/123456::"
+          record["saveServiceCopy"] = "true"
+
+          return 2, 0, record
+      end
+      EOL
+    },
+    {
+      name    = "grep"
+      match   = "*"
+      exclude = ["message.level debug"]
+    }
+  ]
 }

ibm_catalog.json

@@ -225,6 +225,12 @@
                 }
               }
             },
+            {
+              "key": "enable_annotations"
+            },
+            {
+              "key": "log_filters"
+            },
             {
               "key": "logs_agent_selected_log_source_paths"
             },

main.tf

@@ -115,10 +115,16 @@ resource "helm_release" "logs_agent" {
     value = var.enable_multiline
   }
 
+  set {
+    name  = "includeAnnotations"
+    value = var.enable_annotations
+  }
+
   # dummy value hack to force update https://github.com/hashicorp/terraform-provider-helm/issues/515#issuecomment-813088122
   values = [
     yamlencode({
       tolerations        = var.logs_agent_tolerations
+      additionalFilters  = var.log_filters
       resources          = var.logs_agent_resources
       additionalMetadata = local.logs_agent_additional_metadata
       dummy              = uuid()

solutions/fully-configurable/DA-types.md

@@ -108,3 +108,47 @@ In this example:
 ### What It Does
 
 The `logs_agent_resources` variable is used to configure the resource requests and limits for the logs agent pods. This ensures that the logs agent has sufficient resources to operate efficiently while preventing it from consuming excessive resources on the node. The configuration is passed to the Helm chart during deployment, ensuring that the specified resource constraints are applied.
+
+===================================================================
+
+# Configuring Log filters
+
+:exclamation: **Important:** `log_filters` variable should be passed as a list to the input even if there is only one filter which you want to apply.
+
+`log_filters` can be used to apply some filters on the logs which you want to send to Cloud Logs instance such as modifying some fields, dropping info/debug logs to just keep error logs, or running some custom functions with the help of lua filter. For more information please refer [this](https://docs.fluentbit.io/manual/data-pipeline/filters)
+
+### Example `log_filters` Usage
+
+To configure additional filters on logs, you can set the `log_filters` variable in below format.
+
+```hcl
+[
+    {
+      name  = "lua"
+      match = "*"
+      call  = "add_crn_field"
+      code  = <<EOL
+      -- Enrich records with CRN field
+      -- This is just a sample code for showing usage of lua filter
+      function add_crn_field(tag, timestamp, record)
+
+          record["logSourceCRN"] = "crn:v1:bluemix:public:postgres:us-south:a/123456::"
+          record["saveServiceCopy"] = "true"
+
+          return 2, 0, record
+      end
+      EOL
+    },
+    {
+      name  = "grep"
+      match = "*"
+      exclude = ["message.level debug"]
+    }
+]
+```
+
+In this example:
+- First filter in the list is a lua filter with which you can run a custom code on your logs. Here an extra logSourceCRN field is getting appended in the log record.
+- Second filter in the list is grep filter which is dropping all the records which have message.level field set to debug in the log record.
+
+We can use other filters such as modify, nest also.

solutions/fully-configurable/main.tf

@@ -44,4 +44,6 @@ module "logs_agent" {
   wait_till                            = var.wait_till
   wait_till_timeout                    = var.wait_till_timeout
   enable_multiline                     = var.enable_multiline
+  enable_annotations                   = var.enable_annotations
+  log_filters                          = var.log_filters
 }

solutions/fully-configurable/variables.tf

@@ -165,6 +165,23 @@ variable "logs_agent_log_source_namespaces" {
   nullable    = false
 }
 
+variable "enable_annotations" {
+  description = "Set to true to include pod annotations in log records. Default annotations such as pod IP address and container ID, along with any custom annotations on the pod, will be included. This can help filter logs based on pod annotations in Cloud Logs."
+  type        = bool
+  default     = false
+}
+
+variable "log_filters" {
+
+  # variable type is any because filters schema is not fixed and there are many filters each having its unique fields.
+  # logs-agent helm chart expects this variable to be provided in list format even if a single filter is passed.
+
+  description = "List of additional filters to be applied on logs. [Learn more](https://github.com/terraform-ibm-modules/terraform-ibm-logs-agent/blob/main/solutions/fully-configurable/DA-types.md#configuring-log-filters)."
+  type        = any
+  default     = []
+}
+
+
 variable "logs_agent_iam_mode" {
   type        = string
   default     = "TrustedProfile"

variables.tf

@@ -233,8 +233,26 @@ variable "cloud_logs_ingress_port" {
     condition     = contains([3443, 443], var.cloud_logs_ingress_port)
   }
 }
+
 variable "enable_multiline" {
   description = "Enable or disable multiline log support. [Learn more](https://cloud.ibm.com/docs/cloud-logs?topic=cloud-logs-agent-multiline)"
   type        = bool
   default     = false
 }
+
+variable "enable_annotations" {
+  description = "Set to true to include pod annotations in log records. Default annotations such as pod IP address and container ID, along with any custom annotations on the pod, will be included. This can help filter logs based on pod annotations in Cloud Logs."
+  type        = bool
+  default     = false
+}
+
+
+variable "log_filters" {
+
+  # variable type is any because filters schema is not fixed and there are many filters each having its unique fields.
+  # logs-agent helm chart expects this variable to be provided in list format even if a single filter is passed.
+
+  description = "List of additional filters to be applied on logs. [Learn more](https://github.com/terraform-ibm-modules/terraform-ibm-logs-agent/blob/main/solutions/fully-configurable/DA-types.md#configuring-log-filters)."
+  type        = any
+  default     = []
+}