feat: 🎸 add lm-logs to umbrella chart (#115)

charts/lm-logs/.helmignore

@@ -0,0 +1,23 @@
+# Patterns to ignore when building packages.
+# This supports shell glob matching, relative path matching, and
+# negation (prefixed with !). Only one pattern per line.
+.DS_Store
+# Common VCS dirs
+.git/
+.gitignore
+.bzr/
+.bzrignore
+.hg/
+.hgignore
+.svn/
+# Common backup files
+*.swp
+*.bak
+*.tmp
+*.orig
+*~
+# Various IDEs
+.project
+.idea/
+*.tmproj
+.vscode/

charts/lm-logs/Chart.yaml

@@ -0,0 +1,10 @@
+apiVersion: v2
+description: A Helm chart for sending k8s logs to Logic Monitor
+name: lm-logs
+icon: https://logicmonitor.github.io/helm-charts/lm_logo.png
+version: 0.4.0
+maintainers:
+  - email: [email protected]
+    name: LogicMonitor
+appVersion: 1.0.5
+home: https://logicmonitor.github.io/helm-charts

charts/lm-logs/Dockerfile

@@ -0,0 +1,4 @@
+FROM fluent/fluentd-kubernetes-daemonset:v1.16-debian-forward-1
+USER root
+RUN gem install fluent-plugin-lm-logs -v 1.0.5
+RUN gem install fluent-plugin-multi-format-parser -v 1.0.0

charts/lm-logs/README.md

@@ -0,0 +1,78 @@
+## Send Kubernetes logs to LM Logs
+
+This chart deploy a fluentd based daemonset to collect and forward logs to LogicMonitor
+#### Prerequisite
+- LogicMonitor Collector [installed and monitoring your Kubernetes Cluster](https://www.logicmonitor.com/support/monitoring/containers/kubernetes/adding-your-kubernetes-cluster-into-monitoring).
+
+#### Deploy
+
+Install the lm-logs chart, filling in the required values.
+
+``` console
+helm install -n <namespace> \
+--set lm_company_name="<lm_company_name>" \
+--set lm_access_id="<lm_access_id>" \
+--set lm_access_key="<lm_access_key>" \
+lm-logs logicmonitor/lm-logs
+```
+
+#### Parameters
+The following tables lists the configurable parameters of the lm-logs chart and their default values.
+| Parameter                   | Description                                     | Default                                                 |
+|-----------------------------|-------------------------------------------------|---------------------------------------------------------|
+| `global.imagePullSecrets`   | List of global registry secret names            | `[]` (does not add image pull secrets to deployed pods) |
+| `global.nameOverride`       | Global storage class for dynamic provisioning   | `""`                                                    |
+| `global.fullnameOverride`   | Global storage class for dynamic provisioning   | `""`                                                    |
+| `global.lm_company_name`    | LogicMonitor account name                       | `nil`                                                   |
+| `global.lm_access_id`       | LogicMonitor API Token Access ID                | `nil`                                                   |
+| `global.lm_access_key`      | LogicMonitor API Token Access Key               | `nil`                                                   |
+| `image.repository`          | Container image repository                      | `logicmonitor/lm-logs-k8s-fluentd`                      |
+| `image.pullPolicy`          | Container image pull policy                     | `IfNotPresent`                                          |
+| `image.tag`                 | Container image tag                             | `""`                                                    |
+| `resources.limits.memory`   | Container memory resource limit                 | `1000Mi`                                                |
+| `resources.requests.cpu.`   | Container cpu resource requests                 | `300m`                                                  |
+| `resources.requests.memory` | Container memory resource requests              | `700Mi`                                                 |
+| `fluent.device_less_logs`   | beta feature. when set true, do not send resource information. send `service` and `namespace` as metadata when true  | `false`                                                 |
+| `fluent.include_metadata`   | if true send all metadata along with log msg    | `true`                                                  |
+| `fluent.buffer.memory`      | fluentd's buffer memory plugin config           | `flush_interval 1s,chunk_limit_size 8m,flush_thread_count 8`|
+| `tolerations`               | Tolerations for pod assignment	                | `{}`  (evaluated as a template)                         |
+| `nodeSelectors`             | Node labels for pod assignment		            | `{}`  (evaluated as a template)                         |
+| `affinity`                  | Affinity for pod assignment		                | `{}`  (evaluated as a template)                         |
+| `env`                       | Map to add extra environment variables	        | `{}`                                                    |
+| `kubernetes.multiline_start_regexp` | Regexp to match beginning of multiline	| `/^\[(\d{4}-)?\d{2}-\d{2} \d{2}:\d{2}:\d{2}.\d{3}.*\]/` |
+| `kubernetes.cluster_name`       | ClusterName given while adding k8s cluster  	| `""`                                                    |
+
+### Avaialble Environment variables
+For descriptions see: https://github.com/fabric8io/fluent-plugin-kubernetes_metadata_filter
+
+* FLUENT_LOG_LEVEL
+* FLUENT_CONTAINER_TAIL_TAG
+* FLUENT_CONTAINER_TAIL_EXCLUDE_PATH
+* FLUENT_CONTAINER_TAIL_PARSER_TYPE
+* FLUENT_FILTER_KUBERNETES_URL
+* KUBERNETES_VERIFY_SSL
+* KUBERNETES_CA_FILE
+* FLUENT_KUBERNETES_METADATA_SKIP_LABELS
+* FLUENT_KUBERNETES_METADATA_SKIP_CONTAINER_METADATA
+* FLUENT_KUBERNETES_METADATA_SKIP_MASTER_URL
+* FLUENT_KUBERNETES_METADATA_SKIP_NAMESPACE_METADATA
+
+#### New deviceless logs k8s integration (beta)
+Note: This feature may not be available to all customers.
+To enable this feature set `fluent.device_less_logs=true`
+##### Problems with current integration
+- k8s pods are ephemeral, anomaly detection is done per pod. Which creates high volume of anomalies and makes it hard to monitor.
+- When new pod is created, argus takes time to register it as LM Device and resource mapping fails until then. Consequently we miss initial pod logs.
+
+When `fluent.device_less_logs=true`
+Anomaly detection will be done on `namespace` and `service`
+- namespace will be k8s namespace
+- service will be extracted from metadata for yaml in the following priority.
+    - kubernetets.labels.app ( deployments )
+    - kubernetets.labels.app_kubernetes_io/name (daemon sets)
+    - kubernetets.container_name
+    - kubernetets.pod_name
+
+#### Multiline log support for k8s lm logs
+To use regexp to match beginning of multiline set `kubernetes.multiline_start_regexp=<some-regex-pattern>`
+by default the regex is set to `/^\[(\d{4}-)?\d{2}-\d{2} \d{2}:\d{2}:\d{2}.\d{3}.*\]/`

charts/lm-logs/ci/base-sanity-values.yaml

@@ -0,0 +1,3 @@
+lm_company_name: "dummy_company"
+lm_access_id: "dummy_id"
+lm_access_key: "dummy_key"

charts/lm-logs/templates/_helpers.tpl

@@ -0,0 +1,119 @@
+{{/*
+Expand the name of the chart.
+*/}}
+{{- define "fluentd.name" -}}
+{{- default .Chart.Name .Values.nameOverride | trunc 63 | trimSuffix "-" }}
+{{- end }}
+
+{{/*
+Create a default fully qualified app name.
+We truncate at 63 chars because some Kubernetes name fields are limited to this (by the DNS naming spec).
+If release name contains chart name it will be used as a full name.
+*/}}
+{{- define "fluentd.fullname" -}}
+{{- if .Values.fullnameOverride }}
+{{- .Values.fullnameOverride | trunc 63 | trimSuffix "-" }}
+{{- else }}
+{{- $name := default .Chart.Name .Values.nameOverride }}
+{{- if contains $name .Release.Name }}
+{{- .Release.Name | trunc 63 | trimSuffix "-" }}
+{{- else }}
+{{- printf "%s-%s" .Release.Name $name | trunc 63 | trimSuffix "-" }}
+{{- end }}
+{{- end }}
+{{- end }}
+
+{{/*
+Adding validations for clustername for lm-logs to contain only lower alphanumeric or '-' and start and end with an alphanumeric character
+*/}}
+{{- define "kubernetes.cluster_name" -}}
+{{- $cluster := "" -}}
+{{- if .Values.kubernetes.cluster_name -}}
+{{- $cluster = .Values.kubernetes.cluster_name -}}
+{{- else if .Values.global.clusterName -}}
+{{- $cluster = .Values.global.clusterName -}}
+{{- end -}}
+{{- if ne $cluster "" -}}
+{{- if regexMatch "^[a-z0-9][a-z0-9-]*[a-z0-9]$" $cluster }}
+kubernetes.cluster_name {{ $cluster }}
+{{- else -}}
+{{- fail "cluster_name should contain only lower alphanumeric or '-' and should start and end with an alphanumeric character" }}
+{{- end }}
+{{- end }}
+{{- end }}
+
+{{/*
+Create chart name and version as used by the chart label.
+*/}}
+{{- define "fluentd.chart" -}}
+{{- printf "%s-%s" .Chart.Name .Chart.Version | replace "+" "_" | trunc 63 | trimSuffix "-" }}
+{{- end }}
+
+{{/*
+Common labels
+*/}}
+{{- define "fluentd.labels" -}}
+helm.sh/chart: {{ include "fluentd.chart" . }}
+app.kubernetes.io/component: lm-logs-agent
+app.kubernetes.io/part-of: {{ template "fluentd.name" . }}
+{{ include "fluentd.selectorLabels" . }}
+{{- if .Chart.AppVersion }}
+app.kubernetes.io/version: {{ .Chart.AppVersion | quote }}
+{{- end }}
+app.kubernetes.io/managed-by: {{ .Release.Service }}
+{{- if .Values.labels }}
+{{ toYaml .Values.labels }}
+{{- end }}
+{{- end }}
+
+{{/*
+Selector labels
+*/}}
+{{- define "fluentd.selectorLabels" -}}
+app.kubernetes.io/name: {{ include "fluentd.name" . }}
+app.kubernetes.io/instance: {{ .Release.Name }}
+{{- end }}
+
+{{/*
+Common Annotations
+*/}}
+{{- define "fluentd.annotations" -}}
+logicmonitor.com/provider: lm-container
+{{- if .Values.annotations }}
+{{ toYaml .Values.annotations }}
+{{- end }}
+{{- end }}
+
+
+{{/*
+Create the name of the service account to use
+*/}}
+{{- define "fluentd.serviceAccountName" -}}
+{{- default (include "fluentd.fullname" .)}}
+{{- end }}
+
+{{/*
+Return the appropriate apiVersion for rbac.
+*/}}
+{{- define "rbac.apiVersion" -}}
+{{- if .Capabilities.APIVersions.Has "rbac.authorization.k8s.io/v1" }}
+{{- print "rbac.authorization.k8s.io/v1" -}}
+{{- else -}}
+{{- print "rbac.authorization.k8s.io/v1beta1" -}}
+{{- end -}}
+{{- end -}}
+
+{{- define "ds-env" -}}
+{{- $envList := list  -}}
+{{- range $key, $val := .Values.env -}}
+{{- $envProps := dict -}}
+{{- $_ := set $envProps "name" $key -}}
+{{- $_ = set $envProps "value" ($val) -}}
+{{- $envList = append $envList $envProps -}}
+{{- end -}}
+{{- toYaml $envList | nindent 0 -}}
+{{- end -}}
+
+{{- define "fluentd-image" -}}
+"{{ .Values.image.repository }}:{{ .Values.image.tag | default .Chart.AppVersion }}"
+{{- end -}}

charts/lm-logs/templates/configmap.yaml

@@ -0,0 +1,99 @@
+kind: ConfigMap
+apiVersion: v1
+metadata:
+  name: {{ include "fluentd.fullname" . }}
+  labels:
+    {{- include "fluentd.labels" . | nindent 4 }}
+  annotations:
+    {{- include "fluentd.annotations" . | nindent 4 }}
+data:
+  fluent.conf: |
+    @include kubernetes.conf
+
+    <system>
+      log_level "#{ENV['FLUENT_LOG_LEVEL'] || 'warn'}"
+    </system>
+
+    <filter kubernetes.**>
+      @type record_transformer
+      enable_ruby
+      <record>
+        message ${record["log"]} ${record["message"]}
+        timestamp ${record["time"]}
+        {{- if or .Values.kubernetes.cluster_name .Values.global.clusterName }}
+        {{ include "kubernetes.cluster_name" . | nindent 8 }}
+        {{- end}}
+        {{- if  .Values.fluent.device_less_logs }}
+        resource.service.name ${record.dig("kubernetes","labels","app") != nil ? record.dig("kubernetes","labels","app") : record.dig("kubernetes","labels","app_kubernetes_io/name") != nil ? record.dig("kubernetes","labels","app_kubernetes_io/name") :  record.dig("kubernetes","container_name") != nil ?  record.dig("kubernetes","container_name") : record.dig("kubernetes","pod_name") != nil ? record.dig("kubernetes","pod_name") :  "unknown" }
+        resource.service.namespace ${record["kubernetes"]["namespace_name"]}
+        {{- end}}
+      </record>
+      remove_keys log
+    </filter>
+
+    <match kubernetes.**>
+      @type lm
+      company_name {{ if .Values.lm_company_name }} {{ .Values.lm_company_name }} {{ else }} {{ required "A valid .Values.lm_company_name or .Values.global.account entry is required!" .Values.global.account }} {{ end }}
+      resource_mapping {"kubernetes.pod_name": "auto.name"}
+      access_id {{ .Values.lm_access_id | default .Values.global.accessID }}
+      access_key {{ .Values.lm_access_key | default .Values.global.accessKey }}
+      debug false
+      compression gzip
+      include_metadata {{ hasKey .Values.fluent "include_metadata" | ternary .Values.fluent.include_metadata true }}
+      device_less_logs {{ .Values.fluent.device_less_logs | default false }}
+      <buffer>
+        @type memory
+        flush_interval {{ .Values.fluent.buffer.memory.flush_interval | default "1s" }}
+        chunk_limit_size {{ .Values.fluent.buffer.memory.chunk_limit_size | default "8m" }}
+        flush_thread_count {{ .Values.fluent.buffer.memory.flush_thread_count | default "8"}}
+      </buffer>
+    </match>
+  kubernetes.conf: |
+    <source>
+      @type tail
+      @id in_tail_container_logs
+      path /var/log/containers/*.log
+      pos_file /var/log/fluentd-containers.log.pos
+      tag "#{ENV['FLUENT_CONTAINER_TAIL_TAG'] || 'kubernetes.*'}"
+      exclude_path "#{ENV['FLUENT_CONTAINER_TAIL_EXCLUDE_PATH'] || '/var/log/containers/{{ .Chart.Name }}*.log' }"
+      <parse>
+        @type "#{ENV['FLUENT_CONTAINER_TAIL_PARSER_TYPE'] || 'multi_format'}"
+        <pattern>
+          format json
+          keep_time_key true
+          time_format %Y-%m-%dT%H:%M:%S.%NZ
+        </pattern>
+        <pattern>
+          format syslog
+        </pattern>
+        <pattern>
+          format none
+        </pattern>
+      </parse>
+    </source>
+
+    <filter kubernetes.**>
+      @type concat
+      key log
+      seperator ""
+      multiline_start_regexp {{ .Values.kubernetes.multiline_start_regexp }}
+      timeout_label @NORMAL
+    </filter>
+
+    <label @NORMAL>
+      <match kubernetes.**>
+        @type stdout
+      </match>
+    </label>
+
+    <filter kubernetes.**>
+      @type kubernetes_metadata
+      @id filter_kube_metadata
+      kubernetes_url "#{ENV['FLUENT_FILTER_KUBERNETES_URL'] || 'https://' + ENV.fetch('KUBERNETES_SERVICE_HOST') + ':' + ENV.fetch('KUBERNETES_SERVICE_PORT') + '/api'}"
+      verify_ssl "#{ENV['KUBERNETES_VERIFY_SSL'] || true}"
+      ca_file "#{ENV['KUBERNETES_CA_FILE']}"
+      skip_labels "#{ENV['FLUENT_KUBERNETES_METADATA_SKIP_LABELS'] || 'false'}"
+      skip_container_metadata "#{ENV['FLUENT_KUBERNETES_METADATA_SKIP_CONTAINER_METADATA'] || 'false'}"
+      skip_master_url "#{ENV['FLUENT_KUBERNETES_METADATA_SKIP_MASTER_URL'] || 'false'}"
+      skip_namespace_metadata "#{ENV['FLUENT_KUBERNETES_METADATA_SKIP_NAMESPACE_METADATA'] || 'false'}"
+    </filter>

charts/lm-logs/templates/deamonset.yaml

@@ -0,0 +1,53 @@
+apiVersion: apps/v1
+kind: DaemonSet
+metadata:
+  name: {{ include "fluentd.fullname" . }}
+  labels:
+    {{- include "fluentd.labels" . | nindent 4 }}
+  annotations:
+    {{- include "fluentd.annotations" . | nindent 4 }}
+spec:
+  selector:
+    matchLabels:
+      {{- include "fluentd.selectorLabels" . | nindent 6 }}
+  template:
+    metadata:
+      labels:
+        {{- include "fluentd.selectorLabels" . | nindent 8 }}
+    spec:
+      serviceAccountName: {{ include "fluentd.serviceAccountName" . }}
+      containers:
+        - name: {{ .Chart.Name }}
+          securityContext:
+            {{- toYaml .Values.securityContext | nindent 12 }}
+          image: {{ include "fluentd-image" . }}
+          env:
+            - name: K8S_NODE_NAME
+              valueFrom:
+                fieldRef:
+                  fieldPath: spec.nodeName
+            {{ if gt (.Values.env | len) 0 }}
+            {{ include "ds-env" . | nindent 12 }}
+            {{ end }}
+          imagePullPolicy: {{ .Values.image.pullPolicy | default "Always" }}
+          resources:
+            {{- toYaml .Values.resources | nindent 12 }}
+          volumeMounts:
+            {{- toYaml .Values.volumeMounts | nindent 12 }}
+      volumes:
+        - name: fluentconf
+          configMap:
+            name: {{ include "fluentd.fullname" . }}
+        {{- toYaml .Values.volumes | nindent 8 }}
+      {{- with .Values.nodeSelector }}
+      nodeSelector:
+        {{- toYaml . | nindent 8 }}
+      {{- end }}
+      {{- with .Values.affinity }}
+      affinity:
+        {{- toYaml . | nindent 8 }}
+      {{- end }}
+      {{- with .Values.tolerations }}
+      tolerations:
+        {{- toYaml . | nindent 8 }}
+      {{- end }}