chore: adopt kubernetes dql (#62)

* chore: adopt kubernetes DQL

* chore: typo

* chore: add tests and fix quotation of filter values

* chore: add some comments

* chore: address feedback

* chore: don't use component namespace, only selector

* chore: use correct namespace filter

* chore: fix tests

* chore: fix tests

* chore: make kubernetes id annotation required, do not fallback to 'default' namespace

* chore: fix kubernetes namespace, adjust readme, override kubernetes id if label is given

* chore: address readme feedback

* chore: address feedback

* chore: address feedback & improve client-backend request

* chore: type fixes and standalone server fix

* chore: adjust empty and missing annotation screen

* chore: adapt no-resources-found message for kubernetes

Author: Kirdock

README.md

@@ -192,14 +192,34 @@ deployments in your Dynatrace environment.
 />
 ```
 
-_Convention:_ Kubernetes pods with a `backstage.io/component` label will be
-listed for the corresponding Backstage component if they are properly annotated
-in the deployment descriptor:
+_Convention:_ Kubernetes pods will be listed for the corresponding Backstage
+component if they are properly annotated in the deployment descriptor. See
+[annotations](https://backstage.io/docs/features/software-catalog/descriptor-format/#annotations-optional).
+
+Example:
 
 ```yaml
-backstage.io/component: <backstage-namespace>.<backstage-component-name>
+backstage.io/kubernetes-id: kubernetesid
+backstage.io/kubernetes-namespace: namespace
+backstage.io/kubernetes-label-selector: stage=hardening,name=frontend
 ```
 
+- The annotation `backstage.io/kubernetes-id` will look for the Kubernetes label
+  `backstage.io/kubernetes-id`.
+- The annotation `backstage.io/kubernetes-namespace` will look for the
+  Kubernetes namespace.
+- The annotation `backstage.io/kubernetes-label-selector` will look for the
+  labels defined in it. So
+  `backstage.io/kubernetes-label-selector: stage=hardening,name=frontend` will
+  look for a Kubernetes label `stage` with the value `hardening` and a label
+  `name` with the value `frontend`.
+
+If a `backstage.io/kubernetes-label-selector` is given,
+`backstage.io/kubernetes-id` is ignored.
+
+If no namespace is given, it looks for all namespaces. There is no fallback to
+`default`.
+
 The query for fetching the monitoring data for Kubernetes deployments is defined
 here:
 [`dynatrace.kubernetes-deployments`](plugins/dql-backend/src/service/queries.ts).

catalog-info.yaml

@@ -3,6 +3,8 @@ kind: Component
 metadata:
   name: demo-backstage
   description: Backstage Demo instance.
+  annotations:
+    backstage.io/kubernetes-id: kubernetescustom
 spec:
   type: website
   owner: user:default/mjakl

packages/backend/src/plugins/dynatrace-dql.ts

@@ -20,8 +20,5 @@ import { Router } from 'express';
 export default async function createPlugin(
   env: PluginEnvironment,
 ): Promise<Router> {
-  return await createRouter({
-    logger: env.logger,
-    config: env.config,
-  });
+  return await createRouter(env);
 }

plugins/dql-backend/src/service/queries.test.ts

@@ -0,0 +1,99 @@
+/**
+ * @license
+ * Copyright 2024 Dynatrace LLC
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+import { dynatraceQueries, DynatraceQueryKeys } from './queries';
+import { Entity } from '@backstage/catalog-model';
+
+describe('queries', () => {
+  const getEntity = (annotations?: Record<string, string>): Entity => ({
+    apiVersion: '1.0.0',
+    kind: 'component',
+    metadata: { name: 'componentName', annotations },
+  });
+  const defaultApiConfig = {
+    environmentName: 'environment',
+    environmentUrl: 'url',
+  };
+
+  describe(DynatraceQueryKeys.KUBERNETES_DEPLOYMENTS, () => {
+    it('should fail if neither the label selector nor the kubernetes id annotation is provided', () => {
+      // act, assert
+      expect(() =>
+        dynatraceQueries[DynatraceQueryKeys.KUBERNETES_DEPLOYMENTS](
+          getEntity(),
+          defaultApiConfig,
+        ),
+      ).toThrow();
+    });
+
+    it('should return the query without the kubernetesId filter if a label selector is provided', () => {
+      // act
+      const query = dynatraceQueries[DynatraceQueryKeys.KUBERNETES_DEPLOYMENTS](
+        getEntity({
+          'backstage.io/kubernetes-id': 'kubernetesId',
+          'backstage.io/kubernetes-label-selector': 'label=value',
+        }),
+        defaultApiConfig,
+      );
+
+      // assert
+      expect(query).not.toContain(
+        '| filter workload.labels[`backstage.io/kubernetes-id`] == "kubernetesId"',
+      );
+      expect(query).toContain('| filter workload.labels[`label`] == "value"');
+    });
+
+    it('should return the query with the kubernetesId filter ', () => {
+      // act
+      const query = dynatraceQueries[DynatraceQueryKeys.KUBERNETES_DEPLOYMENTS](
+        getEntity({ 'backstage.io/kubernetes-id': 'kubernetesId' }),
+        defaultApiConfig,
+      );
+
+      // assert
+      expect(query).toContain(
+        '| filter workload.labels[`backstage.io/kubernetes-id`] == "kubernetesId"',
+      );
+    });
+
+    it('should return the query with the namespace filter', () => {
+      // act
+      const query = dynatraceQueries[DynatraceQueryKeys.KUBERNETES_DEPLOYMENTS](
+        getEntity({
+          'backstage.io/kubernetes-id': 'kubernetesId',
+          'backstage.io/kubernetes-namespace': 'namespace',
+        }),
+        defaultApiConfig,
+      );
+
+      // assert
+      expect(query).toContain('| filter Namespace == "namespace"');
+    });
+
+    it('should return the query with the label selector filter', () => {
+      // act
+      const query = dynatraceQueries[DynatraceQueryKeys.KUBERNETES_DEPLOYMENTS](
+        getEntity({
+          'backstage.io/kubernetes-label-selector': 'label=value',
+          'backstage.io/kubernetes-namespace': 'namespace',
+        }),
+        defaultApiConfig,
+      );
+
+      // assert
+      expect(query).toContain('| filter workload.labels[`label`] == "value"');
+    });
+  });
+});

plugins/dql-backend/src/service/queries.ts

@@ -13,9 +13,53 @@
  * See the License for the specific language governing permissions and
  * limitations under the License.
  */
+import { generateKubernetesSelectorFilter } from '../utils/labelSelectorParser';
+import { Entity } from '@backstage/catalog-model';
 
-export const dynatraceQueries: Record<string, string | undefined> = {
-  'kubernetes-deployments': `
+export enum DynatraceQueryKeys {
+  KUBERNETES_DEPLOYMENTS = 'kubernetes-deployments',
+}
+
+interface ApiConfig {
+  environmentName: string;
+  environmentUrl: string;
+}
+
+export const isValidDynatraceQueryKey = (
+  key: string,
+): key is DynatraceQueryKeys => key in dynatraceQueries;
+
+export const dynatraceQueries: Record<
+  DynatraceQueryKeys,
+  (entity: Entity, apiConfig: ApiConfig) => string
+> = {
+  [DynatraceQueryKeys.KUBERNETES_DEPLOYMENTS]: (entity, apiConfig) => {
+    const labelSelector =
+      entity.metadata.annotations?.['backstage.io/kubernetes-label-selector'];
+    const kubernetesId =
+      entity.metadata.annotations?.['backstage.io/kubernetes-id'];
+    const namespace =
+      entity.metadata.annotations?.['backstage.io/kubernetes-namespace'];
+
+    const filterLabel = labelSelector
+      ? generateKubernetesSelectorFilter(labelSelector)
+      : '';
+    // if a label filter is given, the id is ignored
+    const filterKubernetesId =
+      filterLabel || !kubernetesId
+        ? ''
+        : `| filter workload.labels[\`backstage.io/kubernetes-id\`] == "${kubernetesId}"`;
+    const filterNamespace = namespace
+      ? `| filter Namespace == "${namespace}"`
+      : '';
+
+    if (!filterKubernetesId && !filterLabel) {
+      throw new Error(
+        'One of the component annotations is required: "backstage.io/kubernetes-id" or "backstage.io/kubernetes-label-selector"',
+      );
+    }
+
+    return `
     fetch dt.entity.cloud_application, from: -10m
     | fields id,
         name = entity.name,
@@ -25,19 +69,22 @@ export const dynatraceQueries: Record<string, string | undefined> = {
         namespace.id = belongs_to[dt.entity.cloud_application_namespace]
     | sort upper(name) asc
     | lookup [fetch dt.entity.cloud_application_instance, from: -10m | fields matchedId = instance_of[dt.entity.cloud_application], podVersion = cloudApplicationLabels[\`app.kubernetes.io/version\`]], sourceField:id, lookupField:matchedId, fields:{podVersion}
-    | fieldsAdd Workload = record({type="link", text=name, url=concat("\${environmentUrl}/ui/apps/dynatrace.kubernetes/resources/pod?entityId=", id)})
+    | fieldsAdd Workload = record({type="link", text=name, url=concat("${apiConfig.environmentUrl}/ui/apps/dynatrace.kubernetes/resources/pod?entityId=", id)})
     | lookup [fetch dt.entity.kubernetes_cluster, from: -10m | fields id, Cluster = entity.name],sourceField:cluster.id, lookupField:id, fields:{Cluster}
     | lookup [fetch dt.entity.cloud_application_namespace, from: -10m | fields id, Namespace = entity.name], sourceField:namespace.id, lookupField:id, fields:{Namespace}
     | lookup [fetch events, from: -30m | filter event.kind == "DAVIS_PROBLEM" | fieldsAdd affected_entity_id = affected_entity_ids[0] | summarize collectDistinct(event.status), by:{display_id, affected_entity_id}, alias:problem_status | filter NOT in(problem_status, "CLOSED") | summarize Problems = count(), by:{affected_entity_id}], sourceField:id, lookupField:affected_entity_id, fields:{Problems}
     | fieldsAdd Problems=coalesce(Problems,0)
     | lookup [ fetch events, from: -30m | filter event.kind=="SECURITY_EVENT" | filter event.category=="VULNERABILITY_MANAGEMENT" | filter event.provider=="Dynatrace" | filter event.type=="VULNERABILITY_STATE_REPORT_EVENT" | filter in(vulnerability.stack,{"CODE_LIBRARY","SOFTWARE","CONTAINER_ORCHESTRATION"}) | filter event.level=="ENTITY" | summarize { workloadId=arrayFirst(takeFirst(related_entities.kubernetes_workloads.ids)), vulnerability.stack=takeFirst(vulnerability.stack)}, by: {vulnerability.id, affected_entity.id} | summarize { Vulnerabilities=count() }, by: {workloadId}], sourceField:id, lookupField:workloadId, fields:{Vulnerabilities}
     | fieldsAdd Vulnerabilities=coalesce(Vulnerabilities,0)
-    | filter workload.labels[\`backstage.io/component\`] == "\${componentNamespace}.\${componentName}"
+    ${filterKubernetesId} 
+    ${filterNamespace} 
+    ${filterLabel}
     | fieldsAdd Logs = record({type="link", text="Show logs", url=concat(
-      "\${environmentUrl}",
+      "${apiConfig.environmentUrl}",
       "/ui/apps/dynatrace.notebooks/intent/view-query#%7B%22dt.query%22%3A%22fetch%20logs%20%7C%20filter%20matchesValue(dt.entity.cloud_application%2C%5C%22",
       id,
       "%5C%22)%20%7C%20sort%20timestamp%20desc%22%2C%22title%22%3A%22Logs%22%7D")})
     | fieldsRemove id, deploymentVersion, podVersion, name, workload.labels, cluster.id, namespace.id
-    | fieldsAdd Environment = "\${environmentName}"`,
+    | fieldsAdd Environment = "${apiConfig.environmentName}"`;
+  },
 };

plugins/dql-backend/src/service/queryCompiler.ts

@@ -16,11 +16,11 @@
 
 export function compileDqlQuery(
   queryToCompile: string,
-  variables: Record<string, unknown>,
+  variables: Record<string, string | undefined>,
 ): string {
   return Array.from(Object.entries(variables)).reduce(
     (query: string, [variable, value]) =>
-      query.replaceAll(`\${${variable}}`, String(value)),
+      query.replaceAll(`\${${variable}}`, String(value ?? '')),
     queryToCompile,
   );
 }

plugins/dql-backend/src/service/queryExecutor.test.ts

@@ -14,13 +14,19 @@
  * limitations under the License.
  */
 import { QueryExecutor } from './queryExecutor';
+import { Entity } from '@backstage/catalog-model';
 
 describe('queryExecutor', () => {
   const executor = new QueryExecutor([], { 'my.id': 'myQuery' });
   const inputVariables = {
     componentNamespace: 'namespace',
     componentName: 'name',
   };
+  const entity: Entity = {
+    apiVersion: '1.0.0',
+    kind: 'component',
+    metadata: { name: 'componentName' },
+  };
 
   describe('Invalid IDs', () => {
     it('should throw an error if a custom query is undefined', async () => {
@@ -33,7 +39,7 @@ describe('queryExecutor', () => {
     it('should throw an error if a Dynatrace query is undefined', async () => {
       // assert
       await expect(() =>
-        executor.executeDynatraceQuery('not.existing', inputVariables),
+        executor.executeDynatraceQuery('not.existing', entity),
       ).rejects.toThrow();
     });
   });
@@ -50,7 +56,7 @@ describe('queryExecutor', () => {
       // act
       const result = await executor.executeDynatraceQuery(
         'kubernetes-deployments',
-        inputVariables,
+        entity,
       );
       // assert
       expect(result).toEqual([]);

plugins/dql-backend/src/service/queryExecutor.ts

@@ -14,8 +14,9 @@
  * limitations under the License.
  */
 import { DynatraceApi } from './dynatraceApi';
-import { dynatraceQueries } from './queries';
+import { dynatraceQueries, isValidDynatraceQueryKey } from './queries';
 import { compileDqlQuery } from './queryCompiler';
+import { Entity } from '@backstage/catalog-model';
 import { TabularData } from '@dynatrace/backstage-plugin-dql-common';
 import { z } from 'zod';
 
@@ -44,30 +45,12 @@ export class QueryExecutor {
     queryId: string,
     variables: ComponentQueryVariables,
   ): Promise<TabularData> {
-    const query = this.queries[queryId];
-    if (!query) {
+    const dqlQuery = this.queries[queryId];
+    if (!dqlQuery) {
       throw new Error(`No custom query to the given id "${queryId}" found`);
     }
-    return this.executeQuery(query, variables);
-  }
 
-  async executeDynatraceQuery(
-    queryId: string,
-    variables: ComponentQueryVariables,
-  ): Promise<TabularData> {
-    const query = dynatraceQueries[queryId];
-    if (!query) {
-      throw new Error(`No Dynatrace query to the given id "${queryId}" found`);
-    }
-    return this.executeQuery(query, variables);
-  }
-
-  private async executeQuery(
-    dqlQuery: string,
-    variables: ComponentQueryVariables,
-  ): Promise<TabularData> {
     componentQueryVariablesSchema.parse(variables);
-
     const results$ = this.apis.map(api => {
       const compiledQuery = compileDqlQuery(dqlQuery, {
         ...variables,
@@ -79,4 +62,18 @@ export class QueryExecutor {
     const results = await Promise.all(results$);
     return results.flatMap(result => result);
   }
+
+  async executeDynatraceQuery(
+    queryId: string,
+    entity: Entity,
+  ): Promise<TabularData> {
+    if (!isValidDynatraceQueryKey(queryId)) {
+      throw new Error(`No Dynatrace query to the given id "${queryId}" found`);
+    }
+    const results$ = this.apis.map(api =>
+      api.executeDqlQuery(dynatraceQueries[queryId](entity, api)),
+    );
+    const results = await Promise.all(results$);
+    return results.flatMap(result => result);
+  }
 }

plugins/dql-backend/src/service/router.test.ts

@@ -16,6 +16,7 @@
 import { createRouter } from './router';
 import { getVoidLogger } from '@backstage/backend-common';
 import { MockConfigApi } from '@backstage/test-utils';
+import { PluginEnvironment } from 'backend/src/types';
 import express from 'express';
 
 describe('createRouter', () => {
@@ -24,6 +25,14 @@ describe('createRouter', () => {
   beforeAll(async () => {
     const router = await createRouter({
       logger: getVoidLogger(),
+      discovery: {
+        async getBaseUrl(): Promise<string> {
+          return '';
+        },
+        async getExternalBaseUrl(): Promise<string> {
+          return '';
+        },
+      },
       config: new MockConfigApi({
         dynatrace: {
           environments: [
@@ -38,7 +47,7 @@ describe('createRouter', () => {
           ],
         },
       }),
-    });
+    } as unknown as PluginEnvironment);
     app = express().use(router);
   });