fix tests and ensure entity annotations are used

examples/scaffolder/SparkAppArgoWorkflow/template.yaml

@@ -6,7 +6,7 @@ metadata:
   description: Creates an Apache Spark Application with Argo Workflows
 spec:
   owner: guest
-  type: service
+  type: job
   # these are the steps which are rendered in the frontend with the form input
   parameters:
     - title: Fill in some steps
@@ -29,21 +29,14 @@ spec:
           ui:options:
             catalogFilter:
               kind: Group
-#        labels:
-#          title: Labels
-#          type: object
-#          additionalProperties:
-#            type: string
-#          description: Labels to apply to the application
-#          ui:autofocus: true
         namespace:
           title: Namespace
           type: string
           default: default
           description: Namespace to deploy this application into. Optional. Defaults to application name.
           ui:options:
             rows: 5
-    - title: Fill in some steps
+    - title: Paste your Spark manifest
       required:
         - manifest
       properties:
@@ -67,13 +60,11 @@ spec:
       name: Fetch Base
       action: fetch:template
       input:
-        url: ./templates
+        url: ./templates/SparkManifest
         values:
           name: ${{ parameters.name }}
           namespace: ${{ parameters.namespace }}
-          clusterName: 'cnoe-packaging-2'
-#          labels: ${{ parameters.labels }}
-#          manifest: ${{ parameters.manifest }}
+          clusterName: 'cnoe-packaging'
     - id: write
       name: write-to-file
       action: roadiehq:utils:fs:write
@@ -92,11 +83,46 @@ spec:
       input:
         path: merge.json
         content: ${{ steps.parse.output.content }}
+    - id: deserialise
+      name: deserialize manifest
+      action: roadiehq:utils:fs:parse
+      input:
+        path: 'merge.json'
+        parser: 'json'
+    - id: fetch-base
+      name: Fetch Base
+      action: fetch:template
+      input:
+        url: ./templates/ArgoWorkflowsManifest
+        values:
+          name: ${{ parameters.name }}
+          namespace: ${{ parameters.namespace }}
+          clusterName: 'cnoe-packaging'
+          manifest: ${{ steps.deserialise.output.content }}
+    - id: apply
+      name: apply manifest
+      action: cnoe:kubernetes:apply
+      input:
+        manifestPath: manifest.yaml
+        namespaced: true
+        clusterName: "cnoe-packaging"
+    - id: create-repo
+      name: Create Repository
+      action: github:repo:create
+      input:
+        repoUrl: github.com?repo=spark-ui-${{parameters.name}}&owner=manabuOrg
+    - id: init-repo
+      name: Initialize Repository
+      action: github:repo:push
+      input:
+        repoUrl: github.com?repo=spark-ui-${{parameters.name}}&owner=manabuOrg
+        defaultBranch: main
     - id: register
       name: Register
       action: catalog:register
       input:
-        catalogInfoPath: 'catalog-info.yaml'
+        catalogInfoPath: '/catalog-info.yaml'
+        repoContentsUrl: ${{ steps['init-repo'].output.repoContentsUrl }}
   output:
     links:
       - title: Open in catalog

examples/scaffolder/SparkAppArgoWorkflow/templates/ArgoWorkflowsManifest/catalog-info.yaml

@@ -0,0 +1,15 @@
+---
+apiVersion: backstage.io/v1alpha1
+kind: Component
+metadata:
+  name: ${{values.name | dump}}
+  annotations:
+    backstage.io/kubernetes-id: "${{values.clusterName}}"
+    apache-spark/cluster-name: "${{values.clusterName}}"
+    argo-workflows/cluster-name: "${{values.clusterName}}"
+    backstage.io/kubernetes-namespace: "${{values.namespace}}"
+    backstage.io/kubernetes-label-selector: "backstage.io/component-id=${{values.name}}"
+spec:
+  type: job
+  lifecycle: experimental
+  owner: ${{ values.owner | default('guest')}}

examples/scaffolder/SparkAppArgoWorkflow/templates/ArgoWorkflowsManifest/manifest.yaml

@@ -0,0 +1,43 @@
+apiVersion: argoproj.io/v1alpha1
+kind: Workflow
+metadata:
+  name: "spark-${{values.name}}"
+  namespace: "${{values.namespace}}"
+  labels:
+    backstage.io/component-id: "${{values.name}}"
+spec:
+  arguments: {}
+  entrypoint: demo-workflow
+  serviceAccountName: argo-workflows
+  workflowMetadata:
+    labels:
+      backstage.io/component-id: "${{values.name}}"
+  templates:
+    - name: demo-workflow
+      steps:
+        - - name: prepare-resources
+            template: prepare-resources
+        - - name: run-sparkapp
+            template: run-sparkapp
+        - - name: cleanup-resources
+            template: cleanup-resources
+          - name: notify-users
+            template: cleanup-resources
+    - name: prepare-resources
+      container:
+        image: docker/whalesay
+        command: [ sleep ]
+        args: [ "10" ]
+    - name: run-sparkapp
+      resource:
+        action: create
+        setOwnerReference: true
+        successCondition: status.applicationState.state == COMPLETED
+        failureCondition: status.applicationState.state in (FAILED, ERROR)
+        manifest: |
+          ${{values.manifest | dump}}
+    - name: cleanup-resources
+      container:
+        image: docker/whalesay
+        command: [ sleep ]
+        args: [ "5" ]

examples/scaffolder/SparkAppArgoWorkflow/templates/SparkManifest/merge.json

@@ -0,0 +1,8 @@
+{
+  "metadata": {
+    "namespace": "${{values.namespace}}",
+    "labels": {
+      "backstage.io/component-id": "${{values.name}}"
+    }
+  }
+}

examples/scaffolder/SparkAppUI/template.yaml

@@ -94,7 +94,7 @@ spec:
         url: ./templates
         values:
           params: ${{parameters}}
-          clusterName: 'cnoe-packaging-2'
+          clusterName: 'cnoe-packaging'
           name: ${{parameters.name}}
           namespace: ${{parameters.namespace}}
     - id: init-repo
@@ -115,7 +115,7 @@ spec:
       input:
         manifestObject: ${{ steps.deserialise.output.content }}
         namespaced: true
-        clusterName: "cnoe-packaging-2"
+        clusterName: "cnoe-packaging"
     - id: register
       name: Register
       action: catalog:register

package.json

@@ -23,6 +23,7 @@
     "postpack": "backstage-cli package postpack"
   },
   "dependencies": {
+    "@backstage/catalog-model": "^1.4.1",
     "@backstage/core-components": "^0.13.1",
     "@backstage/core-plugin-api": "^1.5.1",
     "@backstage/plugin-catalog-react": "^1.7.0",

pi-argo-workflows.yaml

@@ -0,0 +1,60 @@
+apiVersion: argoproj.io/v1alpha1
+kind: Workflow
+metadata:
+  name: spark-operator
+  namespace: default
+spec:
+  arguments: {}
+  entrypoint: demo-workflow
+  serviceAccountName: argo-workflows
+  templates:
+    - name: demo-workflow
+      steps:
+        - - name: sleep
+            template: sleep
+        - - name: spark-operator
+            template: sparkapp
+    - name: sleep
+      container:
+        image: docker/whalesay
+        command: [ sleep ]
+        args: [ "60" ]
+    - name: sparkapp
+      resource:
+        action: create
+        setOwnerReference: true
+        successCondition: status.applicationState.state == COMPLETED
+        failureCondition: status.applicationState.state in (FAILED, ERROR)
+        manifest: |
+          apiVersion: "sparkoperator.k8s.io/v1beta2"
+          kind: SparkApplication
+          metadata:
+            generateName: pyspark-pi-
+            namespace: default
+          spec:
+            type: Python
+            pythonVersion: "3"
+            mode: cluster
+            image: "public.ecr.aws/r1l5w1y9/spark-operator:3.2.1-hadoop-3.3.1-java-11-scala-2.12-python-3.8-latest"
+            mainApplicationFile: "local:///opt/spark/examples/src/main/python/pi.py"
+            sparkVersion: "3.1.1"
+            restartPolicy:
+              type: OnFailure
+              onFailureRetries: 1
+              onFailureRetryInterval: 10
+              onSubmissionFailureRetries: 1
+              onSubmissionFailureRetryInterval: 20
+            driver:
+              cores: 1
+              coreLimit: "1200m"
+              memory: "512m"
+              labels:
+                version: 3.1.1
+              serviceAccount: spark
+            executor:
+              cores: 1
+              instances: 2
+              memory: "512m"
+              serviceAccount: spark
+              labels:
+                version: 3.1.1

pi-success.yaml

@@ -1,23 +1,7 @@
-#
-# Copyright 2017 Google LLC
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-#     https://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
 apiVersion: "sparkoperator.k8s.io/v1beta2"
 kind: SparkApplication
 metadata:
-#  name: spark-pi
-  generateName: spark-pi
+  generateName: spark-pi-
   namespace: default
 spec:
   type: Python

pi.yaml

@@ -1,23 +1,9 @@
-#
-# Copyright 2017 Google LLC
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-#     https://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
 
 apiVersion: "sparkoperator.k8s.io/v1beta2"
 kind: SparkApplication
 metadata:
 #  name: spark-pi
-  generateName: spark-pi
+  generateName: spark-pi-
   namespace: default
 spec:
   type: Python

src/api/index.ts

@@ -46,14 +46,16 @@ export class ApacheSparkClient implements ApacheSparkApi {
   async getSparkApps(
     clusterName: string | undefined,
     namespace: string | undefined,
-    _labels: string,
+    labels: string | undefined,
   ): Promise<ApacheSparkList> {
     const ns = namespace !== undefined ? namespace : 'default';
     const path = `/apis/${API_VERSION}/namespaces/${ns}/${SPARK_APP_PLURAL}`;
     const query = new URLSearchParams({
       [K8s_API_TIMEOUT]: '30',
-      // labelSelector: labels,
     });
+    if (labels) {
+      query.set('labelSelector', labels);
+    }
     const resp = await this.kubernetesApi.proxy({
       clusterName:
         clusterName !== undefined ? clusterName : await this.getFirstCluster(),

src/components/ApacheSparkLogs/ApacheSparkLogs.test.tsx

@@ -1,12 +1,19 @@
 import React from 'react';
 import { render, screen } from '@testing-library/react';
 import { useApi } from '@backstage/core-plugin-api';
+import { useEntity } from '@backstage/plugin-catalog-react';
 import useAsync from 'react-use/lib/useAsync';
 import { ApacheSpark } from '../../api/model';
 import { ApacheSparkDriverLogs } from './ApacheSparkLogs';
+import {
+  APACHE_SPARK_LABEL_SELECTOR_ANNOTATION,
+  CLUSTER_NAME_ANNOTATION,
+  K8S_NAMESPACE_ANNOTATION,
+} from '../../consts';
 
 jest.mock('@backstage/core-plugin-api');
 jest.mock('react-use/lib/useAsync');
+jest.mock('@backstage/plugin-catalog-react');
 
 jest.mock('@backstage/core-components', () => ({
   LogViewer: (props: { text: string }) => {
@@ -17,6 +24,7 @@ jest.mock('@backstage/core-components', () => ({
 describe('ApacheSparkDriverLogs', () => {
   const mockUseApi = useApi as jest.MockedFunction<typeof useApi>;
   const mockUseAsync = useAsync as jest.MockedFunction<typeof useAsync>;
+  const mockUseEntity = useEntity as jest.MockedFunction<typeof useEntity>;
   const mockGetLogs = jest.fn();
   const mockSparkApp = {
     status: {
@@ -30,6 +38,21 @@ describe('ApacheSparkDriverLogs', () => {
     mockUseApi.mockReturnValue({
       getLogs: mockGetLogs,
     });
+    mockUseEntity.mockReturnValue({
+      entity: {
+        apiVersion: 'version',
+        kind: 'kind',
+        metadata: {
+          name: 'name',
+          namespace: 'ns1',
+          annotations: {
+            [K8S_NAMESPACE_ANNOTATION]: 'k8s-ns',
+            [CLUSTER_NAME_ANNOTATION]: 'my-cluster',
+            [APACHE_SPARK_LABEL_SELECTOR_ANNOTATION]: 'env=test',
+          },
+        },
+      },
+    });
   });
 
   afterEach(() => {