ci: Development workflow (#308)

* Sign and package

* Update dependency

* Allow getting OIDC token

* Sign the plugin when building

* Debug dist paths

* Upload to GCS

* Try uploading to integration-artifacts

* Update upload action

* Debug

* Upload with different SA

* Try uploading to old location

* Use Drone SA

* Use migrated version

* Add dev deployment step

* Push to pyroscope bucket again with shared workflow

* Add login to gcs step

* Use new SA

* Fix id/names

* Upload

* Fix bucket name

* Rename artifact name

* Fix outputs

* Trigger argo workflow to deploy to dev

* Display package name for debugging

* Fix syntax error

* Use output for dry run

* Disable dry run

* Make artifacts available publicly

* Test conditional environment

* Adding new environments

* Adding new environments

* Flip for testing no approval

* Add conditional envs

* Remove debug job

* Add docs

* Add more docs

* Update docs

.drone.migration.jsonnet

@@ -21,28 +21,15 @@ local pipeline(name, steps=[], services=[]) = {
   steps: [step('runner identification', ['echo $DRONE_RUNNER_NAME'], 'alpine')] + steps,
   trigger+: {
     ref+: [
-      'refs/heads/main',
-      'refs/pull/**',
       'refs/tags/v*.*.*',
       'refs/tags/weekly-f*',
     ],
   },
 };
 
-local mainOnly = {
-  when: {
-    ref+: [
-      'refs/heads/main',
-      'refs/pull/2/head',
-    ],
-  },
-};
-
-local mainOrReleaseOnly = {
+local releaseOnly = {
   when: {
     ref+: [
-      'refs/heads/main',
-      'refs/pull/2/head',
       'refs/tags/v*.*.*',
       'refs/tags/weekly-f*',
     ],
@@ -192,7 +179,7 @@ local generateTagsStep(depends_on=[]) = step('generate tags', [
       depends_on: [
         'build frontend packages',
       ],
-    } + mainOrReleaseOnly,
+    } + releaseOnly,
 
     step('publish zip to GCS', [], image='plugins/gcs') + {
       depends_on: [
@@ -222,34 +209,6 @@ local generateTagsStep(depends_on=[]) = step('generate tags', [
       },
     } + releaseOnly,
 
-    step('publish zip to GCS with latest-dev', [], image='plugins/gcs') + {
-      depends_on: [
-        'package and sign',
-      ],
-      settings: {
-        acl: 'allUsers:READER',
-        source: 'grafana-pyroscope-app-${DRONE_BUILD_NUMBER}.zip',
-        target: 'grafana-pyroscope-app/releases/grafana-pyroscope-app-edge.zip',
-        token: {
-          from_secret: 'gcs_service_account_key',
-        },
-      },
-    } + mainOnly,
-
-    step('publish zip to GCS with dev-tag', [], image='plugins/gcs') + {
-      depends_on: [
-        'package and sign',
-      ],
-      settings: {
-        acl: 'allUsers:READER',
-        source: 'grafana-pyroscope-app-${DRONE_BUILD_NUMBER}.zip',
-        target: 'grafana-pyroscope-app/releases/grafana-pyroscope-app-${DRONE_COMMIT}.zip',
-        token: {
-          from_secret: 'gcs_service_account_key',
-        },
-      },
-    } + mainOnly,
-
     step('publish zip to GCS with latest', [], image='plugins/gcs') + {
       depends_on: [
         'package and sign',
@@ -309,21 +268,6 @@ local generateTagsStep(depends_on=[]) = step('generate tags', [
     } + releaseOnly,
   ]),
 
-  pipeline('deploy dev', [
-    generateTagsStep(),
-    deployStep('dev'),
-  ]) + {
-    image_pull_secrets: ['gcr_reader'],
-    depends_on: [
-      'build packages',
-    ],
-    trigger+: {
-      ref: [
-        'refs/heads/main',
-      ],
-    },
-  },
-
   pipeline('weekly deploy ops', [
     generateTagsStep(),
     deployStep('ops'),

.github/workflows/ci.yml

@@ -14,6 +14,11 @@ on:
 env:
   BUNDLEWATCH_GITHUB_TOKEN: ${{secrets.BUNDLEWATCH_GITHUB_TOKEN}}
 
+# Required to create OIDC/JWT token required to use shared actions
+permissions:
+  contents: read
+  id-token: write
+
 jobs:
   frontend:
     runs-on: ubuntu-latest
@@ -52,6 +57,20 @@ jobs:
       - name: Compatibility check
         run: npx @grafana/levitate@latest is-compatible --path src/module.ts --target @grafana/data,@grafana/ui,@grafana/runtime
 
+      # The plugin is signed here so it's possible to use the artifact produced by the job directly
+      - name: Setup plugin signing
+        uses: grafana/shared-workflows/actions/get-vault-secrets@main
+        with:
+          vault_instance: ops
+          common_secrets: |
+            SIGN_PLUGIN_ACCESS_POLICY_TOKEN=plugins/sign-plugin-access-policy-token:token
+
+      # create MANIFEST in dist
+      - name: Sign plugin
+        run: yarn sign
+        env:
+          GRAFANA_ACCESS_POLICY_TOKEN: ${{ env.SIGN_PLUGIN_ACCESS_POLICY_TOKEN }}
+
       - uses: actions/upload-artifact@v4
         if: always()
         with:
@@ -111,3 +130,80 @@ jobs:
             e2e/test-reports
             e2e/test-results
           retention-days: 15
+
+  package:
+    # This step creates a zip file with the plugin and publishes it to Google Cloud Storage bucket.
+    # Frontend artifacts have 1 day retention. This step needs to be run within 24 hours after frontend job finished.
+    # Plugin is already signed in frontend job so if you need to use to locally you can just download the artifact
+    # When pushed to main it uses "gcs-no-approval" environment which can be triggered only from main
+    # to push the package automatically without approval
+    name: Package and publish plugin
+    needs: [ frontend ]
+    environment: ${{ github.event_name == 'push' && 'gcs-no-approval' || 'gcs' }}
+    runs-on: ubuntu-latest
+    outputs:
+      package_id: ${{ steps.metadata.outputs.package_id }}
+      sha: ${{ steps.metadata.outputs.sha }}
+    steps:
+      # Required to correctly auth to GCS
+      - name: Prepare - GCS
+        uses: actions/checkout@v4
+
+      - name: Prepare - Download build artifacts
+        uses: actions/download-artifact@v4
+        with:
+          name: build-frontend
+          path: dist
+
+      - name: Get plugin metadata
+        id: metadata
+        run: |
+          sudo apt-get install jq
+          
+          export GRAFANA_PLUGIN_ID=$(cat dist/plugin.json | jq -r .id)
+          export SHA=${{ github.event.pull_request.head.sha || github.sha }}
+          export PACKAGE_ID=${GRAFANA_PLUGIN_ID}-${SHA}
+          echo "plugin_id=${GRAFANA_PLUGIN_ID}" >> $GITHUB_OUTPUT
+          echo "package_id=${PACKAGE_ID}" >> $GITHUB_OUTPUT
+          echo "sha=${SHA}" >> $GITHUB_OUTPUT
+          echo "archive_name=${PACKAGE_ID}.zip" >> $GITHUB_OUTPUT
+
+      # Create zip file with name following conventions [plugin-id]-[sha].zip
+      - name: Package plugin
+        run: |
+          mv dist ${{ steps.metadata.outputs.plugin_id }}
+          zip ${{ steps.metadata.outputs.archive_name }} ${{ steps.metadata.outputs.plugin_id }} -r
+
+      - name: Login to GCS
+        uses: 'google-github-actions/auth@v2'
+        with:
+          workload_identity_provider: ${{ secrets.WORKLOAD_IDENTITY_PROVIDER }}
+          service_account: ${{ secrets.GCS_SERVICE_ACCOUNT }}
+
+      - name: Publish to GCS
+        uses: 'google-github-actions/upload-cloud-storage@v1'
+        with:
+          path: ./
+          destination: 'grafana-pyroscope-app/releases'
+          glob: '*.zip'
+          predefinedAcl: publicRead
+
+  deploy-to-dev:
+    # This triggers Argo workflow that will perform deployment to wave. Pushes to main will trigger deployment automatically
+    name: Deploy to dev
+    needs: [ package ]
+    runs-on: ubuntu-latest
+    environment: ${{ github.event_name == 'push' && 'dev-no-approval' || 'dev' }}
+    steps:
+      - name: Publish to dev
+        run: echo Deploying ${{ needs.package.outputs.sha }} to dev
+      - name: Deploy to dev
+        uses: grafana/shared-workflows/actions/[email protected]
+        with:
+          instance: "ops"
+          namespace: "phlare-cd"
+          workflow_template: "deploy-plugin-dev"
+          parameters: |
+            plugintag=${{ needs.package.outputs.sha }}
+          extra_args: "--name deploy-plugin-dev-${{ needs.package.outputs.sha }}"
+          log_level: "debug"