docs: add packs caching solution DOC-1460 (#4632)

* docs: add caching solution DOC-1460 * docs: add upload logic DOC-1460 * docs: update upload artifact DOC-1460 * docs: update build steps * docs: update download logic * docs: fix folder name * docs: fix folder path * docs: add composite action that builds cached packs DOC-1460 * docs: fix composite action reference * docs: fix composite action path * docs: fix composite action shell DOC-1460 * docs: add missing shell * docs: add failure condition * docs: create static/img/packs * docs: fix broken action * docs: add local workflow script * docs: add exit code handling DOC-1460 * ci: auto-formatting prettier issues * docs: add api-repositories-response to clean-packs * docs: add error handling to writeFile * docs: remove plugin from asset and add async file write function * docs: add directory creation to writeResponseFile * docs: remove push condition from post release * docs: add push condition from post release * docs: change name of workflow * docs: add readme documentation * ci: auto-formatting prettier issues * docs: fix reading of exit code * docs: remove the true on npm run build * docs: disable automatic failure to allow the exit code to be set * docs: add intermediary step * docs: adjust workflow * docs: adjust workflow with failure * docs: remove fail condition * docs: remove intermediary step * docs: add caching in other workflows * docs: add extra context in readme * docs: place make build commands back * docs: add build-ci job * docs: adjust the make commands DOC-1460 * docs: remove spaces * docs: remove spaces * docs: update readme * docs: remove clear from build-ci * docs: replace clear and change action * docs: add exit codes section in the readme * docs: add conditional around jq install --------- Co-authored-by: Carolina Delwing Rosa <[email protected]> Co-authored-by: addetz <[email protected]>
spectrocloud · Nov 19, 2024 · 2227097 · 2227097
1 parent 9e5cfe1
commit 2227097
Show file tree

Hide file tree

Showing 14 changed files with 406 additions and 52 deletions.
diff --git a/.github/actions/build-cached-packs/action.yaml b/.github/actions/build-cached-packs/action.yaml
@@ -0,0 +1,48 @@
+name: "Build with cached packs"
+inputs:
+  gh-token:
+    description: "GitHub Token for authentication"
+    required: true
+
+runs:
+  using: "composite"
+  steps:
+    - name: Install jq (JSON processor) if not found
+      run: |
+        if ! command -v jq &> /dev/null; then
+          sudo apt-get update
+          sudo apt-get install -y jq
+        else
+          echo "jq is already installed. Skipping install..."
+        fi
+      shell: bash
+
+    - name: Download Packs Data
+      run: |
+        # Find the latest packs upload workflow.
+        run_id=$(gh run list --workflow="post_release.yaml" --limit 1 --json databaseId | jq -r '.[0].databaseId')
+        # Remove any downloaded artifacts, should they exist.
+        rm -rf ./downloaded_artifacts
+        # Download the latest artifact to a new dir.
+        gh run download ${run_id} --name build-packs --dir ./downloaded_artifacts
+      shell: bash
+      env:
+        GH_TOKEN: ${{ inputs.gh-token }}
+
+    - name: Unpack packs data
+      run: |
+        # Ensure the correct folders exist.
+        mkdir -p .docusaurus/packs-integrations
+        # Move the files to their correct places in the checked out repository
+        mv downloaded_artifacts/.docusaurus/packs-integrations/* .docusaurus/packs-integrations
+        mkdir -p static/img/packs
+        mv downloaded_artifacts/build/packs/* static/img/packs
+        # Clean up. 
+        rm -rf downloaded_artifacts
+      shell: bash
+
+    - name: Build
+      run: |
+        rm -rf build
+        npm run build
+      shell: bash
diff --git a/.github/workflows/dependabot.yaml b/.github/workflows/dependabot.yaml
@@ -43,4 +43,11 @@ jobs:
 
       - name: Build
         run: |
-          npm run build
+          touch .env
+          make build-ci
+      
+      - name: Build with cached packs
+        if: ${{ env.BUILD_EXIT_CODE == '5' }}
+        uses: ./.github/actions/build-cached-packs
+        with:
+          gh-token: ${{ secrets.GITHUB_TOKEN }}
diff --git a/.github/workflows/nightly-docker-build.yaml b/.github/workflows/nightly-docker-build.yaml
@@ -30,11 +30,16 @@ jobs:
 
       - run: npm ci
 
-      - name: Compile
+      - name: Build
         run: |
           touch .env
-          make build
-
+          make build-ci
+      
+      - name: Build with cached packs
+        if: ${{ env.BUILD_EXIT_CODE == '5' }}
+        uses: ./.github/actions/build-cached-packs
+        with:
+          gh-token: ${{ secrets.GITHUB_TOKEN }}
 
       - name: Set up QEMU
         uses: docker/setup-qemu-action@v3

diff --git a/.github/workflows/post_release.yaml b/.github/workflows/post_release.yaml
@@ -0,0 +1,59 @@
+name: Post Release Processing
+# This workflow is triggered when a workflow run of the "Release to Production" workflow is completed or when manually triggered.
+# The primary purpose of this workflow is to build the site, copy the packs data and upload it as artifacts.
+# The packs data can be used as a fallback when the Palette API cannot return a packs list.
+
+on:
+  workflow_run:
+    workflows: ["Release to Production"]
+    types: [completed]
+  workflow_dispatch:
+
+env:
+  GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+  FULLSTORY_ORGID: ${{ secrets.FULLSTORY_ORGID }}
+  ALGOLIA_ADMIN_KEY: ${{ secrets.ALGOLIA_ADMIN_KEY }}
+  ALGOLIA_APP_ID: ${{ secrets.ALGOLIA_APP_ID }}
+  ALGOLIA_SEARCH_KEY: ${{ secrets.ALGOLIA_SEARCH_KEY }}
+  ALGOLIA_INDEX_NAME: ${{ secrets.ALGOLIA_INDEX_NAME }}
+  PALETTE_API_KEY: ${{ secrets.PALETTE_API_KEY }}
+  DISABLE_PACKS_INTEGRATIONS: ${{ secrets.DISABLE_PACKS_INTEGRATIONS }}
+
+jobs:
+
+  create-assets:
+    name: asset-builds
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout Repository
+        uses: actions/checkout@v4
+
+      - name: Setup Node.js Environment
+        uses: actions/setup-node@v4
+        with:
+          node-version: "20"
+          cache: "npm"
+
+      - name: Install Dependencies
+        run: npm ci
+
+      - name: Build
+        run: |
+          touch .env
+          make build-ci
+      
+      - name: Build with cached packs
+        if: ${{ env.BUILD_EXIT_CODE == '5' }}
+        uses: ./.github/actions/build-cached-packs
+        with:
+          gh-token: ${{ secrets.GITHUB_TOKEN }}
+
+      - name: Upload Build Packs
+        uses: actions/upload-artifact@v4
+        with:
+          name: "build-packs"
+          path: |
+            build/packs
+            .docusaurus/packs-integrations
+          if-no-files-found: error
+          retention-days: 7
diff --git a/.github/workflows/pull_request.yaml b/.github/workflows/pull_request.yaml
@@ -96,4 +96,11 @@ jobs:
 
       - name: Build
         run: |
-          npm run build
+          touch .env
+          make build-ci
+      
+      - name: Build with cached packs
+        if: ${{ env.BUILD_EXIT_CODE == '5' }}
+        uses: ./.github/actions/build-cached-packs
+        with:
+          gh-token: ${{ secrets.GITHUB_TOKEN }}
diff --git a/.github/workflows/release-branch-pr.yaml b/.github/workflows/release-branch-pr.yaml
@@ -61,4 +61,13 @@ jobs:
             🤖 Netlify configured to enable preview build for branch:  ${{env.GITHUB_BRANCH}} . Subsequent commits will automatically trigger a Netlify build preview.
         refresh-message-position: false
 
-    - run: npm run build
+    - name: Build
+      run: |
+        touch .env
+        make build
+      
+    - name: Build with cached packs
+      if: ${{ env.BUILD_EXIT_CODE == '5' }}
+      uses: ./.github/actions/build-cached-packs
+      with:
+        gh-token: ${{ secrets.GITHUB_TOKEN }}
diff --git a/.github/workflows/release-preview.yaml b/.github/workflows/release-preview.yaml
@@ -48,17 +48,25 @@ jobs:
           echo "User-agent: *" >> static/robots.txt
           echo "Disallow: /" >> static/robots.txt
           echo "Sitemap: https://docs-latest.spectrocloud.com/sitemap.xml" >> static/robots.txt
-        
 
       - name: Ensure noIndex is set
         id: check_noindex
         run: node scripts/noindex_docusaurus_config.js $PWD
-
-
+
       - name: Build
         run: |
+          set +e  # Disable automatic stop on command failure
           touch .env
           make build
+          exit_code=$?
+          echo "Build command exit code: $exit_code"
+          echo "BUILD_EXIT_CODE=$exit_code">> $GITHUB_ENV
+      
+      - name: Build with cached packs
+        if: ${{ env.BUILD_EXIT_CODE == '5' }}
+        uses: ./.github/actions/build-cached-packs
+        with:
+          gh-token: ${{ secrets.GITHUB_TOKEN }}
 
       - name: Deploy Preview
         run: |

diff --git a/.github/workflows/release.yaml b/.github/workflows/release.yaml
@@ -54,12 +54,18 @@ jobs:
           cache: "npm"
 
       - run: npm ci
-
-      - name: Compile
+      
+      - name: Build
         run: |
           touch .env
           make versions-ci
-          make build
+          make build-ci
+      
+      - name: Build with cached packs
+        if: ${{ env.BUILD_EXIT_CODE == '5' }}
+        uses: ./.github/actions/build-cached-packs
+        with:
+          gh-token: ${{ secrets.GITHUB_TOKEN }}
 
       - name: Upload to AWS
         run: |
@@ -90,10 +96,20 @@ jobs:
 
       - run: npm ci
 
-      - name: Compile
+      - name: Versions
+        run: |
+      
+      - name: Build
         run: |
+          touch .env
           make versions-ci
-          make build
+          make build-ci
+      
+      - name: Build with cached packs
+        if: ${{ env.BUILD_EXIT_CODE == '5' }}
+        uses: ./.github/actions/build-cached-packs
+        with:
+          gh-token: ${{ secrets.GITHUB_TOKEN }}
 
       - name: Upload to AWS
         run: |

diff --git a/.github/workflows/screenshot_capture.yaml b/.github/workflows/screenshot_capture.yaml
@@ -44,8 +44,16 @@ jobs:
       - name: Install Playwright browsers
         run: npx playwright install --with-deps chromium
 
-      - name: Build Website
-        run: make build
+      - name: Build
+        run: |
+          touch .env
+          make build-ci
+      
+      - name: Build with cached packs
+        if: ${{ env.BUILD_EXIT_CODE == '5' }}
+        uses: ./.github/actions/build-cached-packs
+        with:
+          gh-token: ${{ secrets.GITHUB_TOKEN }}
 
       - name: Upload Build
         uses: actions/upload-artifact@v4

diff --git a/.github/workflows/visual-comparison.yaml b/.github/workflows/visual-comparison.yaml
@@ -72,8 +72,16 @@ jobs:
             attempt_limit: 3
             attempt_delay: 60000 # 1 minute
 
-      - name: Build Website
-        run: make build
+      - name: Build
+        run: |
+          touch .env
+          make build-ci
+      
+      - name: Build with cached packs
+        if: ${{ env.BUILD_EXIT_CODE == '5' }}
+        uses: ./.github/actions/build-cached-packs
+        with:
+          gh-token: ${{ secrets.GITHUB_TOKEN }}
 
       - name: Upload Build
         uses: actions/upload-artifact@v4

diff --git a/Makefile b/Makefile
@@ -54,6 +54,7 @@ clean-versions: ## Clean Docusarus content versions
 clean-packs: ## Clean supplemental packs and pack images
 	rm -rf static/img/packs
 	rm -rf .docusaurus/packs-integrations/api_pack_response.json
+	rm -rf .docusaurus/packs-integrations/api_repositories_response.json
 
 clean-api: ## Clean API docs
 	@echo "cleaning api docs"
@@ -81,12 +82,53 @@ start: ## Start a local development server
 	make generate-partials
 	npm run start
 
+start-cached-packs: ## Start a local development server with cached packs retry.
+	make generate-partials
+	@{ \
+		npm run start; \
+		exit_code=$$?; \
+		if [ "$$exit_code" = "5" ]; then \
+			echo "❌ Start has failed due to missing packs data..."; \
+			echo "ℹ️ Initializing fetch cached packs data..."; \
+			make get-cached-packs; \
+			echo "ℹ️ Retrying start... "; \
+			npm run start;\
+		fi; \
+	}
+
 build: ## Run npm build
 	@echo "building site"
 	npm run clear
 	rm -rf build
 	npm run build
 
+build-cached-packs: ## Run npm build with cached packs retry
+	@echo "building site"
+	npm run clear
+	rm -rf build
+	@{ \
+		npm run build; \
+		exit_code=$$?; \
+		if [ "$$exit_code" = "5" ]; then \
+			echo "❌ Build has failed due to missing packs data..."; \
+			echo "ℹ️ Initializing fetch cached packs data..."; \
+			make get-cached-packs; \
+			echo "ℹ️ Retrying build... "; \
+			npm run build;\
+		fi; \
+	}
+
+build-ci: ## Run npm build in CI environment
+	@echo "building site"
+	npm run clear
+	rm -rf build
+	@{ \
+		npm run build; \
+		exit_code=$$?; \
+		echo "Build exited with code $$exit_code..."; \
+		echo "BUILD_EXIT_CODE=$$exit_code" >> $(GITHUB_ENV); \
+	}
+
 versions: ## Create Docusarus content versions
 	@echo "creating versions"
 	./scripts/versions.sh $(TMPDIR)
@@ -230,6 +272,11 @@ find-unused-images:
 
 generate-partials: ## Generate
 	./scripts/generate-partials.sh
+
+###@ Fetch cached packs assets.
+
+get-cached-packs:
+	./scripts/get-cached-packs.sh
 
 ###@ Aloglia Indexing