From 36ff60c864ed0baac93b1ef174759b7109314932 Mon Sep 17 00:00:00 2001 From: Silvestre Zabala Date: Wed, 22 May 2024 11:18:25 +0200 Subject: [PATCH 1/7] feat(devbox): Add Cloud MTA Build Tool to devbox # Issue For building MTAs the Cloud MTA Build Tool is required. # Fix Add it via local flake --- devbox.json | 80 ++++++++++++++++++++++-------------------- local-flake/flake.nix | 48 +++++++++++++++++++++++++ scripts/asdf2devbox.py | 6 ++-- 3 files changed, 92 insertions(+), 42 deletions(-) diff --git a/devbox.json b/devbox.json index 2498449ee0..8f60edb9e0 100644 --- a/devbox.json +++ b/devbox.json @@ -1,42 +1,46 @@ { - "packages": [ - "path:local-flake#bosh-bootloader", - "path:local-flake#app-autoscaler-cli-plugin", - "path:local-flake#log-cache-cli-plugin", - "path:local-flake#uaac", - "credhub-cli@latest", - "delve@latest", - "gh@latest", - "go-tools@latest", - "gopls@latest", - "nodejs@latest", - "nodePackages.yaml-language-server@latest", - "rubocop@latest", - "rubyPackages.solargraph@latest", - "swagger-cli@latest", - "actionlint@1.6.27", - "direnv@2.34.0", - "gnumake@4.4", - "maven@3.8.6", - "pre-commit@latest", - "bosh-cli@7.5.6", - "golangci-lint@1.57.2", - "yq-go@4.43.1", - "which@latest", - "jq@latest", - "fly@7.10.0", - "python@latest", - "cloudfoundry-cli@8.7.10", - "shellcheck@0.10.0", - "act@0.2.62", - "go@1.21.5", - "ruby@3.3.1", - "ginkgo@2.17.3", - "google-cloud-sdk@latest", - "temurin-bin-17@latest", - "bundix@latest", - "oha@latest" - ], + "packages": { + "path:local-flake#bosh-bootloader": "", + "path:local-flake#app-autoscaler-cli-plugin": "", + "path:local-flake#log-cache-cli-plugin": "", + "path:local-flake#mbt": "", + "path:local-flake#sap-piper": { + "platforms": ["x86_64-linux"] + }, + "path:local-flake#uaac": "", + "credhub-cli": "latest", + "delve": "latest", + "gh": "latest", + "go-tools": "latest", + "gopls": "latest", + "nodejs": "latest", + "nodePackages.yaml-language-server": "latest", + "rubocop": "latest", + "rubyPackages.solargraph": "latest", + "swagger-cli": "latest", + "actionlint": "1.6.27", + "direnv": "2.34.0", + "gnumake": "4.4", + "maven": "3.8.6", + "pre-commit": "latest", + "bosh-cli": "7.5.6", + "golangci-lint": "1.57.2", + "yq-go": "4.43.1", + "which": "latest", + "jq": "latest", + "fly": "7.10.0", + "python": "latest", + "cloudfoundry-cli": "8.7.10", + "shellcheck": "0.10.0", + "act": "0.2.62", + "go": "1.21.5", + "ruby": "3.3.1", + "ginkgo": "2.17.3", + "bundix": "latest", + "oha": "latest", + "google-cloud-sdk": "latest", + "temurin-bin-17": "latest" + }, "shell": { "init_hook": [ "cf install-plugin -f $(which app-autoscaler-cli-plugin)", diff --git a/local-flake/flake.nix b/local-flake/flake.nix index ff108fd774..ad882e08a5 100644 --- a/local-flake/flake.nix +++ b/local-flake/flake.nix @@ -58,6 +58,54 @@ ldflags = ["-s" "-w" "-X main.version=${version}"]; }; + mbt = nixpkgsFor.${system}.buildGoModule rec { + pname = "Cloud MTA Build Tool"; + version = "1.2.26"; + + src = nixpkgsFor.${system}.fetchFromGitHub { + owner = "SAP"; + repo = "cloud-mta-build-tool"; + rev = "v${version}"; + hash = "sha256-DKZ9Nj/sNC9dRjyiu4MKjLrIJWluYlZzUHWqEqtrNt4="; + }; + + vendorHash = "sha256-h8LPsuxvbr/aRhH1vR1fYgBot37yrfiemZTJMKj0zbk="; + + ldflags = ["-s" "-w" "-X main.Version=${version}"]; + + doCheck = false; + + postInstall = '' + pushd "$out/bin" &> /dev/null + ln -s 'cloud-mta-build-tool' 'mbt' + popd + ''; + }; + + sap-piper = nixpkgsFor.${system}.buildGoModule rec { + pname = "Project Piper"; + version = "1.360.0"; + + src = nixpkgsFor.${system}.fetchFromGitHub { + owner = "SAP"; + repo = "jenkins-library"; + rev = "v${version}"; + hash = "sha256-eCg9fgP4pIhbW8zkg6izHb6V1VrkA+4hUCNvjMYOagI="; + }; + + vendorHash = "sha256-qCoz6jGoqGYsTYumrdy4RA3LdVwaMhS+vcCK2n970eg="; + + ldflags = ["-s" "-w" "-X github.com/SAP/jenkins-library/cmd.GitTag=v${version}"]; + + doCheck = false; + + postInstall = '' + pushd "$out/bin" &> /dev/null + ln -s 'jenkins-library' 'piper' + popd + ''; + }; + uaac = nixpkgsFor.${system}.bundlerApp rec { pname = "cf-uaac"; gemdir = ./.; diff --git a/scripts/asdf2devbox.py b/scripts/asdf2devbox.py index 84604416e8..49759d6eb6 100755 --- a/scripts/asdf2devbox.py +++ b/scripts/asdf2devbox.py @@ -11,10 +11,7 @@ def get_installed_version(package): with open(os.path.join(script_dir, '..', 'devbox.json'), 'r') as f: data = json.load(f) - for pkg in data['packages']: - if pkg.startswith(f"{package}@"): - return pkg.split('@')[1] - return None + return data['packages'][package] # Read the .tool-versions file and process each line if __name__ == "__main__": @@ -27,6 +24,7 @@ def get_installed_version(package): "bosh": "bosh-cli", "cf": "cloudfoundry-cli", "concourse": "fly", + "credhub": "credhub-cli", "gcloud": "google-cloud-sdk", "golang": "go", "java": "temurin-bin-17", From 34a22f6ffd195484ea9bd1c2106bf81d8643d465 Mon Sep 17 00:00:00 2001 From: Silvestre Zabala Date: Wed, 22 May 2024 11:20:24 +0200 Subject: [PATCH 2/7] feat(devbox): Build image # Issue To switch over our CI workflows to devbox a docker image is required. # Fix Build it from the Dockerfile generated with `devbox generate dockerfile` --- .github/workflows/image.yaml | 34 +++++++++++++++++++++ ci/dockerfiles/autoscaler-devbox/Dockerfile | 18 +++++++++++ 2 files changed, 52 insertions(+) create mode 100644 ci/dockerfiles/autoscaler-devbox/Dockerfile diff --git a/.github/workflows/image.yaml b/.github/workflows/image.yaml index 0ecc385642..e04110b187 100644 --- a/.github/workflows/image.yaml +++ b/.github/workflows/image.yaml @@ -18,6 +18,40 @@ env: IMAGE_NAME: ${{ github.repository }} jobs: + build-and-push-devbox-image: + runs-on: ubuntu-latest + permissions: + contents: read + packages: write + name: Build and Push app-autoscaler-release-devbox + steps: + - name: Checkout repository + uses: actions/checkout@v4 + + - name: Log in to the Container registry + uses: docker/login-action@v3 + with: + registry: ${{ env.REGISTRY }} + username: ${{ github.actor }} + password: ${{ secrets.GITHUB_TOKEN }} + + - name: Extract metadata (tags, labels) for Docker + id: meta + uses: docker/metadata-action@v5 + with: + images: ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}-devbox + + - name: Build and push + id: build-and-push + uses: docker/build-push-action@v5 + with: + context: . + file: ci/dockerfiles/autoscaler-devbox/Dockerfile + push: ${{ github.event_name == 'push' && github.ref == 'refs/heads/main' }} + tags: ${{ steps.meta.outputs.tags }} + labels: ${{ steps.meta.outputs.labels }} + cache-from: type=registry,ref=${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}-devbox:main + build-and-push-image: runs-on: ubuntu-latest permissions: diff --git a/ci/dockerfiles/autoscaler-devbox/Dockerfile b/ci/dockerfiles/autoscaler-devbox/Dockerfile new file mode 100644 index 0000000000..e15477a733 --- /dev/null +++ b/ci/dockerfiles/autoscaler-devbox/Dockerfile @@ -0,0 +1,18 @@ +FROM jetpackio/devbox@sha256:430afb5a5c9b4ec9f56776e8fcb82fe67fa5fabba3ba67ebb1c13b17d0e8de8d + + +# Installing your devbox project +WORKDIR /code +USER root:root +RUN mkdir -p /code && chown "${DEVBOX_USER}:${DEVBOX_USER}" /code +USER ${DEVBOX_USER}:${DEVBOX_USER} +COPY --chown=${DEVBOX_USER}:${DEVBOX_USER} devbox.json devbox.json +COPY --chown=${DEVBOX_USER}:${DEVBOX_USER} devbox.lock devbox.lock + + +# Step 6: Copying local flakes directories +COPY local-flake local-flake + +RUN devbox run -- echo "Installed Packages." + +CMD ["devbox", "shell"] From 6d493161a94540804b870b403c7dd3e7d49f61d2 Mon Sep 17 00:00:00 2001 From: Silvestre Zabala Date: Wed, 22 May 2024 14:23:56 +0200 Subject: [PATCH 3/7] fix(image workflow): DRY # Issue The image name was constructed multiple times. # Fix Use `IMAGE_NAME` environment variable --- .github/workflows/image.yaml | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/.github/workflows/image.yaml b/.github/workflows/image.yaml index e04110b187..0825c689a8 100644 --- a/.github/workflows/image.yaml +++ b/.github/workflows/image.yaml @@ -15,8 +15,6 @@ on: env: REGISTRY: ghcr.io - IMAGE_NAME: ${{ github.repository }} - jobs: build-and-push-devbox-image: runs-on: ubuntu-latest @@ -24,6 +22,8 @@ jobs: contents: read packages: write name: Build and Push app-autoscaler-release-devbox + env: + IMAGE_NAME: ${{ github.repository }}-devbox steps: - name: Checkout repository uses: actions/checkout@v4 @@ -39,7 +39,7 @@ jobs: id: meta uses: docker/metadata-action@v5 with: - images: ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}-devbox + images: ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }} - name: Build and push id: build-and-push @@ -50,7 +50,7 @@ jobs: push: ${{ github.event_name == 'push' && github.ref == 'refs/heads/main' }} tags: ${{ steps.meta.outputs.tags }} labels: ${{ steps.meta.outputs.labels }} - cache-from: type=registry,ref=${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}-devbox:main + cache-from: type=registry,ref=${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}:main build-and-push-image: runs-on: ubuntu-latest @@ -64,6 +64,8 @@ jobs: matrix: image_suffix: ["tools"] name: Build and Push app-autoscaler-release-${{ matrix.image_suffix }} + env: + IMAGE_NAME: ${{ github.repository }} steps: - name: Checkout repository uses: actions/checkout@v4 From d3dd3161f34f543e5ae09a0c3f4bffea9548c8bd Mon Sep 17 00:00:00 2001 From: Silvestre Zabala Date: Wed, 22 May 2024 15:24:20 +0200 Subject: [PATCH 4/7] fix(Dockerfile): Remove step number # Issue https://github.com/jetify-com/devbox/pull/2073 # Fix Remove step number. --- ci/dockerfiles/autoscaler-devbox/Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ci/dockerfiles/autoscaler-devbox/Dockerfile b/ci/dockerfiles/autoscaler-devbox/Dockerfile index e15477a733..3a3da4b578 100644 --- a/ci/dockerfiles/autoscaler-devbox/Dockerfile +++ b/ci/dockerfiles/autoscaler-devbox/Dockerfile @@ -10,7 +10,7 @@ COPY --chown=${DEVBOX_USER}:${DEVBOX_USER} devbox.json devbox.json COPY --chown=${DEVBOX_USER}:${DEVBOX_USER} devbox.lock devbox.lock -# Step 6: Copying local flakes directories +# Copying local flakes directories COPY local-flake local-flake RUN devbox run -- echo "Installed Packages." From 2f14adc18abf50a2779a7d2f70c5f8844660fd9f Mon Sep 17 00:00:00 2001 From: Silvestre Zabala Date: Wed, 22 May 2024 15:26:14 +0200 Subject: [PATCH 5/7] fix(local-flake): Remove `piper`, rename `mbt` # Issue - `sap-piper` is not needed at the moment. - `mbt` was cryptically named. # Fix - Remove `sap-piper` - Rename `mbt` to the name of the repo --- devbox.json | 5 +---- local-flake/flake.nix | 26 +------------------------- 2 files changed, 2 insertions(+), 29 deletions(-) diff --git a/devbox.json b/devbox.json index 8f60edb9e0..65256241a5 100644 --- a/devbox.json +++ b/devbox.json @@ -3,10 +3,7 @@ "path:local-flake#bosh-bootloader": "", "path:local-flake#app-autoscaler-cli-plugin": "", "path:local-flake#log-cache-cli-plugin": "", - "path:local-flake#mbt": "", - "path:local-flake#sap-piper": { - "platforms": ["x86_64-linux"] - }, + "path:local-flake#cloud-mta-build-tool": "", "path:local-flake#uaac": "", "credhub-cli": "latest", "delve": "latest", diff --git a/local-flake/flake.nix b/local-flake/flake.nix index ad882e08a5..61804f71c9 100644 --- a/local-flake/flake.nix +++ b/local-flake/flake.nix @@ -58,7 +58,7 @@ ldflags = ["-s" "-w" "-X main.version=${version}"]; }; - mbt = nixpkgsFor.${system}.buildGoModule rec { + cloud-mta-build-tool = nixpkgsFor.${system}.buildGoModule rec { pname = "Cloud MTA Build Tool"; version = "1.2.26"; @@ -82,30 +82,6 @@ ''; }; - sap-piper = nixpkgsFor.${system}.buildGoModule rec { - pname = "Project Piper"; - version = "1.360.0"; - - src = nixpkgsFor.${system}.fetchFromGitHub { - owner = "SAP"; - repo = "jenkins-library"; - rev = "v${version}"; - hash = "sha256-eCg9fgP4pIhbW8zkg6izHb6V1VrkA+4hUCNvjMYOagI="; - }; - - vendorHash = "sha256-qCoz6jGoqGYsTYumrdy4RA3LdVwaMhS+vcCK2n970eg="; - - ldflags = ["-s" "-w" "-X github.com/SAP/jenkins-library/cmd.GitTag=v${version}"]; - - doCheck = false; - - postInstall = '' - pushd "$out/bin" &> /dev/null - ln -s 'jenkins-library' 'piper' - popd - ''; - }; - uaac = nixpkgsFor.${system}.bundlerApp rec { pname = "cf-uaac"; gemdir = ./.; From ea75773c94a8fc8eee28bf253c2c58ee45e1b186 Mon Sep 17 00:00:00 2001 From: Silvestre Zabala Date: Wed, 22 May 2024 15:27:01 +0200 Subject: [PATCH 6/7] chore(asdf2devbox): Remove `credhub` mapping # Issue `credhub` is not part of `.tool-versions`, so no mapping is needed # Fix Remove `credhub` mapping --- scripts/asdf2devbox.py | 1 - 1 file changed, 1 deletion(-) diff --git a/scripts/asdf2devbox.py b/scripts/asdf2devbox.py index 49759d6eb6..5ea7540e9f 100755 --- a/scripts/asdf2devbox.py +++ b/scripts/asdf2devbox.py @@ -24,7 +24,6 @@ def get_installed_version(package): "bosh": "bosh-cli", "cf": "cloudfoundry-cli", "concourse": "fly", - "credhub": "credhub-cli", "gcloud": "google-cloud-sdk", "golang": "go", "java": "temurin-bin-17", From f9814b2d6520df53f4a5fbe2ca9e15c8eeccc407 Mon Sep 17 00:00:00 2001 From: Silvestre Zabala Date: Thu, 23 May 2024 16:18:29 +0200 Subject: [PATCH 7/7] feat(tests): Make PostgreSQL-based tests runnable in `devbox` --- devbox.json | 7 ++- devbox.lock | 137 ++++++++++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 143 insertions(+), 1 deletion(-) diff --git a/devbox.json b/devbox.json index 65256241a5..fd6941da81 100644 --- a/devbox.json +++ b/devbox.json @@ -36,7 +36,12 @@ "bundix": "latest", "oha": "latest", "google-cloud-sdk": "latest", - "temurin-bin-17": "latest" + "temurin-bin-17": "latest", + "postgresql": "latest", + "glibcLocales": { + "version": "latest", + "platforms": ["x86_64-linux", "aarch64-linux"] + } }, "shell": { "init_hook": [ diff --git a/devbox.lock b/devbox.lock index d09b14340c..3413e2bbf7 100644 --- a/devbox.lock +++ b/devbox.lock @@ -389,6 +389,34 @@ } } }, + "glibcLocales@latest": { + "last_modified": "2024-05-12T16:19:40Z", + "resolved": "github:NixOS/nixpkgs/3281bec7174f679eabf584591e75979a258d8c40#glibcLocales", + "source": "devbox-search", + "version": "2.39-31", + "systems": { + "aarch64-linux": { + "outputs": [ + { + "name": "out", + "path": "/nix/store/7l54yzsqvvv4nl77dwqgijk4v0cdi4cp-glibc-locales-2.39-31", + "default": true + } + ], + "store_path": "/nix/store/7l54yzsqvvv4nl77dwqgijk4v0cdi4cp-glibc-locales-2.39-31" + }, + "x86_64-linux": { + "outputs": [ + { + "name": "out", + "path": "/nix/store/xx7dhm1wd49q1vmr1ix71a9pp5j33imd-glibc-locales-2.39-31", + "default": true + } + ], + "store_path": "/nix/store/xx7dhm1wd49q1vmr1ix71a9pp5j33imd-glibc-locales-2.39-31" + } + } + }, "gnumake@4.4": { "last_modified": "2024-03-18T12:38:16Z", "resolved": "github:NixOS/nixpkgs/e367f7a1fb93137af22a3908f00b9a35e2d286a7#gnumake", @@ -834,6 +862,115 @@ } } }, + "postgresql@latest": { + "last_modified": "2024-05-18T11:33:04Z", + "plugin_version": "0.0.2", + "resolved": "github:NixOS/nixpkgs/dc1b0202a60553747917379c5f1dea111a49153c#postgresql", + "source": "devbox-search", + "version": "15.6", + "systems": { + "aarch64-darwin": { + "outputs": [ + { + "name": "out", + "path": "/nix/store/hf37ss7zbnbmikwwwxqcy2qcpssd6r07-postgresql-15.6", + "default": true + }, + { + "name": "man", + "path": "/nix/store/9xfmv711wd5hynis7dxfrxxkj28mqxbv-postgresql-15.6-man", + "default": true + }, + { + "name": "doc", + "path": "/nix/store/sla8lgp7nyh0v9q3nrl18xb0zgvz9n51-postgresql-15.6-doc" + }, + { + "name": "lib", + "path": "/nix/store/9rr7271db1ar9nfhq54w4ynka139gzi9-postgresql-15.6-lib" + } + ], + "store_path": "/nix/store/hf37ss7zbnbmikwwwxqcy2qcpssd6r07-postgresql-15.6" + }, + "aarch64-linux": { + "outputs": [ + { + "name": "out", + "path": "/nix/store/n4p239nfp5ravq7l4sb5637r0kl05pjf-postgresql-15.6", + "default": true + }, + { + "name": "man", + "path": "/nix/store/1ryba5708rjdx52y911l2vjcy5al7149-postgresql-15.6-man", + "default": true + }, + { + "name": "debug", + "path": "/nix/store/y342j29cs5b26wsk0sblbsz4j61mv6q6-postgresql-15.6-debug" + }, + { + "name": "doc", + "path": "/nix/store/l5kii042xy5psx3lc1gslafb148rq76f-postgresql-15.6-doc" + }, + { + "name": "lib", + "path": "/nix/store/16ivm477m09rsjq1lh23vz2g01vz82ym-postgresql-15.6-lib" + } + ], + "store_path": "/nix/store/n4p239nfp5ravq7l4sb5637r0kl05pjf-postgresql-15.6" + }, + "x86_64-darwin": { + "outputs": [ + { + "name": "out", + "path": "/nix/store/v0r2wr868mll94bmpcp2qlfs1n6hvx9r-postgresql-15.6", + "default": true + }, + { + "name": "man", + "path": "/nix/store/bph3x4zj8pd2j70drzv38g7fdpk6sqqa-postgresql-15.6-man", + "default": true + }, + { + "name": "lib", + "path": "/nix/store/l8504sv505kjnwzp393fj480h0s2jy6z-postgresql-15.6-lib" + }, + { + "name": "doc", + "path": "/nix/store/6m16izd6hfw16m9624nh7yh1c7q62h19-postgresql-15.6-doc" + } + ], + "store_path": "/nix/store/v0r2wr868mll94bmpcp2qlfs1n6hvx9r-postgresql-15.6" + }, + "x86_64-linux": { + "outputs": [ + { + "name": "out", + "path": "/nix/store/fjmy8i4daxjb6sxnsnwfbni00z08xs7r-postgresql-15.6", + "default": true + }, + { + "name": "man", + "path": "/nix/store/k7j3cmdpjc23cxd7axkc8j0yvq5cfhw5-postgresql-15.6-man", + "default": true + }, + { + "name": "debug", + "path": "/nix/store/a40v9ac0xrjvdcxrnhm1fyvnj411vj9c-postgresql-15.6-debug" + }, + { + "name": "doc", + "path": "/nix/store/zwkbfv20fswsf8ralxbc51k0qx2bsf63-postgresql-15.6-doc" + }, + { + "name": "lib", + "path": "/nix/store/b5y6zhpjgjbxdq4v7c8bk4hrhhs6f19m-postgresql-15.6-lib" + } + ], + "store_path": "/nix/store/fjmy8i4daxjb6sxnsnwfbni00z08xs7r-postgresql-15.6" + } + } + }, "pre-commit@latest": { "last_modified": "2024-03-17T01:03:25Z", "resolved": "github:NixOS/nixpkgs/299d4668ba61600311553920d9fd9c102145b2cb#pre-commit",