Merge pull request #5359 from onflow/tarak/new-cd-images-v0.33

Backport PR 5254

.github/workflows/builds.yml

@@ -1,6 +1,6 @@
+name: Build Node Docker Images
 # This workflow is used to build and push one-off images for specific node types. This is useful
 # when deploying hotfixes or any time a change is not needed for all node roles.
-name: Build Node Docker Images
 
 on:
   workflow_dispatch:
@@ -38,9 +38,10 @@ on:
         type: boolean
         description: 'Observer'
         required: false
-      include_without_netgo:
+      # GHA allows only up to 10 inputs - regroup two entries in one
+      include_alternative_builds:
         type: boolean
-        description: 'Build `without_netgo` images'
+        description: 'Build amd64 `without_adx` and `without_netgo_without_adx` images, and arm64 images'
         required: false
 
 jobs:
@@ -111,17 +112,19 @@ jobs:
       run: |
         gcloud auth configure-docker
 
-    - name: Build/Push ${{ matrix.role }} images
+    - name: Build/Push ${{ matrix.role }} amd64 images with adx (default)
       env:
         IMAGE_TAG: ${{ inputs.docker_tag }}
         CADENCE_DEPLOY_KEY: ${{ secrets.CADENCE_DEPLOY_KEY }}
       run: |
-        make docker-build-${{ matrix.role }} docker-push-${{ matrix.role }}
+        make docker-build-${{ matrix.role }}-with-adx docker-push-${{ matrix.role }}-with-adx
 
-    - name: Build/Push ${{ matrix.role }} without_netgo images
-      if: ${{ inputs.include_without_netgo }}
+    - name: Build/Push ${{ matrix.role }} amd64 images without netgo and without adx, arm64 images
+      if: ${{ inputs.include_alternative_builds }}
       env:
         IMAGE_TAG: ${{ inputs.docker_tag }}
         CADENCE_DEPLOY_KEY: ${{ secrets.CADENCE_DEPLOY_KEY }}
       run: |
-        make docker-build-${{ matrix.role }}-without-netgo docker-push-${{ matrix.role }}-without-netgo
+        make docker-build-${{ matrix.role }}-without-adx docker-push-${{ matrix.role }}-without-adx \
+          docker-build-${{ matrix.role }}-without-netgo-without-adx docker-push-${{ matrix.role }}-without-netgo-without-adx \
+          docker-cross-build-${{ matrix.role }}-arm docker-push-${{ matrix.role }}-arm

.github/workflows/cd.yml

@@ -33,9 +33,13 @@ jobs:
       env:
         CADENCE_DEPLOY_KEY: ${{ secrets.CADENCE_DEPLOY_KEY }}
       run: |
-        make docker-build-flow
-        make docker-build-flow-without-netgo
+        make docker-build-flow-with-adx
+        make docker-build-flow-without-adx
+        make docker-build-flow-without-netgo-without-adx
+        make docker-cross-build-flow-arm
     - name: Docker push
       run: |
-        make docker-push-flow
-        make docker-push-flow-without-netgo
+        make docker-push-flow-with-adx
+        make docker-push-flow-without-adx
+        make docker-push-flow-without-netgo-without-adx
+        make docker-push-flow-arm

.github/workflows/ci.yml

@@ -189,7 +189,7 @@ jobs:
       - name: Docker build
         env:
           CADENCE_DEPLOY_KEY: ${{ secrets.CADENCE_DEPLOY_KEY }}
-        run: make docker-build-flow docker-build-flow-corrupt
+        run: make docker-native-build-flow docker-native-build-flow-corrupt
       - name: Save Docker images
         run: |
           docker save \

.github/workflows/flaky-test-monitor.yml

@@ -165,7 +165,7 @@ jobs:
           go-version: ${{ env.GO_VERSION }}
           cache: true
       - name: Docker build
-        run: make docker-build-flow docker-build-flow-corrupt
+        run: make docker-native-build-flow docker-native-build-flow-corrupt
       - name: Run tests
         run: make -es -C integration ${{ matrix.target }} > test-output
         timeout-minutes: 100

README.md

@@ -100,13 +100,13 @@ The recommended way to build and run Flow for local development is using Docker.
 Build a Docker image for all nodes:
 
 ```bash
-make docker-build-flow
+make docker-native-build-flow
 ```
 
 Build a Docker image for a particular node role (replace `$ROLE` with `collection`, `consensus`, etc.):
 
 ```bash
-make docker-build-$ROLE
+make docker-native-build-$ROLE
 ```
 
 ### Local Network

cmd/Dockerfile

@@ -6,7 +6,7 @@
 FROM golang:1.20-bullseye AS build-setup
 
 RUN apt-get update
-RUN apt-get -y install zip
+RUN apt-get -y install zip apt-utils gcc-aarch64-linux-gnu
 
 ## (2) Setup crypto dependencies
 FROM build-setup AS build-env
@@ -27,7 +27,7 @@ COPY . .
 RUN git config --global [email protected]:.insteadOf https://github.com/
 RUN mkdir ~/.ssh
 
-# Add GitHub known host
+# Add GitHub known host to avoid prompts or failures on key check
 RUN ssh-keyscan -t rsa github.com >> ~/.ssh/known_hosts
 
 ####################################
@@ -36,19 +36,22 @@ FROM build-env as build-production
 WORKDIR /app
 
 ARG GOARCH=amd64
-
 # TAGS can be overriden to modify the go build tags (e.g. build without netgo)
 ARG TAGS="netgo"
-# CGO_FLAG can be overwritten
-ARG CGO_FLAG
+# CC flag can be overwritten to specify a C compiler
+ARG CC=""
+# CGO_FLAG uses ADX instructions by default, flag can be overwritten to build without ADX
+ARG CGO_FLAG=""
 
 # Keep Go's build cache between builds.
 # https://github.com/golang/go/issues/27719#issuecomment-514747274
 RUN --mount=type=cache,sharing=locked,target=/go/pkg/mod \
     --mount=type=cache,target=/root/.cache/go-build \
     --mount=type=secret,id=cadence_deploy_key \
+    # We evaluate the SSH agent to safely pass in a key for cloning dependencies
+    # We explicitly use ";" rather than && as we want to safely pass if it is unavailable
     eval `ssh-agent -s` && printf "%s\n" "$(cat /run/secrets/cadence_deploy_key)" | ssh-add - ; \
-    CGO_ENABLED=1 GOOS=linux CGO_FLAGS="${CGO_FLAG}" go build --tags "${TAGS}" -ldflags "-extldflags -static \
+    CGO_ENABLED=1 GOOS=linux GOARCH=${GOARCH} CC="${CC}" CGO_FLAGS="${CGO_FLAG}" go build --tags "${TAGS}" -ldflags "-extldflags -static \
     -X 'github.com/onflow/flow-go/cmd/build.commit=${COMMIT}' -X  'github.com/onflow/flow-go/cmd/build.semver=${VERSION}'" \
     -o ./app ${TARGET}
 
@@ -66,13 +69,16 @@ ENTRYPOINT ["/bin/app"]
 FROM build-env as build-debug
 WORKDIR /app
 ARG GOARCH=amd64
-
+ARG CC=""
+ARG CGO_FLAG=""
 RUN --mount=type=ssh \
     --mount=type=cache,sharing=locked,target=/go/pkg/mod \
     --mount=type=cache,target=/root/.cache/go-build \
     --mount=type=secret,id=cadence_deploy_key \
+    # We evaluate the SSH agent to safely pass in a key for cloning dependencies
+    # We explicitly use ";" rather than && as we want to safely pass if it is unavailable
     eval `ssh-agent -s` && printf "%s\n" "$(cat /run/secrets/cadence_deploy_key)" | ssh-add - ; \
-    CGO_ENABLED=1 GOOS=linux CGO_FLAGS="${CGO_FLAG}" go build --tags "netgo" -ldflags "-extldflags -static \
+    CGO_ENABLED=1 GOOS=linux GOARCH=${GOARCH} CC="${CC}" CGO_FLAGS="${CGO_FLAG}" go build --tags "netgo" -ldflags "-extldflags -static \
     -X 'github.com/onflow/flow-go/cmd/build.commit=${COMMIT}' -X  'github.com/onflow/flow-go/cmd/build.semver=${VERSION}'" \
     -gcflags="all=-N -l" -o ./app ${TARGET}
 

crypto_adx_flag.mk

@@ -1,5 +1,7 @@
-# This script can be imported by Makefiles in order to set the `CRYPTO_FLAG` automatically.
-# The `CRYPTO_FLAG` is a Go command flag that should be used when the machine's CPU executing 
+# This script can be imported by Makefiles in order to set the `CRYPTO_FLAG` automatically for 
+# a native build (build and run on the same machine NOT for cross-compilation).
+#
+# The `CRYPTO_FLAG` is a Go command flag that should be used when the target machine's CPU executing 
 # the command may not support ADX instructions.
 # For new machines that support ADX instructions, the `CRYPTO_FLAG` flag is not needed (or set
 # to an empty string).  
@@ -14,12 +16,14 @@ else
 	ADX_SUPPORT := 1
 endif
 
+DISABLE_ADX := "-O2 -D__BLST_PORTABLE__"
+
 # Then, set `CRYPTO_FLAG`
 # the crypto package uses BLST source files underneath which may use ADX instructions.
 ifeq ($(ADX_SUPPORT), 1)
 # if ADX instructions are supported on the current machine, default is to use a fast ADX implementation 
 	CRYPTO_FLAG := ""
 else
 # if ADX instructions aren't supported, this CGO flags uses a slower non-ADX implementation 
-	CRYPTO_FLAG := "-O -D__BLST_PORTABLE__"
-endif
+	CRYPTO_FLAG := $(DISABLE_ADX)
+endif

engine/execution/ingestion/stop/stop_control_test.go

@@ -865,9 +865,11 @@ func Test_StopControlWorkers(t *testing.T) {
 func TestPatchedVersion(t *testing.T) {
 	require.True(t, semver.New("0.31.20").LessThan(*semver.New("0.31.21")))
 	require.True(t, semver.New("0.31.20-patch.1").LessThan(*semver.New("0.31.20"))) // be careful with this one
-	require.True(t, semver.New("0.31.20-without-netgo").LessThan(*semver.New("0.31.20")))
+	require.True(t, semver.New("0.31.20-without-adx").LessThan(*semver.New("0.31.20")))
 
 	// a special build created with "+" would not change the version priority for standard and pre-release versions
-	require.True(t, semver.New("0.31.20+without-netgo").Equal(*semver.New("0.31.20")))
-	require.True(t, semver.New("0.31.20-patch.1+without-netgo").Equal(*semver.New("0.31.20-patch.1")))
+	require.True(t, semver.New("0.31.20+without-adx").Equal(*semver.New("0.31.20")))
+	require.True(t, semver.New("0.31.20-patch.1+without-adx").Equal(*semver.New("0.31.20-patch.1")))
+	require.True(t, semver.New("0.31.20+without-netgo-without-adx").Equal(*semver.New("0.31.20")))
+	require.True(t, semver.New("0.31.20+arm").Equal(*semver.New("0.31.20")))
 }

integration/README.md

@@ -14,15 +14,15 @@ Since the test cases run docker instances as a network of nodes, we need to ensu
 To ensure the latest docker images have been built, you can run:
 
 ```
-make docker-build-access
-make docker-build-collection
-make docker-build-consensus
-make docker-build-execution
-make docker-build-verification
-make docker-build-ghost
+make docker-native-build-access
+make docker-native-build-collection
+make docker-native-build-consensus
+make docker-native-build-execution
+make docker-native-build-verification
+make docker-native-build-ghost
 ```
 
-Or simply run `make docker-build-flow`
+Or simply run `make docker-native-build-flow`
 
 After images have been built, we can run the integration tests:
 ```
@@ -65,11 +65,11 @@ Because launching a full execution node in the consensus integration tests will
 ### Rebuild image when debugging
 During test cases debugging, you might want to update some code. However, if you run `make integration-test` after updating the code, the new change will not be included, because the integration tests still use the old code from the docker image, which was built before adding the changes.
 
-So you need to rebuild all the images by running `make docker-build-flow` again before re-running the integration tests.
+So you need to rebuild all the images by running `make docker-native-build-flow` again before re-running the integration tests.
 
 Rebuilding all images takes quite some time, here is a shortcut:
 
-If consensus's code was changed, then only consensus's image need to be rebuilt, so simply run `make docker-build-consensus` instead of rebuilding all the images.
+If consensus's code was changed, then only consensus's image need to be rebuilt, so simply run `make docker-native-build-consensus` instead of rebuilding all the images.
 
 ### Organization
 
@@ -81,4 +81,4 @@ in the Makefile.
 
 To send random transactions, for example to load test a network, run `cd integration/localnet; make load`.
 
-In order to build a docker container with the benchmarking binary, run `make docker-build-loader` from the root of this repository.
+In order to build a docker container with the benchmarking binary, run `make docker-native-build-loader` from the root of this repository.