From 4b6da0a60e467a97bdc7489ffc376e16f5b4a2bd Mon Sep 17 00:00:00 2001 From: MyroTk Date: Thu, 25 Jul 2024 15:14:49 -0700 Subject: [PATCH 1/6] enable arm integration tests --- .github/workflows/release_branches.yml | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/.github/workflows/release_branches.yml b/.github/workflows/release_branches.yml index b2a6f0bd15dc..45eeebdbc400 100644 --- a/.github/workflows/release_branches.yml +++ b/.github/workflows/release_branches.yml @@ -512,6 +512,15 @@ jobs: test_name: Integration tests (release) runner_type: altinity-on-demand, altinity-type-cpx51, altinity-in-ash, altinity-image-x86-system-ubuntu-22.04 data: ${{ needs.RunConfig.outputs.data }} + IntegrationTestsAarch64: + needs: [RunConfig, BuilderDebAarch64] + if: ${{ !failure() && !cancelled() }} + uses: ./.github/workflows/reusable_test.yml + secrets: inherit + with: + test_name: Integration tests (aarch64) + runner_type: altinity-on-demand, altinity-type-cax41, altinity-in-ash, altinity-image-arm-system-ubuntu-22.04 + data: ${{ needs.RunConfig.outputs.data }} ############################################################################################# ##################################### REGRESSION TESTS ###################################### ############################################################################################# @@ -583,6 +592,7 @@ jobs: - IntegrationTestsAsan - IntegrationTestsTsan - IntegrationTestsRelease + - IntegrationTestsAarch64 - CompatibilityCheckX86 - CompatibilityCheckAarch64 - RegressionTestsRelease From 9032980b21dd69b2cb2f0e7029c53df6924864c2 Mon Sep 17 00:00:00 2001 From: MyroTk <44327070+MyroTk@users.noreply.github.com> Date: Thu, 25 Jul 2024 17:39:27 -0700 Subject: [PATCH 2/6] Update release_branches.yml --- .github/workflows/release_branches.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/release_branches.yml b/.github/workflows/release_branches.yml index 45eeebdbc400..8a5592f77ac1 100644 --- a/.github/workflows/release_branches.yml +++ b/.github/workflows/release_branches.yml @@ -519,7 +519,7 @@ jobs: secrets: inherit with: test_name: Integration tests (aarch64) - runner_type: altinity-on-demand, altinity-type-cax41, altinity-in-ash, altinity-image-arm-system-ubuntu-22.04 + runner_type: altinity-on-demand, altinity-type-cax41, altinity-image-arm-system-ubuntu-22.04 data: ${{ needs.RunConfig.outputs.data }} ############################################################################################# ##################################### REGRESSION TESTS ###################################### From d3b44e6af32dc20d36a7dd5c95b9741046a28657 Mon Sep 17 00:00:00 2001 From: MyroTk <44327070+MyroTk@users.noreply.github.com> Date: Wed, 11 Sep 2024 01:27:28 -0700 Subject: [PATCH 3/6] Build integration images for arm --- docker/images.json | 3 --- 1 file changed, 3 deletions(-) diff --git a/docker/images.json b/docker/images.json index c94001539f31..24661bbbfd53 100644 --- a/docker/images.json +++ b/docker/images.json @@ -16,7 +16,6 @@ "dependent": [] }, "docker/test/integration/base": { - "only_amd64": true, "name": "altinityinfra/integration-test", "dependent": [] }, @@ -65,7 +64,6 @@ "dependent": [] }, "docker/test/integration/runner": { - "only_amd64": true, "name": "altinityinfra/integration-tests-runner", "dependent": [] }, @@ -88,7 +86,6 @@ "dependent": [] }, "docker/test/integration/helper_container": { - "only_amd64": true, "name": "altinityinfra/integration-helper", "dependent": [] }, From f7c7373f7ae82f87fab32b46fd9d0d9235586257 Mon Sep 17 00:00:00 2001 From: MyroTk <44327070+MyroTk@users.noreply.github.com> Date: Wed, 11 Sep 2024 01:37:32 -0700 Subject: [PATCH 4/6] Update ci.py --- tests/ci/ci.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/tests/ci/ci.py b/tests/ci/ci.py index 537f7e9442ba..8ed1adaea149 100644 --- a/tests/ci/ci.py +++ b/tests/ci/ci.py @@ -1286,7 +1286,9 @@ def _configure_docker_jobs(docker_digest_or_latest: bool) -> Dict: # find if it's possible to use the setting of /etc/docker/daemon.json (https://github.com/docker/cli/issues/4484#issuecomment-1688095463) docker_images_helper.docker_login() missing_multi_dict = check_missing_images_on_dockerhub(imagename_digest_dict) + print(imagename_digest_dict) #debug missing_multi = list(missing_multi_dict) + print(missing_multi_dict) #debug missing_amd64 = [] missing_aarch64 = [] if not docker_digest_or_latest: @@ -1306,6 +1308,7 @@ def _configure_docker_jobs(docker_digest_or_latest: bool) -> Dict: "aarch64", ) ) + print(missing_aarch64) #debug else: if missing_multi: assert False, f"Missing images [{missing_multi}], cannot proceed" From 60f16813eb04263af6823d04a8d4a702cce277e4 Mon Sep 17 00:00:00 2001 From: MyroTk <44327070+MyroTk@users.noreply.github.com> Date: Wed, 11 Sep 2024 02:30:04 -0700 Subject: [PATCH 5/6] Force integration arm rebuild --- tests/ci/ci.py | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/tests/ci/ci.py b/tests/ci/ci.py index 8ed1adaea149..38543e28034f 100644 --- a/tests/ci/ci.py +++ b/tests/ci/ci.py @@ -1285,10 +1285,9 @@ def _configure_docker_jobs(docker_digest_or_latest: bool) -> Dict: # FIXME: we need login as docker manifest inspect goes directly to one of the *.docker.com hosts instead of "registry-mirrors" : ["http://dockerhub-proxy.dockerhub-proxy-zone:5000"] # find if it's possible to use the setting of /etc/docker/daemon.json (https://github.com/docker/cli/issues/4484#issuecomment-1688095463) docker_images_helper.docker_login() - missing_multi_dict = check_missing_images_on_dockerhub(imagename_digest_dict) - print(imagename_digest_dict) #debug + # missing_multi_dict = check_missing_images_on_dockerhub(imagename_digest_dict) #REMOVEME: force clickhouse to build integration + missing_multi_dict = {'altinityinfra/integration-test' : imagename_digest_dict['altinityinfra/integration-test'], 'altinityinfra/integration-tests-runner':imagename_digest_dict['altinityinfra/integration-tests-runner'], 'altinityinfra/integration-helper':imagename_digest_dict['altinityinfra/integration-helper']} missing_multi = list(missing_multi_dict) - print(missing_multi_dict) #debug missing_amd64 = [] missing_aarch64 = [] if not docker_digest_or_latest: @@ -1308,7 +1307,6 @@ def _configure_docker_jobs(docker_digest_or_latest: bool) -> Dict: "aarch64", ) ) - print(missing_aarch64) #debug else: if missing_multi: assert False, f"Missing images [{missing_multi}], cannot proceed" From a4948315dc89d770856dbf8be484142331f59c83 Mon Sep 17 00:00:00 2001 From: MyroTk <44327070+MyroTk@users.noreply.github.com> Date: Wed, 11 Sep 2024 04:04:18 -0700 Subject: [PATCH 6/6] Update confluent-kafka --- docker/test/integration/runner/Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docker/test/integration/runner/Dockerfile b/docker/test/integration/runner/Dockerfile index 7036b68c9594..ff1aa045f064 100644 --- a/docker/test/integration/runner/Dockerfile +++ b/docker/test/integration/runner/Dockerfile @@ -68,7 +68,7 @@ RUN python3 -m pip install --no-cache-dir \ azure-storage-blob~=12.19.0\ boto3~=1.34.24 \ cassandra-driver~=3.28.0\ - confluent-kafka~=1.9.2 \ + confluent-kafka~=2.3.0 \ delta-spark~=2.3.0 \ dict2xml~=1.7.3 \ dicttoxml~=1.7.16 \