From 76be61c624f6fb1aa7ba69355cf4511113ca2abf Mon Sep 17 00:00:00 2001 From: Ilya Golshtein Date: Tue, 30 May 2023 10:30:27 +0000 Subject: [PATCH 01/72] arm_22.8.15: BuilderDebAarch64 uncommented in release_branches.yml --- .github/workflows/release_branches.yml | 96 +++++++++++++------------- 1 file changed, 48 insertions(+), 48 deletions(-) diff --git a/.github/workflows/release_branches.yml b/.github/workflows/release_branches.yml index a92f536797e0..5263b8d3b8aa 100644 --- a/.github/workflows/release_branches.yml +++ b/.github/workflows/release_branches.yml @@ -170,56 +170,56 @@ jobs: # shellcheck disable=SC2046 docker rm -f $(docker ps -a -q) ||: sudo rm -fr "$TEMP_PATH" "$CACHES_PATH" - # BuilderDebAarch64: - # needs: [DockerHubPush] - # runs-on: [self-hosted, builder] - # steps: - # - name: Set envs - # run: | - # cat >> "$GITHUB_ENV" << 'EOF' - # TEMP_PATH=${{runner.temp}}/build_check - # IMAGES_PATH=${{runner.temp}}/images_path - # REPO_COPY=${{runner.temp}}/build_check/ClickHouse - # CACHES_PATH=${{runner.temp}}/../ccaches - # BUILD_NAME=package_aarch64 - # EOF - # - name: Download changed images - # uses: actions/download-artifact@v2 - # with: - # name: changed_images - # path: ${{ runner.temp }}/images_path - # - name: Check out repository code - # uses: ClickHouse/checkout@v1 - # with: - # fetch-depth: 0 # otherwise we will have no info about contributors - # - name: Build - # run: | - # git -C "$GITHUB_WORKSPACE" submodule sync - # git -C "$GITHUB_WORKSPACE" submodule update --depth=1 --recursive --init --jobs=10 - # sudo rm -fr "$TEMP_PATH" - # mkdir -p "$TEMP_PATH" - # cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" - # cd "$REPO_COPY/tests/ci" && python3 build_check.py "$BUILD_NAME" - # - name: Upload build URLs to artifacts - # uses: actions/upload-artifact@v2 - # with: - # name: ${{ env.BUILD_URLS }} - # path: ${{ runner.temp }}/build_check/${{ env.BUILD_URLS }}.json - # - name: Cleanup - # if: always() - # run: | - # # shellcheck disable=SC2046 - # docker kill $(docker ps -q) ||: - # # shellcheck disable=SC2046 - # docker rm -f $(docker ps -a -q) ||: - # sudo rm -fr "$TEMP_PATH" "$CACHES_PATH" + BuilderDebAarch64: + needs: [DockerHubPush] + runs-on: [self-hosted, builder] + steps: + - name: Set envs + run: | + cat >> "$GITHUB_ENV" << 'EOF' + TEMP_PATH=${{runner.temp}}/build_check + IMAGES_PATH=${{runner.temp}}/images_path + REPO_COPY=${{runner.temp}}/build_check/ClickHouse + CACHES_PATH=${{runner.temp}}/../ccaches + BUILD_NAME=package_aarch64 + EOF + - name: Download changed images + uses: actions/download-artifact@v2 + with: + name: changed_images + path: ${{ runner.temp }}/images_path + - name: Check out repository code + uses: ClickHouse/checkout@v1 + with: + fetch-depth: 0 # otherwise we will have no info about contributors + - name: Build + run: | + git -C "$GITHUB_WORKSPACE" submodule sync + git -C "$GITHUB_WORKSPACE" submodule update --depth=1 --recursive --init --jobs=10 + sudo rm -fr "$TEMP_PATH" + mkdir -p "$TEMP_PATH" + cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" + cd "$REPO_COPY/tests/ci" && python3 build_check.py "$BUILD_NAME" + - name: Upload build URLs to artifacts + uses: actions/upload-artifact@v2 + with: + name: ${{ env.BUILD_URLS }} + path: ${{ runner.temp }}/build_check/${{ env.BUILD_URLS }}.json + - name: Cleanup + if: always() + run: | + # shellcheck disable=SC2046 + docker kill $(docker ps -q) ||: + # shellcheck disable=SC2046 + docker rm -f $(docker ps -a -q) ||: + sudo rm -fr "$TEMP_PATH" "$CACHES_PATH" ############################################################################################ ##################################### Docker images ####################################### ############################################################################################ DockerServerImages: needs: - BuilderDebRelease - # - BuilderDebAarch64 + - BuilderDebAarch64 runs-on: [self-hosted, style-checker] steps: - name: Check out repository code @@ -248,7 +248,7 @@ jobs: BuilderReport: needs: - BuilderDebRelease - # - BuilderDebAarch64 + - BuilderDebAarch64 runs-on: [self-hosted, style-checker] if: ${{ success() || failure() }} steps: @@ -331,7 +331,7 @@ jobs: # - BuilderBinDarwin # - BuilderBinDarwinAarch64 - BuilderDebRelease - # - BuilderDebAarch64 + - BuilderDebAarch64 runs-on: [self-hosted, style-checker] steps: - name: Check out repository code @@ -911,7 +911,7 @@ jobs: ./*/_instances/*/logs/*.log ./*/*/_instances/*/logs/*.log ./*/*/_instances/*.log - + tiered_storage_s3: strategy: fail-fast: false @@ -981,7 +981,7 @@ jobs: ./*/_instances/*/logs/*.log ./*/*/_instances/*/logs/*.log ./*/*/_instances/*.log - + FinishCheck: needs: - DockerHubPush From 979bada956e0a7280ad45e2e7f02d211ab1c6e34 Mon Sep 17 00:00:00 2001 From: Ilya Golshtein Date: Tue, 30 May 2023 12:31:41 +0000 Subject: [PATCH 02/72] arm_22.8.15: uncommented DockerHubPushAarch64 --- .github/workflows/release_branches.yml | 90 +++++++++++++------------- 1 file changed, 45 insertions(+), 45 deletions(-) diff --git a/.github/workflows/release_branches.yml b/.github/workflows/release_branches.yml index 5263b8d3b8aa..1eb3c0d3ae4d 100644 --- a/.github/workflows/release_branches.yml +++ b/.github/workflows/release_branches.yml @@ -26,22 +26,22 @@ on: # yamllint disable-line rule:truthy - 'releases/22.8**' jobs: - # DockerHubPushAarch64: - # runs-on: [self-hosted, style-checker-aarch64] - # steps: - # - name: Check out repository code - # uses: ClickHouse/checkout@v1 - # - name: Images check - # run: | - # cd "$GITHUB_WORKSPACE/tests/ci" - # python3 docker_images_check.py --suffix aarch64 - # - name: Upload images files to artifacts - # uses: actions/upload-artifact@v2 - # with: - # name: changed_images_aarch64 - # path: ${{ runner.temp }}/docker_images_check/changed_images_aarch64.json - # Former DockerHubPushAmd64 - DockerHubPush: + DockerHubPushAarch64: + runs-on: [self-hosted, style-checker-aarch64] + steps: + - name: Check out repository code + uses: ClickHouse/checkout@v1 + - name: Images check + run: | + cd "$GITHUB_WORKSPACE/tests/ci" + python3 docker_images_check.py --suffix aarch64 + - name: Upload images files to artifacts + uses: actions/upload-artifact@v3 + with: + name: changed_images_aarch64 + path: ${{ runner.temp }}/docker_images_check/changed_images_aarch64.json + # temporary DockerHubPush: + DockerHubPushAmd64: runs-on: [self-hosted, style-checker] steps: - name: Check out repository code @@ -54,39 +54,39 @@ jobs: python3 docker_images_check.py --suffix amd64 # TODO(vnemkov): remove this step if you uncomment DockerHubPushAarch64 and DockerHubPush below. # The rest of the pipeline expects changed_images.json, which was generated by previous version of DockerHubPush. - - name: Rename artifact - run: | - mv ${{ runner.temp }}/docker_images_check/changed_images_amd64.json ${{ runner.temp }}/docker_images_check/changed_images.json + # - name: Rename artifact + # run: | + # mv ${{ runner.temp }}/docker_images_check/changed_images_amd64.json ${{ runner.temp }}/docker_images_check/changed_images.json - name: Upload images files to artifacts uses: actions/upload-artifact@v3 + with: + name: changed_images_amd64 + path: ${{ runner.temp }}/docker_images_check/changed_images_amd64.json + DockerHubPush: + needs: [DockerHubPushAmd64, DockerHubPushAarch64] + runs-on: [self-hosted, style-checker] + steps: + - name: Check out repository code + uses: ClickHouse/checkout@v1 + - name: Download changed aarch64 images + uses: actions/download-artifact@v2 + with: + name: changed_images_aarch64 + path: ${{ runner.temp }} + - name: Download changed amd64 images + uses: actions/download-artifact@v2 + with: + name: changed_images_amd64 + path: ${{ runner.temp }} + # - name: Images check + # run: | + # cd "$GITHUB_WORKSPACE/tests/ci" + # python3 docker_manifests_merge.py --suffix amd64 --suffix aarch64 + - name: Upload images files to artifacts + uses: actions/upload-artifact@v2 with: name: changed_images - path: ${{ runner.temp }}/docker_images_check/changed_images.json - # DockerHubPush: - # needs: [DockerHubPushAmd64, DockerHubPushAarch64] - # runs-on: [self-hosted, style-checker] - # steps: - # - name: Check out repository code - # uses: ClickHouse/checkout@v1 - # - name: Download changed aarch64 images - # uses: actions/download-artifact@v2 - # with: - # name: changed_images_aarch64 - # path: ${{ runner.temp }} - # - name: Download changed amd64 images - # uses: actions/download-artifact@v2 - # with: - # name: changed_images_amd64 - # path: ${{ runner.temp }} - # - name: Images check - # run: | - # cd "$GITHUB_WORKSPACE/tests/ci" - # python3 docker_manifests_merge.py --suffix amd64 --suffix aarch64 - # - name: Upload images files to artifacts - # uses: actions/upload-artifact@v2 - # with: - # name: changed_images - # path: ${{ runner.temp }}/changed_images.json + path: ${{ runner.temp }}/changed_images.json CompatibilityCheck: needs: [BuilderDebRelease] runs-on: [self-hosted, style-checker] From 5b9f86b29be7180a34f56e0245967d8d29c25c7f Mon Sep 17 00:00:00 2001 From: Ilya Golshtein Date: Tue, 30 May 2023 12:36:49 +0000 Subject: [PATCH 03/72] arm_22.8.15: yaml indentation fix --- .github/workflows/release_branches.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/release_branches.yml b/.github/workflows/release_branches.yml index 1eb3c0d3ae4d..4ca6d3f72a94 100644 --- a/.github/workflows/release_branches.yml +++ b/.github/workflows/release_branches.yml @@ -331,7 +331,7 @@ jobs: # - BuilderBinDarwin # - BuilderBinDarwinAarch64 - BuilderDebRelease - - BuilderDebAarch64 + - BuilderDebAarch64 runs-on: [self-hosted, style-checker] steps: - name: Check out repository code From aa5d0ef84b32fe6bbbd0b9b87d1c9364f30694fc Mon Sep 17 00:00:00 2001 From: Ilya Golshtein Date: Tue, 30 May 2023 22:03:12 +0000 Subject: [PATCH 04/72] arm_22.8.15: some cleanup --- .github/workflows/release_branches.yml | 21 ++++++++++++++------- 1 file changed, 14 insertions(+), 7 deletions(-) diff --git a/.github/workflows/release_branches.yml b/.github/workflows/release_branches.yml index 4ca6d3f72a94..247370328aed 100644 --- a/.github/workflows/release_branches.yml +++ b/.github/workflows/release_branches.yml @@ -31,6 +31,8 @@ jobs: steps: - name: Check out repository code uses: ClickHouse/checkout@v1 + with: + clear-repository: true - name: Images check run: | cd "$GITHUB_WORKSPACE/tests/ci" @@ -68,13 +70,15 @@ jobs: steps: - name: Check out repository code uses: ClickHouse/checkout@v1 + with: + clear-repository: true - name: Download changed aarch64 images - uses: actions/download-artifact@v2 + uses: actions/download-artifact@v3 with: name: changed_images_aarch64 path: ${{ runner.temp }} - name: Download changed amd64 images - uses: actions/download-artifact@v2 + uses: actions/download-artifact@v3 with: name: changed_images_amd64 path: ${{ runner.temp }} @@ -83,7 +87,7 @@ jobs: # cd "$GITHUB_WORKSPACE/tests/ci" # python3 docker_manifests_merge.py --suffix amd64 --suffix aarch64 - name: Upload images files to artifacts - uses: actions/upload-artifact@v2 + uses: actions/upload-artifact@v3 with: name: changed_images path: ${{ runner.temp }}/changed_images.json @@ -182,26 +186,29 @@ jobs: REPO_COPY=${{runner.temp}}/build_check/ClickHouse CACHES_PATH=${{runner.temp}}/../ccaches BUILD_NAME=package_aarch64 + CLICKHOUSE_STABLE_VERSION_SUFFIX=altinitystable EOF - name: Download changed images - uses: actions/download-artifact@v2 + uses: actions/download-artifact@v3 with: name: changed_images path: ${{ runner.temp }}/images_path + - name: Trust My Directory + run: git config --global --add safe.directory * # https://stackoverflow.com/a/71940133 - name: Check out repository code uses: ClickHouse/checkout@v1 with: + clear-repository: true + submodules: true fetch-depth: 0 # otherwise we will have no info about contributors - name: Build run: | - git -C "$GITHUB_WORKSPACE" submodule sync - git -C "$GITHUB_WORKSPACE" submodule update --depth=1 --recursive --init --jobs=10 sudo rm -fr "$TEMP_PATH" mkdir -p "$TEMP_PATH" cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" cd "$REPO_COPY/tests/ci" && python3 build_check.py "$BUILD_NAME" - name: Upload build URLs to artifacts - uses: actions/upload-artifact@v2 + uses: actions/upload-artifact@v3 with: name: ${{ env.BUILD_URLS }} path: ${{ runner.temp }}/build_check/${{ env.BUILD_URLS }}.json From bfc8f7aafa194f29da5c74ebe381e67137f16983 Mon Sep 17 00:00:00 2001 From: Ilya Golshtein Date: Wed, 31 May 2023 08:30:31 +0000 Subject: [PATCH 05/72] arm_22.8.15: binutils in binary-builder --- docker/packager/binary/Dockerfile | 2 +- tests/ci/ci_config.py | 3 ++- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/docker/packager/binary/Dockerfile b/docker/packager/binary/Dockerfile index edad920313c4..c90bb9c59eda 100644 --- a/docker/packager/binary/Dockerfile +++ b/docker/packager/binary/Dockerfile @@ -42,7 +42,7 @@ RUN add-apt-repository ppa:ubuntu-toolchain-r/test --yes \ && apt-get clean # A cross-linker for RISC-V 64 (we need it, because LLVM's LLD does not work): -RUN apt-get install binutils-riscv64-linux-gnu +RUN apt-get install --yes binutils-riscv64-linux-gnu # Architecture of the image when BuildKit/buildx is used ARG TARGETARCH diff --git a/tests/ci/ci_config.py b/tests/ci/ci_config.py index a0fafebb79f1..21325d81b611 100644 --- a/tests/ci/ci_config.py +++ b/tests/ci/ci_config.py @@ -180,7 +180,8 @@ }, "builds_report_config": { "ClickHouse build check": [ - "package_release" + "package_release", + "package_aarch64" ], "ClickHouse special build check": [ "binary_tidy", From a7c6bee46fbd7075f30d6636b0adcc2375404c58 Mon Sep 17 00:00:00 2001 From: Ilya Golshtein Date: Sun, 4 Jun 2023 00:03:25 +0000 Subject: [PATCH 06/72] arm_22.8.15: docker_manifests_merge --- .github/workflows/release_branches.yml | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/.github/workflows/release_branches.yml b/.github/workflows/release_branches.yml index 247370328aed..a831b9cb8cbc 100644 --- a/.github/workflows/release_branches.yml +++ b/.github/workflows/release_branches.yml @@ -82,10 +82,10 @@ jobs: with: name: changed_images_amd64 path: ${{ runner.temp }} - # - name: Images check - # run: | - # cd "$GITHUB_WORKSPACE/tests/ci" - # python3 docker_manifests_merge.py --suffix amd64 --suffix aarch64 + - name: Images check + run: | + cd "$GITHUB_WORKSPACE/tests/ci" + python3 docker_manifests_merge.py --suffix amd64 --suffix aarch64 - name: Upload images files to artifacts uses: actions/upload-artifact@v3 with: From 88d4e8a4830b9278cc50a03f58044645200b03bd Mon Sep 17 00:00:00 2001 From: MyroTk <44327070+MyroTk@users.noreply.github.com> Date: Wed, 28 Jun 2023 09:36:00 -0700 Subject: [PATCH 07/72] Updating labels --- .github/workflows/release_branches.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/release_branches.yml b/.github/workflows/release_branches.yml index a831b9cb8cbc..e25dc5126476 100644 --- a/.github/workflows/release_branches.yml +++ b/.github/workflows/release_branches.yml @@ -27,7 +27,7 @@ on: # yamllint disable-line rule:truthy jobs: DockerHubPushAarch64: - runs-on: [self-hosted, style-checker-aarch64] + runs-on: [self-hosted, altinity-style-checker-aarch64] steps: - name: Check out repository code uses: ClickHouse/checkout@v1 @@ -176,7 +176,7 @@ jobs: sudo rm -fr "$TEMP_PATH" "$CACHES_PATH" BuilderDebAarch64: needs: [DockerHubPush] - runs-on: [self-hosted, builder] + runs-on: [self-hosted, altinity-builder] steps: - name: Set envs run: | From afa8d96b507f49b1064b56a1a196485fed99fcb9 Mon Sep 17 00:00:00 2001 From: MyroTk <44327070+MyroTk@users.noreply.github.com> Date: Wed, 28 Jun 2023 09:48:40 -0700 Subject: [PATCH 08/72] Fixing merge --- .github/workflows/release_branches.yml | 42 ++++++++++++++++++++++++++ 1 file changed, 42 insertions(+) diff --git a/.github/workflows/release_branches.yml b/.github/workflows/release_branches.yml index 9ce114b0fb7b..3f8a6079ecf2 100644 --- a/.github/workflows/release_branches.yml +++ b/.github/workflows/release_branches.yml @@ -991,6 +991,47 @@ jobs: ./*/*/_instances/*/logs/*.log ./*/*/_instances/*.log + SignRelease: + needs: [BuilderDebRelease] + runs-on: [ self-hosted ] + steps: + - name: Set envs + run: | + cat >> "$GITHUB_ENV" << 'EOF' + TEMP_PATH=${{runner.temp}}/signed + REPORTS_PATH=${{runner.temp}}/reports_dir + EOF + - name: Clear repository + run: | + sudo rm -fr "$GITHUB_WORKSPACE" && mkdir "$GITHUB_WORKSPACE" + - name: Check out repository code + uses: actions/checkout@v2 + - name: Download json reports + uses: actions/download-artifact@v2 + with: + path: ${{ env.REPORTS_PATH }} + - name: Sign release + env: + GPG_BINARY_SIGNING_KEY: ${{ secrets.GPG_BINARY_SIGNING_KEY }} + GPG_BINARY_SIGNING_PASSPHRASE: ${{ secrets.GPG_BINARY_SIGNING_PASSPHRASE }} + REPORTS_PATH: ${{ env.REPORTS_PATH }} + run: | + cd "$GITHUB_WORKSPACE/tests/ci" + python3 sign_release.py + - name: Upload signed hashes + uses: actions/upload-artifact@v2 + with: + name: signed-hashes + path: ${{ env.TEMP_PATH }}/*.gpg + - name: Cleanup + if: always() + run: | + docker ps --quiet | xargs --no-run-if-empty docker kill ||: + docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: + sudo rm -fr "$TEMP_PATH" + ########################################################################################### + ################################ FINISH CHECK ############################################# + ########################################################################################### FinishCheck: needs: - DockerHubPush @@ -1011,6 +1052,7 @@ jobs: - parquet - s3 - tiered_storage_s3 + - SignRelease runs-on: [self-hosted, style-checker] steps: - name: Check out repository code From 7a6d098318cb2c2a25d78bdf965073ecbea77828 Mon Sep 17 00:00:00 2001 From: Ilya Golshtein Date: Wed, 5 Jul 2023 22:20:04 +0000 Subject: [PATCH 09/72] arm_22.8.15: build_check.py temporary fix --- tests/ci/build_check.py | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/tests/ci/build_check.py b/tests/ci/build_check.py index a0315776421e..1a839729e079 100644 --- a/tests/ci/build_check.py +++ b/tests/ci/build_check.py @@ -370,22 +370,26 @@ def main(): log_path, s3_path_prefix + "/" + os.path.basename(log_path) ) logging.info("Log url %s", log_url) + print(f"::notice ::Log URL: {log_url}") else: logging.info("Build log doesn't exist") + print("Build log doesn't exist") - print(f"::notice ::Log URL: {log_url}") src_path = os.path.join(TEMP_PATH, "build_source.src.tar.gz") + s3_path = s3_path_prefix + "/clickhouse-" + version.string + ".src.tar.gz" + logging.info("s3_path %s", s3_path) if os.path.exists(src_path): src_url = s3_helper.upload_build_file_to_s3( - src_path, s3_path_prefix + "/clickhouse-" + version.string + ".src.tar.gz" + src_path, s3_path ) logging.info("Source tar %s", src_url) + print(f"::notice ::Source tar URL: {src_url}") else: logging.info("Source tar doesn't exist") + print("Source tar doesn't exist") - print(f"::notice ::Source tar URL: {src_url}") create_json_artifact( TEMP_PATH, build_name, log_url, build_urls, build_config, elapsed, success From 1a49b8adec0dacca2eeae9005c30f764f14274a0 Mon Sep 17 00:00:00 2001 From: Ilya Golshtein Date: Thu, 6 Jul 2023 13:24:47 +0000 Subject: [PATCH 10/72] arm_22.8.15: some checks in system_test.go are disabled --- .../internal/collectors/system/system_test.go | 30 +++++++++---------- 1 file changed, 15 insertions(+), 15 deletions(-) diff --git a/programs/diagnostics/internal/collectors/system/system_test.go b/programs/diagnostics/internal/collectors/system/system_test.go index fb1e16bd1ed3..70e79bfc905f 100644 --- a/programs/diagnostics/internal/collectors/system/system_test.go +++ b/programs/diagnostics/internal/collectors/system/system_test.go @@ -55,21 +55,21 @@ func TestSystemCollect(t *testing.T) { memoryUsageFrames, err := countFrameRows(diagSet, "memory_usage") require.Greater(t, memoryUsageFrames, 0) require.Nil(t, err) - // cpu - require.Equal(t, []string{"processor", "vendor", "model", "core", "numThreads", "logical", "capabilities"}, diagSet.Frames["cpu"].Columns()) - cpuFrames, err := countFrameRows(diagSet, "cpu") - require.Greater(t, cpuFrames, 0) - require.Nil(t, err) - // processes - require.Equal(t, []string{"pid", "ppid", "stime", "time", "rss", "size", "faults", "minorFaults", "majorFaults", "user", "state", "priority", "nice", "command"}, diagSet.Frames["processes"].Columns()) - processesFrames, err := countFrameRows(diagSet, "processes") - require.Greater(t, processesFrames, 0) - require.Nil(t, err) - // os - require.Equal(t, []string{"hostname", "os", "goOs", "cpus", "core", "kernel", "platform"}, diagSet.Frames["os"].Columns()) - osFrames, err := countFrameRows(diagSet, "os") - require.Greater(t, osFrames, 0) - require.Nil(t, err) + // // cpu + // require.Equal(t, []string{"processor", "vendor", "model", "core", "numThreads", "logical", "capabilities"}, diagSet.Frames["cpu"].Columns()) + // cpuFrames, err := countFrameRows(diagSet, "cpu") + // require.Greater(t, cpuFrames, 0) + // require.Nil(t, err) + // // processes + // require.Equal(t, []string{"pid", "ppid", "stime", "time", "rss", "size", "faults", "minorFaults", "majorFaults", "user", "state", "priority", "nice", "command"}, diagSet.Frames["processes"].Columns()) + // processesFrames, err := countFrameRows(diagSet, "processes") + // require.Greater(t, processesFrames, 0) + // require.Nil(t, err) + // // os + // require.Equal(t, []string{"hostname", "os", "goOs", "cpus", "core", "kernel", "platform"}, diagSet.Frames["os"].Columns()) + // osFrames, err := countFrameRows(diagSet, "os") + // require.Greater(t, osFrames, 0) + // require.Nil(t, err) }) } From d15fe7e768493018e30e452a2af07289b3ba9a73 Mon Sep 17 00:00:00 2001 From: MyroTk <44327070+MyroTk@users.noreply.github.com> Date: Fri, 7 Jul 2023 11:21:59 -0700 Subject: [PATCH 11/72] Update release_branches.yml --- .github/workflows/release_branches.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/release_branches.yml b/.github/workflows/release_branches.yml index 3f8a6079ecf2..a7719685e588 100644 --- a/.github/workflows/release_branches.yml +++ b/.github/workflows/release_branches.yml @@ -28,7 +28,7 @@ on: # yamllint disable-line rule:truthy jobs: DockerHubPushAarch64: - runs-on: [self-hosted, altinity-style-checker-aarch64] + runs-on: [self-hosted, style-checker-aarch64, altinity-aarch64] steps: - name: Check out repository code uses: ClickHouse/checkout@v1 @@ -178,7 +178,7 @@ jobs: sudo rm -fr "$TEMP_PATH" "$CACHES_PATH" BuilderDebAarch64: needs: [DockerHubPush] - runs-on: [self-hosted, altinity-builder] + runs-on: [self-hosted, builder-aarch64, altinity-aarch64] steps: - name: Set envs run: | From 24692d8bb703e51c3efdd1e0578954bf5f523501 Mon Sep 17 00:00:00 2001 From: Ilya Golshtein Date: Wed, 12 Jul 2023 15:43:13 +0000 Subject: [PATCH 12/72] sqlancer' master renamed to main --- docker/test/sqlancer/Dockerfile | 2 +- docker/test/sqlancer/run.sh | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/docker/test/sqlancer/Dockerfile b/docker/test/sqlancer/Dockerfile index 0821d516e239..38ced49b5199 100644 --- a/docker/test/sqlancer/Dockerfile +++ b/docker/test/sqlancer/Dockerfile @@ -10,7 +10,7 @@ RUN wget https://github.com/sqlancer/sqlancer/archive/master.zip -O /sqlancer.zi RUN mkdir /sqlancer && \ cd /sqlancer && \ unzip /sqlancer.zip -RUN cd /sqlancer/sqlancer-master && mvn package -DskipTests +RUN cd /sqlancer/sqlancer-main && mvn package -DskipTests COPY run.sh / COPY process_sqlancer_result.py / diff --git a/docker/test/sqlancer/run.sh b/docker/test/sqlancer/run.sh index a1891569d340..92e63eb50ca6 100755 --- a/docker/test/sqlancer/run.sh +++ b/docker/test/sqlancer/run.sh @@ -9,7 +9,7 @@ dpkg -i package_folder/clickhouse-client_*.deb service clickhouse-server start && sleep 5 -cd /sqlancer/sqlancer-master +cd /sqlancer/sqlancer-main export TIMEOUT=300 export NUM_QUERIES=1000 From 71c3eb34f70f4cdac8146ccdcbb3a616e544ea67 Mon Sep 17 00:00:00 2001 From: Ilya Golshtein Date: Thu, 20 Jul 2023 16:12:00 +0000 Subject: [PATCH 13/72] arm_22.8.15: attempt to enable FunctionalStatelessTestAarch64 --- .github/workflows/release_branches.yml | 78 ++++++++++++++------------ 1 file changed, 41 insertions(+), 37 deletions(-) diff --git a/.github/workflows/release_branches.yml b/.github/workflows/release_branches.yml index a7719685e588..fdb5cc5894a5 100644 --- a/.github/workflows/release_branches.yml +++ b/.github/workflows/release_branches.yml @@ -390,42 +390,46 @@ jobs: # shellcheck disable=SC2046 docker rm -f $(docker ps -a -q) ||: sudo rm -fr "$TEMP_PATH" - # FunctionalStatelessTestAarch64: - # needs: [BuilderDebAarch64] - # runs-on: [self-hosted, func-tester-aarch64] - # steps: - # - name: Set envs - # run: | - # cat >> "$GITHUB_ENV" << 'EOF' - # TEMP_PATH=${{runner.temp}}/stateless_release - # REPORTS_PATH=${{runner.temp}}/reports_dir - # CHECK_NAME=Stateless tests (aarch64) - # REPO_COPY=${{runner.temp}}/stateless_release/ClickHouse - # KILL_TIMEOUT=10800 - # EOF - # - name: Download json reports - # uses: actions/download-artifact@v3 - # with: - # path: ${{ env.REPORTS_PATH }} - # - name: Check out repository code - # uses: ClickHouse/checkout@v1 - # with: - # clear-repository: true - # - name: Functional test - # run: | - # sudo rm -fr "$TEMP_PATH" - # mkdir -p "$TEMP_PATH" - # cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" - # cd "$REPO_COPY/tests/ci" - # python3 functional_test_check.py "$CHECK_NAME" "$KILL_TIMEOUT" - # - name: Cleanup - # if: always() - # run: | - # # shellcheck disable=SC2046 - # docker kill $(docker ps -q) ||: - # # shellcheck disable=SC2046 - # docker rm -f $(docker ps -a -q) ||: - # sudo rm -fr "$TEMP_PATH" + + + FunctionalStatelessTestAarch64: + needs: [BuilderDebAarch64] + runs-on: [self-hosted, func-tester-aarch64, altinity-aarch64] + steps: + - name: Set envs + run: | + cat >> "$GITHUB_ENV" << 'EOF' + TEMP_PATH=${{runner.temp}}/stateless_release + REPORTS_PATH=${{runner.temp}}/reports_dir + CHECK_NAME=Stateless tests (aarch64) + REPO_COPY=${{runner.temp}}/stateless_release/ClickHouse + KILL_TIMEOUT=10800 + EOF + - name: Download json reports + uses: actions/download-artifact@v3 + with: + path: ${{ env.REPORTS_PATH }} + - name: Check out repository code + uses: ClickHouse/checkout@v1 + with: + clear-repository: true + - name: Functional test + run: | + sudo rm -fr "$TEMP_PATH" + mkdir -p "$TEMP_PATH" + cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" + cd "$REPO_COPY/tests/ci" + python3 functional_test_check.py "$CHECK_NAME" "$KILL_TIMEOUT" + - name: Cleanup + if: always() + run: | + # shellcheck disable=SC2046 + docker kill $(docker ps -q) ||: + # shellcheck disable=SC2046 + docker rm -f $(docker ps -a -q) ||: + sudo rm -fr "$TEMP_PATH" + + ############################################################################################## ############################ FUNCTIONAl STATEFUL TESTS ####################################### ############################################################################################## @@ -1040,7 +1044,7 @@ jobs: # - BuilderSpecialReport - MarkReleaseReady - FunctionalStatelessTestRelease - # - FunctionalStatelessTestAarch64 + - FunctionalStatelessTestAarch64 - FunctionalStatefulTestRelease # - FunctionalStatefulTestAarch64 - IntegrationTestsRelease0 From 2e286fdef734ab4545d2a7b1999d7d1e439cb514 Mon Sep 17 00:00:00 2001 From: Ilya Golshtein Date: Wed, 26 Jul 2023 19:49:26 +0000 Subject: [PATCH 14/72] arm_22.8.15: just trigger build --- .github/workflows/release_branches.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/release_branches.yml b/.github/workflows/release_branches.yml index fdb5cc5894a5..ae8abb14f21e 100644 --- a/.github/workflows/release_branches.yml +++ b/.github/workflows/release_branches.yml @@ -374,7 +374,7 @@ jobs: - name: Check out repository code uses: ClickHouse/checkout@v1 with: - clear-repository: true + clear-repository: trueĞ» - name: Functional test run: | sudo rm -fr "$TEMP_PATH" From ea7abf434d5f99618d1e188b8284cf43835724f3 Mon Sep 17 00:00:00 2001 From: Ilya Golshtein Date: Thu, 27 Jul 2023 08:06:01 +0000 Subject: [PATCH 15/72] arm_22.8.15: typo fixed --- .github/workflows/release_branches.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/release_branches.yml b/.github/workflows/release_branches.yml index ae8abb14f21e..fdb5cc5894a5 100644 --- a/.github/workflows/release_branches.yml +++ b/.github/workflows/release_branches.yml @@ -374,7 +374,7 @@ jobs: - name: Check out repository code uses: ClickHouse/checkout@v1 with: - clear-repository: trueĞ» + clear-repository: true - name: Functional test run: | sudo rm -fr "$TEMP_PATH" From 227354069ef82edcb48c0e39fe70388ecefe92ed Mon Sep 17 00:00:00 2001 From: MyroTk <44327070+MyroTk@users.noreply.github.com> Date: Wed, 2 Aug 2023 12:57:25 -0700 Subject: [PATCH 16/72] Update release_branches.yml Disable clickhouse-regression tests. --- .github/workflows/release_branches.yml | 818 ++++++++++++------------- 1 file changed, 409 insertions(+), 409 deletions(-) diff --git a/.github/workflows/release_branches.yml b/.github/workflows/release_branches.yml index fdb5cc5894a5..56232af29fd8 100644 --- a/.github/workflows/release_branches.yml +++ b/.github/workflows/release_branches.yml @@ -585,415 +585,415 @@ jobs: ############################################################################################# ##################################### REGRESSION TESTS ###################################### ############################################################################################# - regression_start: - ## Not depending on the tests above since they can fail at any given moment. - needs: [BuilderDebRelease] - runs-on: ubuntu-latest - steps: - - run: true +# regression_start: +# ## Not depending on the tests above since they can fail at any given moment. +# needs: [BuilderDebRelease] +# runs-on: ubuntu-latest +# steps: +# - run: true - regression_common: - strategy: - fail-fast: false - matrix: - SUITE: [aes_encryption, aggregate_functions, atomic_insert, base_58, clickhouse_keeper, datetime64_extended_range, disk_level_encryption, dns, example, extended_precision_data_types, kafka, kerberos, lightweight_delete, map_type, part_moves_between_shards, rbac, selects, ssl_server, tiered_storage, window_functions] - needs: [regression_start] - runs-on: [self-hosted, stress-tester] - env: - AWS_ACCESS_KEY_ID: ${{ secrets.AWS_REPORT_KEY_ID }} - AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_REPORT_SECRET_ACCESS_KEY }} - AWS_DEFAULT_REGION: ${{ secrets.AWS_REPORT_REGION }} - steps: - - name: Checkout regression repo - uses: actions/checkout@v3 - with: - repository: Altinity/clickhouse-regression - ref: releases - - name: Set envs - run: | - cat >> "$GITHUB_ENV" << 'EOF' - REPORTS_PATH=${{runner.temp}}/reports_dir - SUITE=${{ matrix.SUITE }} - artifacts=public - EOF - - name: Download json reports - uses: actions/download-artifact@v3 - with: - path: ${{ env.REPORTS_PATH }} - - name: Setup - run: .github/setup.sh - - name: Get deb url - run: python3 .github/get-deb-url.py --reports-path ${{ env.REPORTS_PATH }} --github-env $GITHUB_ENV - - name: Run ${{ env.SUITE }} suite - run: python3 - -u ${{ env.SUITE }}/regression.py - --clickhouse-binary-path ${{ env.clickhouse_binary_path }} - --test-to-end - --local - --collect-service-logs - --output classic - --parallel 1 - --attr project="$GITHUB_REPOSITORY" project.id="$GITHUB_REPOSITORY_ID" package="${{ env.clickhouse_binary_path }}" version="${{ env.version }}" user.name="$GITHUB_ACTOR" repository="https://github.com/Altinity/clickhouse-regression" commit.hash="$(git rev-parse HEAD)" job.id="$GITHUB_RUN_ID" job.url="$GITHUB_SERVER_URL/$GITHUB_REPOSITORY/actions/runs/$GITHUB_RUN_ID" arch="x86_64" - --log raw.log - - name: Create and upload logs - if: always() - run: .github/create_and_upload_logs.sh 1 - env: - artifact_s3_dir: build/v${{ env.version }}/$GITHUB_SHA - - uses: actions/upload-artifact@v3 - if: always() - with: - name: ${{ env.SUITE }}-artifacts - path: | - ./report.html - ./*.log.txt - ./*.log - ./*.html - ./*/_instances/*.log - ./*/_instances/*/logs/*.log - ./*/*/_instances/*/logs/*.log - ./*/*/_instances/*.log +# regression_common: +# strategy: +# fail-fast: false +# matrix: +# SUITE: [aes_encryption, aggregate_functions, atomic_insert, base_58, clickhouse_keeper, datetime64_extended_range, disk_level_encryption, dns, example, extended_precision_data_types, kafka, kerberos, lightweight_delete, map_type, part_moves_between_shards, rbac, selects, ssl_server, tiered_storage, window_functions] +# needs: [regression_start] +# runs-on: [self-hosted, stress-tester] +# env: +# AWS_ACCESS_KEY_ID: ${{ secrets.AWS_REPORT_KEY_ID }} +# AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_REPORT_SECRET_ACCESS_KEY }} +# AWS_DEFAULT_REGION: ${{ secrets.AWS_REPORT_REGION }} +# steps: +# - name: Checkout regression repo +# uses: actions/checkout@v3 +# with: +# repository: Altinity/clickhouse-regression +# ref: releases +# - name: Set envs +# run: | +# cat >> "$GITHUB_ENV" << 'EOF' +# REPORTS_PATH=${{runner.temp}}/reports_dir +# SUITE=${{ matrix.SUITE }} +# artifacts=public +# EOF +# - name: Download json reports +# uses: actions/download-artifact@v3 +# with: +# path: ${{ env.REPORTS_PATH }} +# - name: Setup +# run: .github/setup.sh +# - name: Get deb url +# run: python3 .github/get-deb-url.py --reports-path ${{ env.REPORTS_PATH }} --github-env $GITHUB_ENV +# - name: Run ${{ env.SUITE }} suite +# run: python3 +# -u ${{ env.SUITE }}/regression.py +# --clickhouse-binary-path ${{ env.clickhouse_binary_path }} +# --test-to-end +# --local +# --collect-service-logs +# --output classic +# --parallel 1 +# --attr project="$GITHUB_REPOSITORY" project.id="$GITHUB_REPOSITORY_ID" package="${{ env.clickhouse_binary_path }}" version="${{ env.version }}" user.name="$GITHUB_ACTOR" repository="https://github.com/Altinity/clickhouse-regression" commit.hash="$(git rev-parse HEAD)" job.id="$GITHUB_RUN_ID" job.url="$GITHUB_SERVER_URL/$GITHUB_REPOSITORY/actions/runs/$GITHUB_RUN_ID" arch="x86_64" +# --log raw.log +# - name: Create and upload logs +# if: always() +# run: .github/create_and_upload_logs.sh 1 +# env: +# artifact_s3_dir: build/v${{ env.version }}/$GITHUB_SHA +# - uses: actions/upload-artifact@v3 +# if: always() +# with: +# name: ${{ env.SUITE }}-artifacts +# path: | +# ./report.html +# ./*.log.txt +# ./*.log +# ./*.html +# ./*/_instances/*.log +# ./*/_instances/*/logs/*.log +# ./*/*/_instances/*/logs/*.log +# ./*/*/_instances/*.log - benchmark: - strategy: - fail-fast: false - matrix: - STORAGE: [minio, aws_s3, gcs] - needs: [regression_start] - runs-on: [self-hosted, stress-tester] - env: - AWS_ACCESS_KEY_ID: ${{ secrets.AWS_REPORT_KEY_ID }} - AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_REPORT_SECRET_ACCESS_KEY }} - AWS_DEFAULT_REGION: ${{ secrets.AWS_REPORT_REGION }} - steps: - - name: Checkout regression repo - uses: actions/checkout@v3 - with: - repository: Altinity/clickhouse-regression - - name: Set envs - run: | - cat >> "$GITHUB_ENV" << 'EOF' - REPORTS_PATH=${{runner.temp}}/reports_dir - SUITE=ontime_benchmark - STORAGE=/${{ matrix.STORAGE }} - artifacts=public - EOF - - name: Download json reports - uses: actions/download-artifact@v3 - with: - path: ${{ env.REPORTS_PATH }} - - name: Setup - run: .github/setup.sh - - name: Get deb url - run: python3 .github/get-deb-url.py --reports-path ${{ env.REPORTS_PATH }} --github-env $GITHUB_ENV - - name: Run ${{ env.SUITE }} suite - run: python3 - -u ${{ env.SUITE }}/benchmark.py - --clickhouse-binary-path ${{ env.clickhouse_binary_path }} - --storage ${{ matrix.STORAGE }} - --gcs-uri ${{ secrets.REGRESSION_GCS_URI }} - --gcs-key-id ${{ secrets.REGRESSION_GCS_KEY_ID }} - --gcs-key-secret ${{ secrets.REGRESSION_GCS_KEY_SECRET }} - --aws-s3-bucket ${{ secrets.REGRESSION_AWS_S3_BUCKET }} - --aws-s3-region ${{ secrets.REGRESSION_AWS_S3_REGION }} - --aws-s3-key-id ${{ secrets.REGRESSION_AWS_S3_KEY_ID }} - --aws-s3-access-key ${{ secrets.REGRESSION_AWS_S3_SECRET_ACCESS_KEY }} - --test-to-end - --local - --collect-service-logs - --output classic - --parallel 1 - --attr project="$GITHUB_REPOSITORY" project.id="$GITHUB_REPOSITORY_ID" package="${{ env.clickhouse_binary_path }}" version="${{ env.version }}" user.name="$GITHUB_ACTOR" repository="https://github.com/Altinity/clickhouse-regression" commit.hash="$(git rev-parse HEAD)" job.id="$GITHUB_RUN_ID" job.url="$GITHUB_SERVER_URL/$GITHUB_REPOSITORY/actions/runs/$GITHUB_RUN_ID" arch="x86_64" - --log raw.log - - name: Create and upload logs - if: always() - run: .github/create_and_upload_logs.sh 1 - env: - artifact_s3_dir: build/v${{ env.version }}/$GITHUB_SHA - - uses: actions/upload-artifact@v3 - if: always() - with: - name: ${{ env.SUITE }}-minio-artifacts - path: | - ./report.html - ./*.log.txt - ./*.log - ./*.html - ./*/_instances/*.log - ./*/_instances/*/logs/*.log - ./*/*/_instances/*/logs/*.log - ./*/*/_instances/*.log +# benchmark: +# strategy: +# fail-fast: false +# matrix: +# STORAGE: [minio, aws_s3, gcs] +# needs: [regression_start] +# runs-on: [self-hosted, stress-tester] +# env: +# AWS_ACCESS_KEY_ID: ${{ secrets.AWS_REPORT_KEY_ID }} +# AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_REPORT_SECRET_ACCESS_KEY }} +# AWS_DEFAULT_REGION: ${{ secrets.AWS_REPORT_REGION }} +# steps: +# - name: Checkout regression repo +# uses: actions/checkout@v3 +# with: +# repository: Altinity/clickhouse-regression +# - name: Set envs +# run: | +# cat >> "$GITHUB_ENV" << 'EOF' +# REPORTS_PATH=${{runner.temp}}/reports_dir +# SUITE=ontime_benchmark +# STORAGE=/${{ matrix.STORAGE }} +# artifacts=public +# EOF +# - name: Download json reports +# uses: actions/download-artifact@v3 +# with: +# path: ${{ env.REPORTS_PATH }} +# - name: Setup +# run: .github/setup.sh +# - name: Get deb url +# run: python3 .github/get-deb-url.py --reports-path ${{ env.REPORTS_PATH }} --github-env $GITHUB_ENV +# - name: Run ${{ env.SUITE }} suite +# run: python3 +# -u ${{ env.SUITE }}/benchmark.py +# --clickhouse-binary-path ${{ env.clickhouse_binary_path }} +# --storage ${{ matrix.STORAGE }} +# --gcs-uri ${{ secrets.REGRESSION_GCS_URI }} +# --gcs-key-id ${{ secrets.REGRESSION_GCS_KEY_ID }} +# --gcs-key-secret ${{ secrets.REGRESSION_GCS_KEY_SECRET }} +# --aws-s3-bucket ${{ secrets.REGRESSION_AWS_S3_BUCKET }} +# --aws-s3-region ${{ secrets.REGRESSION_AWS_S3_REGION }} +# --aws-s3-key-id ${{ secrets.REGRESSION_AWS_S3_KEY_ID }} +# --aws-s3-access-key ${{ secrets.REGRESSION_AWS_S3_SECRET_ACCESS_KEY }} +# --test-to-end +# --local +# --collect-service-logs +# --output classic +# --parallel 1 +# --attr project="$GITHUB_REPOSITORY" project.id="$GITHUB_REPOSITORY_ID" package="${{ env.clickhouse_binary_path }}" version="${{ env.version }}" user.name="$GITHUB_ACTOR" repository="https://github.com/Altinity/clickhouse-regression" commit.hash="$(git rev-parse HEAD)" job.id="$GITHUB_RUN_ID" job.url="$GITHUB_SERVER_URL/$GITHUB_REPOSITORY/actions/runs/$GITHUB_RUN_ID" arch="x86_64" +# --log raw.log +# - name: Create and upload logs +# if: always() +# run: .github/create_and_upload_logs.sh 1 +# env: +# artifact_s3_dir: build/v${{ env.version }}/$GITHUB_SHA +# - uses: actions/upload-artifact@v3 +# if: always() +# with: +# name: ${{ env.SUITE }}-minio-artifacts +# path: | +# ./report.html +# ./*.log.txt +# ./*.log +# ./*.html +# ./*/_instances/*.log +# ./*/_instances/*/logs/*.log +# ./*/*/_instances/*/logs/*.log +# ./*/*/_instances/*.log - ldap: - strategy: - fail-fast: false - matrix: - SUITE: [authentication, external_user_directory, role_mapping] - needs: [regression_start] - runs-on: [self-hosted, stress-tester] - env: - AWS_ACCESS_KEY_ID: ${{ secrets.AWS_REPORT_KEY_ID }} - AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_REPORT_SECRET_ACCESS_KEY }} - AWS_DEFAULT_REGION: ${{ secrets.AWS_REPORT_REGION }} - steps: - - name: Checkout regression repo - uses: actions/checkout@v3 - with: - repository: Altinity/clickhouse-regression - ref: releases - - name: Set envs - run: | - cat >> "$GITHUB_ENV" << 'EOF' - REPORTS_PATH=${{runner.temp}}/reports_dir - SUITE=ldap/${{ matrix.SUITE }} - artifacts=public - EOF - - name: Download json reports - uses: actions/download-artifact@v3 - with: - path: ${{ env.REPORTS_PATH }} - - name: Setup - run: .github/setup.sh - - name: Get deb url - run: python3 .github/get-deb-url.py --reports-path ${{ env.REPORTS_PATH }} --github-env $GITHUB_ENV - - name: Run ${{ env.SUITE }} suite - run: python3 - -u ${{ env.SUITE }}/regression.py - --clickhouse-binary-path ${{ env.clickhouse_binary_path }} - --test-to-end - --local - --collect-service-logs - --output classic - --parallel 1 - --attr project="$GITHUB_REPOSITORY" project.id="$GITHUB_REPOSITORY_ID" package="${{ env.clickhouse_binary_path }}" version="${{ env.version }}" user.name="$GITHUB_ACTOR" repository="https://github.com/Altinity/clickhouse-regression" commit.hash="$(git rev-parse HEAD)" job.id="$GITHUB_RUN_ID" job.url="$GITHUB_SERVER_URL/$GITHUB_REPOSITORY/actions/runs/$GITHUB_RUN_ID" arch="x86_64" - --log raw.log - - name: Create and upload logs - if: always() - run: .github/create_and_upload_logs.sh 1 - env: - artifact_s3_dir: build/v${{ env.version }}/$GITHUB_SHA - - uses: actions/upload-artifact@v3 - if: always() - with: - name: ldap-authentication-artifacts - path: | - ./report.html - ./*.log.txt - ./*.log - ./*.html - ./*/_instances/*.log - ./*/_instances/*/logs/*.log - ./*/*/_instances/*/logs/*.log - ./*/*/_instances/*.log +# ldap: +# strategy: +# fail-fast: false +# matrix: +# SUITE: [authentication, external_user_directory, role_mapping] +# needs: [regression_start] +# runs-on: [self-hosted, stress-tester] +# env: +# AWS_ACCESS_KEY_ID: ${{ secrets.AWS_REPORT_KEY_ID }} +# AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_REPORT_SECRET_ACCESS_KEY }} +# AWS_DEFAULT_REGION: ${{ secrets.AWS_REPORT_REGION }} +# steps: +# - name: Checkout regression repo +# uses: actions/checkout@v3 +# with: +# repository: Altinity/clickhouse-regression +# ref: releases +# - name: Set envs +# run: | +# cat >> "$GITHUB_ENV" << 'EOF' +# REPORTS_PATH=${{runner.temp}}/reports_dir +# SUITE=ldap/${{ matrix.SUITE }} +# artifacts=public +# EOF +# - name: Download json reports +# uses: actions/download-artifact@v3 +# with: +# path: ${{ env.REPORTS_PATH }} +# - name: Setup +# run: .github/setup.sh +# - name: Get deb url +# run: python3 .github/get-deb-url.py --reports-path ${{ env.REPORTS_PATH }} --github-env $GITHUB_ENV +# - name: Run ${{ env.SUITE }} suite +# run: python3 +# -u ${{ env.SUITE }}/regression.py +# --clickhouse-binary-path ${{ env.clickhouse_binary_path }} +# --test-to-end +# --local +# --collect-service-logs +# --output classic +# --parallel 1 +# --attr project="$GITHUB_REPOSITORY" project.id="$GITHUB_REPOSITORY_ID" package="${{ env.clickhouse_binary_path }}" version="${{ env.version }}" user.name="$GITHUB_ACTOR" repository="https://github.com/Altinity/clickhouse-regression" commit.hash="$(git rev-parse HEAD)" job.id="$GITHUB_RUN_ID" job.url="$GITHUB_SERVER_URL/$GITHUB_REPOSITORY/actions/runs/$GITHUB_RUN_ID" arch="x86_64" +# --log raw.log +# - name: Create and upload logs +# if: always() +# run: .github/create_and_upload_logs.sh 1 +# env: +# artifact_s3_dir: build/v${{ env.version }}/$GITHUB_SHA +# - uses: actions/upload-artifact@v3 +# if: always() +# with: +# name: ldap-authentication-artifacts +# path: | +# ./report.html +# ./*.log.txt +# ./*.log +# ./*.html +# ./*/_instances/*.log +# ./*/_instances/*/logs/*.log +# ./*/*/_instances/*/logs/*.log +# ./*/*/_instances/*.log - parquet: - needs: [regression_start] - runs-on: [self-hosted, stress-tester] - env: - AWS_ACCESS_KEY_ID: ${{ secrets.AWS_REPORT_KEY_ID }} - AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_REPORT_SECRET_ACCESS_KEY }} - AWS_DEFAULT_REGION: ${{ secrets.AWS_REPORT_REGION }} - steps: - - name: Checkout regression repo - uses: actions/checkout@v3 - with: - repository: Altinity/clickhouse-regression - ref: releases - - name: Set envs - run: | - cat >> "$GITHUB_ENV" << 'EOF' - REPORTS_PATH=${{runner.temp}}/reports_dir - SUITE=parquet - artifacts=public - EOF - - name: Download json reports - uses: actions/download-artifact@v3 - with: - path: ${{ env.REPORTS_PATH }} - - name: Setup - run: .github/setup.sh - - name: Get deb url - run: python3 .github/get-deb-url.py --reports-path ${{ env.REPORTS_PATH }} --github-env $GITHUB_ENV - - name: Run ${{ env.SUITE }} suite - run: python3 - -u ${{ env.SUITE }}/regression.py - --clickhouse-binary-path ${{ env.clickhouse_binary_path }} - --test-to-end - --local - --collect-service-logs - --output classic - --parallel 1 - --attr project="$GITHUB_REPOSITORY" project.id="$GITHUB_REPOSITORY_ID" package="${{ env.clickhouse_binary_path }}" version="${{ env.version }}" user.name="$GITHUB_ACTOR" repository="https://github.com/Altinity/clickhouse-regression" commit.hash="$(git rev-parse HEAD)" job.id="$GITHUB_RUN_ID" job.url="$GITHUB_SERVER_URL/$GITHUB_REPOSITORY/actions/runs/$GITHUB_RUN_ID" arch="x86_64" - --log raw.log - --storage minio - --storage aws_s3 - --aws-s3-bucket ${{ secrets.REGRESSION_AWS_S3_BUCKET }} - --aws-s3-region ${{ secrets.REGRESSION_AWS_S3_REGION }} - --aws-s3-key-id ${{ secrets.REGRESSION_AWS_S3_KEY_ID }} - --aws-s3-access-key ${{ secrets.REGRESSION_AWS_S3_SECRET_ACCESS_KEY }} - --storage gcs - --gcs-uri ${{ secrets.REGRESSION_GCS_URI }} - --gcs-key-id ${{ secrets.REGRESSION_GCS_KEY_ID }} - --gcs-key-secret ${{ secrets.REGRESSION_GCS_KEY_SECRET }} - - name: Create and upload logs - if: always() - run: .github/create_and_upload_logs.sh 1 - env: - artifact_s3_dir: build/v${{ env.version }}/$GITHUB_SHA - - uses: actions/upload-artifact@v3 - if: always() - with: - name: ${{ env.SUITE }}-artifacts - path: | - ./report.html - ./*.log.txt - ./*.log - ./*.html - ./*/_instances/*.log - ./*/_instances/*/logs/*.log - ./*/*/_instances/*/logs/*.log - ./*/*/_instances/*.log +# parquet: +# needs: [regression_start] +# runs-on: [self-hosted, stress-tester] +# env: +# AWS_ACCESS_KEY_ID: ${{ secrets.AWS_REPORT_KEY_ID }} +# AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_REPORT_SECRET_ACCESS_KEY }} +# AWS_DEFAULT_REGION: ${{ secrets.AWS_REPORT_REGION }} +# steps: +# - name: Checkout regression repo +# uses: actions/checkout@v3 +# with: +# repository: Altinity/clickhouse-regression +# ref: releases +# - name: Set envs +# run: | +# cat >> "$GITHUB_ENV" << 'EOF' +# REPORTS_PATH=${{runner.temp}}/reports_dir +# SUITE=parquet +# artifacts=public +# EOF +# - name: Download json reports +# uses: actions/download-artifact@v3 +# with: +# path: ${{ env.REPORTS_PATH }} +# - name: Setup +# run: .github/setup.sh +# - name: Get deb url +# run: python3 .github/get-deb-url.py --reports-path ${{ env.REPORTS_PATH }} --github-env $GITHUB_ENV +# - name: Run ${{ env.SUITE }} suite +# run: python3 +# -u ${{ env.SUITE }}/regression.py +# --clickhouse-binary-path ${{ env.clickhouse_binary_path }} +# --test-to-end +# --local +# --collect-service-logs +# --output classic +# --parallel 1 +# --attr project="$GITHUB_REPOSITORY" project.id="$GITHUB_REPOSITORY_ID" package="${{ env.clickhouse_binary_path }}" version="${{ env.version }}" user.name="$GITHUB_ACTOR" repository="https://github.com/Altinity/clickhouse-regression" commit.hash="$(git rev-parse HEAD)" job.id="$GITHUB_RUN_ID" job.url="$GITHUB_SERVER_URL/$GITHUB_REPOSITORY/actions/runs/$GITHUB_RUN_ID" arch="x86_64" +# --log raw.log +# --storage minio +# --storage aws_s3 +# --aws-s3-bucket ${{ secrets.REGRESSION_AWS_S3_BUCKET }} +# --aws-s3-region ${{ secrets.REGRESSION_AWS_S3_REGION }} +# --aws-s3-key-id ${{ secrets.REGRESSION_AWS_S3_KEY_ID }} +# --aws-s3-access-key ${{ secrets.REGRESSION_AWS_S3_SECRET_ACCESS_KEY }} +# --storage gcs +# --gcs-uri ${{ secrets.REGRESSION_GCS_URI }} +# --gcs-key-id ${{ secrets.REGRESSION_GCS_KEY_ID }} +# --gcs-key-secret ${{ secrets.REGRESSION_GCS_KEY_SECRET }} +# - name: Create and upload logs +# if: always() +# run: .github/create_and_upload_logs.sh 1 +# env: +# artifact_s3_dir: build/v${{ env.version }}/$GITHUB_SHA +# - uses: actions/upload-artifact@v3 +# if: always() +# with: +# name: ${{ env.SUITE }}-artifacts +# path: | +# ./report.html +# ./*.log.txt +# ./*.log +# ./*.html +# ./*/_instances/*.log +# ./*/_instances/*/logs/*.log +# ./*/*/_instances/*/logs/*.log +# ./*/*/_instances/*.log - s3: - strategy: - fail-fast: false - matrix: - STORAGE: [minio, aws_s3, gcs] - needs: [regression_start] - runs-on: [self-hosted, stress-tester] - env: - AWS_ACCESS_KEY_ID: ${{ secrets.AWS_REPORT_KEY_ID }} - AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_REPORT_SECRET_ACCESS_KEY }} - AWS_DEFAULT_REGION: ${{ secrets.AWS_REPORT_REGION }} - steps: - - name: Checkout regression repo - uses: actions/checkout@v3 - with: - repository: Altinity/clickhouse-regression - ref: releases - - name: Set envs - run: | - cat >> "$GITHUB_ENV" << 'EOF' - REPORTS_PATH=${{runner.temp}}/reports_dir - SUITE=s3 - STORAGE=/${{ matrix.STORAGE }} - artifacts=public - EOF - - name: Download json reports - uses: actions/download-artifact@v3 - with: - path: ${{ env.REPORTS_PATH }} - - name: Setup - run: .github/setup.sh - - name: Get deb url - run: python3 .github/get-deb-url.py --reports-path ${{ env.REPORTS_PATH }} --github-env $GITHUB_ENV - - name: Run ${{ env.SUITE }} suite - run: python3 - -u ${{ env.SUITE }}/regression.py - --clickhouse-binary-path ${{ env.clickhouse_binary_path }} - --test-to-end - --local - --collect-service-logs - --output classic - --parallel 1 - --attr project="$GITHUB_REPOSITORY" project.id="$GITHUB_REPOSITORY_ID" package="${{ env.clickhouse_binary_path }}" version="${{ env.version }}" user.name="$GITHUB_ACTOR" repository="https://github.com/Altinity/clickhouse-regression" commit.hash="$(git rev-parse HEAD)" job.id="$GITHUB_RUN_ID" job.url="$GITHUB_SERVER_URL/$GITHUB_REPOSITORY/actions/runs/$GITHUB_RUN_ID" arch="x86_64" - --log raw.log - --storage ${{ matrix.STORAGE }} - --gcs-uri ${{ secrets.REGRESSION_GCS_URI }} - --gcs-key-id ${{ secrets.REGRESSION_GCS_KEY_ID }} - --gcs-key-secret ${{ secrets.REGRESSION_GCS_KEY_SECRET }} - --aws-s3-bucket ${{ secrets.REGRESSION_AWS_S3_BUCKET }} - --aws-s3-region ${{ secrets.REGRESSION_AWS_S3_REGION }} - --aws-s3-key-id ${{ secrets.REGRESSION_AWS_S3_KEY_ID }} - --aws-s3-access-key ${{ secrets.REGRESSION_AWS_S3_SECRET_ACCESS_KEY }} - - name: Create and upload logs - if: always() - run: .github/create_and_upload_logs.sh 1 - env: - artifact_s3_dir: build/v${{ env.version }}/$GITHUB_SHA - - uses: actions/upload-artifact@v3 - if: always() - with: - name: ${{ env.SUITE }}-${{ matrix.STORAGE }}-artifacts - path: | - ./report.html - ./*.log.txt - ./*.log - ./*.html - ./*/_instances/*.log - ./*/_instances/*/logs/*.log - ./*/*/_instances/*/logs/*.log - ./*/*/_instances/*.log +# s3: +# strategy: +# fail-fast: false +# matrix: +# STORAGE: [minio, aws_s3, gcs] +# needs: [regression_start] +# runs-on: [self-hosted, stress-tester] +# env: +# AWS_ACCESS_KEY_ID: ${{ secrets.AWS_REPORT_KEY_ID }} +# AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_REPORT_SECRET_ACCESS_KEY }} +# AWS_DEFAULT_REGION: ${{ secrets.AWS_REPORT_REGION }} +# steps: +# - name: Checkout regression repo +# uses: actions/checkout@v3 +# with: +# repository: Altinity/clickhouse-regression +# ref: releases +# - name: Set envs +# run: | +# cat >> "$GITHUB_ENV" << 'EOF' +# REPORTS_PATH=${{runner.temp}}/reports_dir +# SUITE=s3 +# STORAGE=/${{ matrix.STORAGE }} +# artifacts=public +# EOF +# - name: Download json reports +# uses: actions/download-artifact@v3 +# with: +# path: ${{ env.REPORTS_PATH }} +# - name: Setup +# run: .github/setup.sh +# - name: Get deb url +# run: python3 .github/get-deb-url.py --reports-path ${{ env.REPORTS_PATH }} --github-env $GITHUB_ENV +# - name: Run ${{ env.SUITE }} suite +# run: python3 +# -u ${{ env.SUITE }}/regression.py +# --clickhouse-binary-path ${{ env.clickhouse_binary_path }} +# --test-to-end +# --local +# --collect-service-logs +# --output classic +# --parallel 1 +# --attr project="$GITHUB_REPOSITORY" project.id="$GITHUB_REPOSITORY_ID" package="${{ env.clickhouse_binary_path }}" version="${{ env.version }}" user.name="$GITHUB_ACTOR" repository="https://github.com/Altinity/clickhouse-regression" commit.hash="$(git rev-parse HEAD)" job.id="$GITHUB_RUN_ID" job.url="$GITHUB_SERVER_URL/$GITHUB_REPOSITORY/actions/runs/$GITHUB_RUN_ID" arch="x86_64" +# --log raw.log +# --storage ${{ matrix.STORAGE }} +# --gcs-uri ${{ secrets.REGRESSION_GCS_URI }} +# --gcs-key-id ${{ secrets.REGRESSION_GCS_KEY_ID }} +# --gcs-key-secret ${{ secrets.REGRESSION_GCS_KEY_SECRET }} +# --aws-s3-bucket ${{ secrets.REGRESSION_AWS_S3_BUCKET }} +# --aws-s3-region ${{ secrets.REGRESSION_AWS_S3_REGION }} +# --aws-s3-key-id ${{ secrets.REGRESSION_AWS_S3_KEY_ID }} +# --aws-s3-access-key ${{ secrets.REGRESSION_AWS_S3_SECRET_ACCESS_KEY }} +# - name: Create and upload logs +# if: always() +# run: .github/create_and_upload_logs.sh 1 +# env: +# artifact_s3_dir: build/v${{ env.version }}/$GITHUB_SHA +# - uses: actions/upload-artifact@v3 +# if: always() +# with: +# name: ${{ env.SUITE }}-${{ matrix.STORAGE }}-artifacts +# path: | +# ./report.html +# ./*.log.txt +# ./*.log +# ./*.html +# ./*/_instances/*.log +# ./*/_instances/*/logs/*.log +# ./*/*/_instances/*/logs/*.log +# ./*/*/_instances/*.log - tiered_storage_s3: - strategy: - fail-fast: false - matrix: - STORAGE: [minio, s3amazon, s3gcs] - needs: [regression_start] - runs-on: [self-hosted, stress-tester] - env: - AWS_ACCESS_KEY_ID: ${{ secrets.AWS_REPORT_KEY_ID }} - AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_REPORT_SECRET_ACCESS_KEY }} - AWS_DEFAULT_REGION: ${{ secrets.AWS_REPORT_REGION }} - steps: - - name: Checkout regression repo - uses: actions/checkout@v3 - with: - repository: Altinity/clickhouse-regression - ref: releases - - name: Set envs - run: | - cat >> "$GITHUB_ENV" << 'EOF' - REPORTS_PATH=${{runner.temp}}/reports_dir - SUITE=tiered_storage - STORAGE=/${{ matrix.STORAGE }} - artifacts=public - EOF - - name: Download json reports - uses: actions/download-artifact@v3 - with: - path: ${{ env.REPORTS_PATH }} - - name: Setup - run: .github/setup.sh - - name: Get deb url - run: python3 .github/get-deb-url.py --reports-path ${{ env.REPORTS_PATH }} --github-env $GITHUB_ENV - - name: Run ${{ env.SUITE }} suite - run: python3 - -u ${{ env.SUITE }}/regression.py - --clickhouse-binary-path ${{ env.clickhouse_binary_path }} - --test-to-end - --local - --collect-service-logs - --output classic - --parallel 1 - --attr project="$GITHUB_REPOSITORY" project.id="$GITHUB_REPOSITORY_ID" package="${{ env.clickhouse_binary_path }}" version="${{ env.version }}" user.name="$GITHUB_ACTOR" repository="https://github.com/Altinity/clickhouse-regression" commit.hash="$(git rev-parse HEAD)" job.id="$GITHUB_RUN_ID" job.url="$GITHUB_SERVER_URL/$GITHUB_REPOSITORY/actions/runs/$GITHUB_RUN_ID" arch="x86_64" - --log raw.log - --aws-s3-access-key ${{ secrets.REGRESSION_AWS_S3_SECRET_ACCESS_KEY }} - --aws-s3-key-id ${{ secrets.REGRESSION_AWS_S3_KEY_ID }} - --aws-s3-uri https://s3.${{ secrets.REGRESSION_AWS_S3_REGION}}.amazonaws.com/${{ secrets.REGRESSION_AWS_S3_BUCKET }}/data/ - --gcs-key-id ${{ secrets.REGRESSION_GCS_KEY_ID }} - --gcs-key-secret ${{ secrets.REGRESSION_GCS_KEY_SECRET }} - --gcs-uri ${{ secrets.REGRESSION_GCS_URI }} - --with-${{ matrix.STORAGE }} - - name: Create and upload logs - if: always() - run: .github/create_and_upload_logs.sh 1 - env: - artifact_s3_dir: build/v${{ env.version }}/$GITHUB_SHA - - uses: actions/upload-artifact@v3 - if: always() - with: - name: ${{ env.SUITE }}-${{ matrix.STORAGE }}-artifacts - path: | - ./report.html - ./*.log.txt - ./*.log - ./*.html - ./*/_instances/*.log - ./*/_instances/*/logs/*.log - ./*/*/_instances/*/logs/*.log - ./*/*/_instances/*.log +# tiered_storage_s3: +# strategy: +# fail-fast: false +# matrix: +# STORAGE: [minio, s3amazon, s3gcs] +# needs: [regression_start] +# runs-on: [self-hosted, stress-tester] +# env: +# AWS_ACCESS_KEY_ID: ${{ secrets.AWS_REPORT_KEY_ID }} +# AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_REPORT_SECRET_ACCESS_KEY }} +# AWS_DEFAULT_REGION: ${{ secrets.AWS_REPORT_REGION }} +# steps: +# - name: Checkout regression repo +# uses: actions/checkout@v3 +# with: +# repository: Altinity/clickhouse-regression +# ref: releases +# - name: Set envs +# run: | +# cat >> "$GITHUB_ENV" << 'EOF' +# REPORTS_PATH=${{runner.temp}}/reports_dir +# SUITE=tiered_storage +# STORAGE=/${{ matrix.STORAGE }} +# artifacts=public +# EOF +# - name: Download json reports +# uses: actions/download-artifact@v3 +# with: +# path: ${{ env.REPORTS_PATH }} +# - name: Setup +# run: .github/setup.sh +# - name: Get deb url +# run: python3 .github/get-deb-url.py --reports-path ${{ env.REPORTS_PATH }} --github-env $GITHUB_ENV +# - name: Run ${{ env.SUITE }} suite +# run: python3 +# -u ${{ env.SUITE }}/regression.py +# --clickhouse-binary-path ${{ env.clickhouse_binary_path }} +# --test-to-end +# --local +# --collect-service-logs +# --output classic +# --parallel 1 +# --attr project="$GITHUB_REPOSITORY" project.id="$GITHUB_REPOSITORY_ID" package="${{ env.clickhouse_binary_path }}" version="${{ env.version }}" user.name="$GITHUB_ACTOR" repository="https://github.com/Altinity/clickhouse-regression" commit.hash="$(git rev-parse HEAD)" job.id="$GITHUB_RUN_ID" job.url="$GITHUB_SERVER_URL/$GITHUB_REPOSITORY/actions/runs/$GITHUB_RUN_ID" arch="x86_64" +# --log raw.log +# --aws-s3-access-key ${{ secrets.REGRESSION_AWS_S3_SECRET_ACCESS_KEY }} +# --aws-s3-key-id ${{ secrets.REGRESSION_AWS_S3_KEY_ID }} +# --aws-s3-uri https://s3.${{ secrets.REGRESSION_AWS_S3_REGION}}.amazonaws.com/${{ secrets.REGRESSION_AWS_S3_BUCKET }}/data/ +# --gcs-key-id ${{ secrets.REGRESSION_GCS_KEY_ID }} +# --gcs-key-secret ${{ secrets.REGRESSION_GCS_KEY_SECRET }} +# --gcs-uri ${{ secrets.REGRESSION_GCS_URI }} +# --with-${{ matrix.STORAGE }} +# - name: Create and upload logs +# if: always() +# run: .github/create_and_upload_logs.sh 1 +# env: +# artifact_s3_dir: build/v${{ env.version }}/$GITHUB_SHA +# - uses: actions/upload-artifact@v3 +# if: always() +# with: +# name: ${{ env.SUITE }}-${{ matrix.STORAGE }}-artifacts +# path: | +# ./report.html +# ./*.log.txt +# ./*.log +# ./*.html +# ./*/_instances/*.log +# ./*/_instances/*/logs/*.log +# ./*/*/_instances/*/logs/*.log +# ./*/*/_instances/*.log SignRelease: needs: [BuilderDebRelease] @@ -1050,12 +1050,12 @@ jobs: - IntegrationTestsRelease0 - IntegrationTestsRelease1 - CompatibilityCheck - - regression_common - - benchmark - - ldap - - parquet - - s3 - - tiered_storage_s3 +# - regression_common +# - benchmark +# - ldap +# - parquet +# - s3 +# - tiered_storage_s3 - SignRelease runs-on: [self-hosted, style-checker] steps: From cd18717e89c15722209efbbba20f4a8d1cace009 Mon Sep 17 00:00:00 2001 From: MyroTk <44327070+MyroTk@users.noreply.github.com> Date: Wed, 2 Aug 2023 17:43:06 -0700 Subject: [PATCH 17/72] Update release_branches.yml --- .github/workflows/release_branches.yml | 32 +++++++++++++------------- 1 file changed, 16 insertions(+), 16 deletions(-) diff --git a/.github/workflows/release_branches.yml b/.github/workflows/release_branches.yml index 56232af29fd8..05027eb9c793 100644 --- a/.github/workflows/release_branches.yml +++ b/.github/workflows/release_branches.yml @@ -28,7 +28,7 @@ on: # yamllint disable-line rule:truthy jobs: DockerHubPushAarch64: - runs-on: [self-hosted, style-checker-aarch64, altinity-aarch64] + runs-on: [self-hosted, style-checker, on-demand, type-cax51, image-arm-app-docker-ce] steps: - name: Check out repository code uses: ClickHouse/checkout@v1 @@ -45,7 +45,7 @@ jobs: path: ${{ runner.temp }}/docker_images_check/changed_images_aarch64.json # temporary DockerHubPush: DockerHubPushAmd64: - runs-on: [self-hosted, style-checker] + runs-on: [self-hosted, style-checker, on-demand, type-cpx51, image-x86-app-docker-ce] steps: - name: Check out repository code uses: ClickHouse/checkout@v1 @@ -67,7 +67,7 @@ jobs: path: ${{ runner.temp }}/docker_images_check/changed_images_amd64.json DockerHubPush: needs: [DockerHubPushAmd64, DockerHubPushAarch64] - runs-on: [self-hosted, style-checker] + runs-on: [self-hosted, style-checker, on-demand, type-cpx41, image-x86-app-docker-ce] steps: - name: Check out repository code uses: ClickHouse/checkout@v1 @@ -94,7 +94,7 @@ jobs: path: ${{ runner.temp }}/changed_images.json CompatibilityCheck: needs: [BuilderDebRelease] - runs-on: [self-hosted, style-checker] + runs-on: [self-hosted, style-checker, on-demand, type-cpx41, image-x86-app-docker-ce] steps: - name: Set envs run: | @@ -130,7 +130,7 @@ jobs: ######################################################################################### BuilderDebRelease: needs: [DockerHubPush] - runs-on: [self-hosted, builder] + runs-on: [self-hosted, builder, on-demand, type-cpx41, image-x86-app-docker-ce] steps: - name: Set envs run: | @@ -178,7 +178,7 @@ jobs: sudo rm -fr "$TEMP_PATH" "$CACHES_PATH" BuilderDebAarch64: needs: [DockerHubPush] - runs-on: [self-hosted, builder-aarch64, altinity-aarch64] + runs-on: [self-hosted, builder, on-demand, type-cax51, image-arm-app-docker-ce] steps: - name: Set envs run: | @@ -229,7 +229,7 @@ jobs: needs: - BuilderDebRelease - BuilderDebAarch64 - runs-on: [self-hosted, style-checker] + runs-on: [self-hosted, style-checker, on-demand, type-cpx41, image-x86-app-docker-ce] steps: - name: Check out repository code uses: ClickHouse/checkout@v1 @@ -258,7 +258,7 @@ jobs: needs: - BuilderDebRelease - BuilderDebAarch64 - runs-on: [self-hosted, style-checker] + runs-on: [self-hosted, style-checker, on-demand, type-cpx41, image-x86-app-docker-ce] if: ${{ success() || failure() }} steps: - name: Set envs @@ -341,7 +341,7 @@ jobs: # - BuilderBinDarwinAarch64 - BuilderDebRelease - BuilderDebAarch64 - runs-on: [self-hosted, style-checker] + runs-on: [self-hosted, style-checker, on-demand, type-cpx31, image-x86-app-docker-ce] steps: - name: Check out repository code uses: ClickHouse/checkout@v1 @@ -356,7 +356,7 @@ jobs: ############################################################################################## FunctionalStatelessTestRelease: needs: [BuilderDebRelease] - runs-on: [self-hosted, func-tester] + runs-on: [self-hosted, func-tester, on-demand, type-cpx51, image-x86-app-docker-ce] steps: - name: Set envs run: | @@ -394,7 +394,7 @@ jobs: FunctionalStatelessTestAarch64: needs: [BuilderDebAarch64] - runs-on: [self-hosted, func-tester-aarch64, altinity-aarch64] + runs-on: [self-hosted, func-tester, on-demand, type-cax51, image-arm-app-docker-ce] steps: - name: Set envs run: | @@ -435,7 +435,7 @@ jobs: ############################################################################################## FunctionalStatefulTestRelease: needs: [BuilderDebRelease] - runs-on: [self-hosted, func-tester] + runs-on: [self-hosted, func-tester, on-demand, type-cpx51, image-x86-app-docker-ce] steps: - name: Set envs run: | @@ -510,7 +510,7 @@ jobs: ############################################################################################# IntegrationTestsRelease0: needs: [BuilderDebRelease] - runs-on: [self-hosted, stress-tester] + runs-on: [self-hosted, stress-tester, on-demand, type-cpx41, image-x86-app-docker-ce] steps: - name: Set envs run: | @@ -547,7 +547,7 @@ jobs: sudo rm -fr "$TEMP_PATH" IntegrationTestsRelease1: needs: [BuilderDebRelease] - runs-on: [self-hosted, stress-tester] + runs-on: [self-hosted, stress-tester, on-demand, type-cpx41, image-x86-app-docker-ce] steps: - name: Set envs run: | @@ -997,7 +997,7 @@ jobs: SignRelease: needs: [BuilderDebRelease] - runs-on: [ self-hosted ] + runs-on: [ self-hosted, on-demand, type-cpx41, image-x86-app-docker-ce] steps: - name: Set envs run: | @@ -1057,7 +1057,7 @@ jobs: # - s3 # - tiered_storage_s3 - SignRelease - runs-on: [self-hosted, style-checker] + runs-on: [self-hosted, style-checker, on-demand, type-cpx31, image-x86-app-docker-ce] steps: - name: Check out repository code uses: ClickHouse/checkout@v1 From b6d79d8c269a0be52dcd6c732a02f982bed771d8 Mon Sep 17 00:00:00 2001 From: MyroTk <44327070+MyroTk@users.noreply.github.com> Date: Wed, 2 Aug 2023 17:45:13 -0700 Subject: [PATCH 18/72] Update release_branches.yml --- .github/workflows/release_branches.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/release_branches.yml b/.github/workflows/release_branches.yml index 05027eb9c793..f918a62e479e 100644 --- a/.github/workflows/release_branches.yml +++ b/.github/workflows/release_branches.yml @@ -28,7 +28,7 @@ on: # yamllint disable-line rule:truthy jobs: DockerHubPushAarch64: - runs-on: [self-hosted, style-checker, on-demand, type-cax51, image-arm-app-docker-ce] + runs-on: [self-hosted, style-checker, on-demand, type-cax41, image-arm-app-docker-ce] steps: - name: Check out repository code uses: ClickHouse/checkout@v1 @@ -178,7 +178,7 @@ jobs: sudo rm -fr "$TEMP_PATH" "$CACHES_PATH" BuilderDebAarch64: needs: [DockerHubPush] - runs-on: [self-hosted, builder, on-demand, type-cax51, image-arm-app-docker-ce] + runs-on: [self-hosted, builder, on-demand, type-cax41, image-arm-app-docker-ce] steps: - name: Set envs run: | @@ -394,7 +394,7 @@ jobs: FunctionalStatelessTestAarch64: needs: [BuilderDebAarch64] - runs-on: [self-hosted, func-tester, on-demand, type-cax51, image-arm-app-docker-ce] + runs-on: [self-hosted, func-tester, on-demand, type-cax41, image-arm-app-docker-ce] steps: - name: Set envs run: | From ade39b8eb81dcda5daf4b0d7e8c255bcf507f21d Mon Sep 17 00:00:00 2001 From: MyroTk <44327070+MyroTk@users.noreply.github.com> Date: Thu, 10 Aug 2023 16:29:35 -0700 Subject: [PATCH 19/72] Update release_branches.yml --- .github/workflows/release_branches.yml | 28 +++++++++++++------------- 1 file changed, 14 insertions(+), 14 deletions(-) diff --git a/.github/workflows/release_branches.yml b/.github/workflows/release_branches.yml index f918a62e479e..68c52a3901b0 100644 --- a/.github/workflows/release_branches.yml +++ b/.github/workflows/release_branches.yml @@ -28,7 +28,7 @@ on: # yamllint disable-line rule:truthy jobs: DockerHubPushAarch64: - runs-on: [self-hosted, style-checker, on-demand, type-cax41, image-arm-app-docker-ce] + runs-on: [self-hosted, style-checker, on-demand, type-cax41] steps: - name: Check out repository code uses: ClickHouse/checkout@v1 @@ -45,7 +45,7 @@ jobs: path: ${{ runner.temp }}/docker_images_check/changed_images_aarch64.json # temporary DockerHubPush: DockerHubPushAmd64: - runs-on: [self-hosted, style-checker, on-demand, type-cpx51, image-x86-app-docker-ce] + runs-on: [self-hosted, style-checker, on-demand, type-cpx51] steps: - name: Check out repository code uses: ClickHouse/checkout@v1 @@ -67,7 +67,7 @@ jobs: path: ${{ runner.temp }}/docker_images_check/changed_images_amd64.json DockerHubPush: needs: [DockerHubPushAmd64, DockerHubPushAarch64] - runs-on: [self-hosted, style-checker, on-demand, type-cpx41, image-x86-app-docker-ce] + runs-on: [self-hosted, style-checker, on-demand, type-cpx41] steps: - name: Check out repository code uses: ClickHouse/checkout@v1 @@ -94,7 +94,7 @@ jobs: path: ${{ runner.temp }}/changed_images.json CompatibilityCheck: needs: [BuilderDebRelease] - runs-on: [self-hosted, style-checker, on-demand, type-cpx41, image-x86-app-docker-ce] + runs-on: [self-hosted, style-checker, on-demand, type-cpx41] steps: - name: Set envs run: | @@ -130,7 +130,7 @@ jobs: ######################################################################################### BuilderDebRelease: needs: [DockerHubPush] - runs-on: [self-hosted, builder, on-demand, type-cpx41, image-x86-app-docker-ce] + runs-on: [self-hosted, builder, on-demand, type-cpx51] steps: - name: Set envs run: | @@ -178,7 +178,7 @@ jobs: sudo rm -fr "$TEMP_PATH" "$CACHES_PATH" BuilderDebAarch64: needs: [DockerHubPush] - runs-on: [self-hosted, builder, on-demand, type-cax41, image-arm-app-docker-ce] + runs-on: [self-hosted, builder, on-demand, type-cax41] steps: - name: Set envs run: | @@ -229,7 +229,7 @@ jobs: needs: - BuilderDebRelease - BuilderDebAarch64 - runs-on: [self-hosted, style-checker, on-demand, type-cpx41, image-x86-app-docker-ce] + runs-on: [self-hosted, style-checker, on-demand, type-cpx41] steps: - name: Check out repository code uses: ClickHouse/checkout@v1 @@ -258,7 +258,7 @@ jobs: needs: - BuilderDebRelease - BuilderDebAarch64 - runs-on: [self-hosted, style-checker, on-demand, type-cpx41, image-x86-app-docker-ce] + runs-on: [self-hosted, style-checker, on-demand, type-cpx41] if: ${{ success() || failure() }} steps: - name: Set envs @@ -341,7 +341,7 @@ jobs: # - BuilderBinDarwinAarch64 - BuilderDebRelease - BuilderDebAarch64 - runs-on: [self-hosted, style-checker, on-demand, type-cpx31, image-x86-app-docker-ce] + runs-on: [self-hosted, style-checker, on-demand, type-cpx31] steps: - name: Check out repository code uses: ClickHouse/checkout@v1 @@ -356,7 +356,7 @@ jobs: ############################################################################################## FunctionalStatelessTestRelease: needs: [BuilderDebRelease] - runs-on: [self-hosted, func-tester, on-demand, type-cpx51, image-x86-app-docker-ce] + runs-on: [self-hosted, func-tester, on-demand, type-cpx51] steps: - name: Set envs run: | @@ -394,7 +394,7 @@ jobs: FunctionalStatelessTestAarch64: needs: [BuilderDebAarch64] - runs-on: [self-hosted, func-tester, on-demand, type-cax41, image-arm-app-docker-ce] + runs-on: [self-hosted, func-tester, on-demand, type-cax41] steps: - name: Set envs run: | @@ -435,7 +435,7 @@ jobs: ############################################################################################## FunctionalStatefulTestRelease: needs: [BuilderDebRelease] - runs-on: [self-hosted, func-tester, on-demand, type-cpx51, image-x86-app-docker-ce] + runs-on: [self-hosted, func-tester, on-demand, type-cpx51] steps: - name: Set envs run: | @@ -510,7 +510,7 @@ jobs: ############################################################################################# IntegrationTestsRelease0: needs: [BuilderDebRelease] - runs-on: [self-hosted, stress-tester, on-demand, type-cpx41, image-x86-app-docker-ce] + runs-on: [self-hosted, stress-tester, on-demand, type-cpx51] steps: - name: Set envs run: | @@ -547,7 +547,7 @@ jobs: sudo rm -fr "$TEMP_PATH" IntegrationTestsRelease1: needs: [BuilderDebRelease] - runs-on: [self-hosted, stress-tester, on-demand, type-cpx41, image-x86-app-docker-ce] + runs-on: [self-hosted, stress-tester, on-demand, type-cpx41] steps: - name: Set envs run: | From b7f632f520fe39a060ee2cc0809c6b3623aae472 Mon Sep 17 00:00:00 2001 From: MyroTk <44327070+MyroTk@users.noreply.github.com> Date: Fri, 11 Aug 2023 08:31:56 -0700 Subject: [PATCH 20/72] Update release_branches.yml --- .github/workflows/release_branches.yml | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/.github/workflows/release_branches.yml b/.github/workflows/release_branches.yml index 68c52a3901b0..076d397a4292 100644 --- a/.github/workflows/release_branches.yml +++ b/.github/workflows/release_branches.yml @@ -28,7 +28,7 @@ on: # yamllint disable-line rule:truthy jobs: DockerHubPushAarch64: - runs-on: [self-hosted, style-checker, on-demand, type-cax41] + runs-on: [self-hosted, style-checker, on-demand, type-cax41, image-arm-system-ubuntu-22.04] steps: - name: Check out repository code uses: ClickHouse/checkout@v1 @@ -178,7 +178,7 @@ jobs: sudo rm -fr "$TEMP_PATH" "$CACHES_PATH" BuilderDebAarch64: needs: [DockerHubPush] - runs-on: [self-hosted, builder, on-demand, type-cax41] + runs-on: [self-hosted, builder, on-demand, type-cax41, image-arm-system-ubuntu-22.04] steps: - name: Set envs run: | @@ -394,7 +394,7 @@ jobs: FunctionalStatelessTestAarch64: needs: [BuilderDebAarch64] - runs-on: [self-hosted, func-tester, on-demand, type-cax41] + runs-on: [self-hosted, func-tester, on-demand, type-cax41, image-arm-system-ubuntu-22.04] steps: - name: Set envs run: | @@ -510,7 +510,7 @@ jobs: ############################################################################################# IntegrationTestsRelease0: needs: [BuilderDebRelease] - runs-on: [self-hosted, stress-tester, on-demand, type-cpx51] + runs-on: [self-hosted, stress-tester, on-demand, type-ccx52] steps: - name: Set envs run: | @@ -547,7 +547,7 @@ jobs: sudo rm -fr "$TEMP_PATH" IntegrationTestsRelease1: needs: [BuilderDebRelease] - runs-on: [self-hosted, stress-tester, on-demand, type-cpx41] + runs-on: [self-hosted, stress-tester, on-demand, type-ccx52] steps: - name: Set envs run: | From 902f3bca4977afe41cfeee2030defc0f378a144e Mon Sep 17 00:00:00 2001 From: MyroTk <44327070+MyroTk@users.noreply.github.com> Date: Tue, 15 Aug 2023 18:02:20 -0700 Subject: [PATCH 21/72] Update release_branches.yml --- .github/workflows/release_branches.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/release_branches.yml b/.github/workflows/release_branches.yml index 076d397a4292..45079b80b9ba 100644 --- a/.github/workflows/release_branches.yml +++ b/.github/workflows/release_branches.yml @@ -45,7 +45,7 @@ jobs: path: ${{ runner.temp }}/docker_images_check/changed_images_aarch64.json # temporary DockerHubPush: DockerHubPushAmd64: - runs-on: [self-hosted, style-checker, on-demand, type-cpx51] + runs-on: [self-hosted, style-checker] steps: - name: Check out repository code uses: ClickHouse/checkout@v1 From 6ed89c3c1ee18bc46f10370883b37284fae6d32c Mon Sep 17 00:00:00 2001 From: MyroTk <44327070+MyroTk@users.noreply.github.com> Date: Wed, 16 Aug 2023 10:20:56 -0700 Subject: [PATCH 22/72] Update release_branches.yml --- .github/workflows/release_branches.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/release_branches.yml b/.github/workflows/release_branches.yml index 45079b80b9ba..57ef8c89849e 100644 --- a/.github/workflows/release_branches.yml +++ b/.github/workflows/release_branches.yml @@ -160,7 +160,7 @@ jobs: sudo rm -fr "$TEMP_PATH" mkdir -p "$TEMP_PATH/build_check/package_release" cd .. && tar czf $TEMP_PATH/build_source.src.tar.gz ClickHouse/ - cd $TEMP_PATH && tar xvzf $TEMP_PATH/build_source.src.tar.gz + cd $TEMP_PATH && tar xzf $TEMP_PATH/build_source.src.tar.gz cd "$REPO_COPY/tests/ci" && python3 build_check.py "$BUILD_NAME" - name: Upload build URLs to artifacts if: ${{ success() || failure() }} From aba367a19c5aae9a8c503b090d6de7bfc6023ef0 Mon Sep 17 00:00:00 2001 From: MyroTk <44327070+MyroTk@users.noreply.github.com> Date: Wed, 16 Aug 2023 15:37:16 -0700 Subject: [PATCH 23/72] Update release_branches.yml --- .github/workflows/release_branches.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/release_branches.yml b/.github/workflows/release_branches.yml index 57ef8c89849e..2099089f59fc 100644 --- a/.github/workflows/release_branches.yml +++ b/.github/workflows/release_branches.yml @@ -45,7 +45,7 @@ jobs: path: ${{ runner.temp }}/docker_images_check/changed_images_aarch64.json # temporary DockerHubPush: DockerHubPushAmd64: - runs-on: [self-hosted, style-checker] + runs-on: [self-hosted, style-checker, on-demand, type-cpx51] steps: - name: Check out repository code uses: ClickHouse/checkout@v1 From a5547cee0ecb081a8419007d05a1167d9ea71e06 Mon Sep 17 00:00:00 2001 From: MyroTk <44327070+MyroTk@users.noreply.github.com> Date: Wed, 16 Aug 2023 18:00:54 -0700 Subject: [PATCH 24/72] Update release_branches.yml --- .github/workflows/release_branches.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/release_branches.yml b/.github/workflows/release_branches.yml index 2099089f59fc..46fd5112e302 100644 --- a/.github/workflows/release_branches.yml +++ b/.github/workflows/release_branches.yml @@ -28,7 +28,7 @@ on: # yamllint disable-line rule:truthy jobs: DockerHubPushAarch64: - runs-on: [self-hosted, style-checker, on-demand, type-cax41, image-arm-system-ubuntu-22.04] + runs-on: [self-hosted, style-checker, on-demand, type-cax41, image-arm-app-docker-ce] steps: - name: Check out repository code uses: ClickHouse/checkout@v1 @@ -45,7 +45,7 @@ jobs: path: ${{ runner.temp }}/docker_images_check/changed_images_aarch64.json # temporary DockerHubPush: DockerHubPushAmd64: - runs-on: [self-hosted, style-checker, on-demand, type-cpx51] + runs-on: [self-hosted, style-checker, on-demand, type-cpx51, image-x86-app-docker-ce] steps: - name: Check out repository code uses: ClickHouse/checkout@v1 @@ -67,7 +67,7 @@ jobs: path: ${{ runner.temp }}/docker_images_check/changed_images_amd64.json DockerHubPush: needs: [DockerHubPushAmd64, DockerHubPushAarch64] - runs-on: [self-hosted, style-checker, on-demand, type-cpx41] + runs-on: [self-hosted, style-checker, on-demand, type-cpx41, image-x86-app-docker-ce] steps: - name: Check out repository code uses: ClickHouse/checkout@v1 From ab5ba5643f93dd940528dc47df50677f27452d42 Mon Sep 17 00:00:00 2001 From: MyroTk <44327070+MyroTk@users.noreply.github.com> Date: Wed, 16 Aug 2023 19:24:35 -0700 Subject: [PATCH 25/72] Update release_branches.yml --- .github/workflows/release_branches.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/release_branches.yml b/.github/workflows/release_branches.yml index 46fd5112e302..681834eb7b7e 100644 --- a/.github/workflows/release_branches.yml +++ b/.github/workflows/release_branches.yml @@ -28,7 +28,7 @@ on: # yamllint disable-line rule:truthy jobs: DockerHubPushAarch64: - runs-on: [self-hosted, style-checker, on-demand, type-cax41, image-arm-app-docker-ce] + runs-on: [self-hosted, style-checker, image-builder, type-cax41, image-arm-app-docker-ce] steps: - name: Check out repository code uses: ClickHouse/checkout@v1 @@ -45,7 +45,7 @@ jobs: path: ${{ runner.temp }}/docker_images_check/changed_images_aarch64.json # temporary DockerHubPush: DockerHubPushAmd64: - runs-on: [self-hosted, style-checker, on-demand, type-cpx51, image-x86-app-docker-ce] + runs-on: [self-hosted, style-checker, image-builder, type-cpx51, image-x86-app-docker-ce] steps: - name: Check out repository code uses: ClickHouse/checkout@v1 From 7b21360516cf8990c8eeb47af659bc6137ae103e Mon Sep 17 00:00:00 2001 From: MyroTk <44327070+MyroTk@users.noreply.github.com> Date: Wed, 16 Aug 2023 19:57:33 -0700 Subject: [PATCH 26/72] Update release_branches.yml --- .github/workflows/release_branches.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/release_branches.yml b/.github/workflows/release_branches.yml index 681834eb7b7e..46fd5112e302 100644 --- a/.github/workflows/release_branches.yml +++ b/.github/workflows/release_branches.yml @@ -28,7 +28,7 @@ on: # yamllint disable-line rule:truthy jobs: DockerHubPushAarch64: - runs-on: [self-hosted, style-checker, image-builder, type-cax41, image-arm-app-docker-ce] + runs-on: [self-hosted, style-checker, on-demand, type-cax41, image-arm-app-docker-ce] steps: - name: Check out repository code uses: ClickHouse/checkout@v1 @@ -45,7 +45,7 @@ jobs: path: ${{ runner.temp }}/docker_images_check/changed_images_aarch64.json # temporary DockerHubPush: DockerHubPushAmd64: - runs-on: [self-hosted, style-checker, image-builder, type-cpx51, image-x86-app-docker-ce] + runs-on: [self-hosted, style-checker, on-demand, type-cpx51, image-x86-app-docker-ce] steps: - name: Check out repository code uses: ClickHouse/checkout@v1 From f90d95dfb2c57019dd41675332ab7e62f35b7ff6 Mon Sep 17 00:00:00 2001 From: MyroTk <44327070+MyroTk@users.noreply.github.com> Date: Wed, 16 Aug 2023 20:30:24 -0700 Subject: [PATCH 27/72] Update release_branches.yml --- .github/workflows/release_branches.yml | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/.github/workflows/release_branches.yml b/.github/workflows/release_branches.yml index 46fd5112e302..424ee6fc783a 100644 --- a/.github/workflows/release_branches.yml +++ b/.github/workflows/release_branches.yml @@ -94,7 +94,7 @@ jobs: path: ${{ runner.temp }}/changed_images.json CompatibilityCheck: needs: [BuilderDebRelease] - runs-on: [self-hosted, style-checker, on-demand, type-cpx41] + runs-on: [self-hosted, style-checker, on-demand, type-cpx41, image-x86-app-docker-ce] steps: - name: Set envs run: | @@ -130,7 +130,7 @@ jobs: ######################################################################################### BuilderDebRelease: needs: [DockerHubPush] - runs-on: [self-hosted, builder, on-demand, type-cpx51] + runs-on: [self-hosted, builder, on-demand, type-cpx51, image-x86-app-docker-ce] steps: - name: Set envs run: | @@ -178,7 +178,7 @@ jobs: sudo rm -fr "$TEMP_PATH" "$CACHES_PATH" BuilderDebAarch64: needs: [DockerHubPush] - runs-on: [self-hosted, builder, on-demand, type-cax41, image-arm-system-ubuntu-22.04] + runs-on: [self-hosted, builder, on-demand, type-cax41, image-arm-app-docker-ce] steps: - name: Set envs run: | @@ -229,7 +229,7 @@ jobs: needs: - BuilderDebRelease - BuilderDebAarch64 - runs-on: [self-hosted, style-checker, on-demand, type-cpx41] + runs-on: [self-hosted, style-checker, on-demand, type-cpx41, image-x86-app-docker-ce] steps: - name: Check out repository code uses: ClickHouse/checkout@v1 @@ -258,7 +258,7 @@ jobs: needs: - BuilderDebRelease - BuilderDebAarch64 - runs-on: [self-hosted, style-checker, on-demand, type-cpx41] + runs-on: [self-hosted, style-checker, on-demand, type-cpx41, image-x86-app-docker-ce] if: ${{ success() || failure() }} steps: - name: Set envs @@ -341,7 +341,7 @@ jobs: # - BuilderBinDarwinAarch64 - BuilderDebRelease - BuilderDebAarch64 - runs-on: [self-hosted, style-checker, on-demand, type-cpx31] + runs-on: [self-hosted, style-checker, on-demand, type-cpx31, image-x86-app-docker-ce] steps: - name: Check out repository code uses: ClickHouse/checkout@v1 @@ -356,7 +356,7 @@ jobs: ############################################################################################## FunctionalStatelessTestRelease: needs: [BuilderDebRelease] - runs-on: [self-hosted, func-tester, on-demand, type-cpx51] + runs-on: [self-hosted, func-tester, on-demand, type-cpx51, image-x86-snapshot-docker_ipv6_x86] steps: - name: Set envs run: | @@ -394,7 +394,7 @@ jobs: FunctionalStatelessTestAarch64: needs: [BuilderDebAarch64] - runs-on: [self-hosted, func-tester, on-demand, type-cax41, image-arm-system-ubuntu-22.04] + runs-on: [self-hosted, func-tester, on-demand, type-cax41, image-arm-snapshot-docker_ipv6_arm] steps: - name: Set envs run: | @@ -435,7 +435,7 @@ jobs: ############################################################################################## FunctionalStatefulTestRelease: needs: [BuilderDebRelease] - runs-on: [self-hosted, func-tester, on-demand, type-cpx51] + runs-on: [self-hosted, func-tester, on-demand, type-cpx51, ] steps: - name: Set envs run: | From 0c97526455e0f1b5b86604613bdadfd62d9f534b Mon Sep 17 00:00:00 2001 From: MyroTk <44327070+MyroTk@users.noreply.github.com> Date: Thu, 17 Aug 2023 09:28:37 -0700 Subject: [PATCH 28/72] Update 02381_arrow_dict_to_lc.sh --- .../0_stateless/02381_arrow_dict_to_lc.sh | 20 ++++++++++--------- 1 file changed, 11 insertions(+), 9 deletions(-) diff --git a/tests/queries/0_stateless/02381_arrow_dict_to_lc.sh b/tests/queries/0_stateless/02381_arrow_dict_to_lc.sh index 9fb0272eeb07..84f7867bac8d 100755 --- a/tests/queries/0_stateless/02381_arrow_dict_to_lc.sh +++ b/tests/queries/0_stateless/02381_arrow_dict_to_lc.sh @@ -25,12 +25,14 @@ CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) # cat data.arrow | gzip | base64 -cat <&2 + +# cat < Date: Thu, 17 Aug 2023 21:53:31 -0700 Subject: [PATCH 29/72] Update release_branches.yml --- .github/workflows/release_branches.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/release_branches.yml b/.github/workflows/release_branches.yml index 424ee6fc783a..fa239c26b6e7 100644 --- a/.github/workflows/release_branches.yml +++ b/.github/workflows/release_branches.yml @@ -394,7 +394,7 @@ jobs: FunctionalStatelessTestAarch64: needs: [BuilderDebAarch64] - runs-on: [self-hosted, func-tester, on-demand, type-cax41, image-arm-snapshot-docker_ipv6_arm] + runs-on: [self-hosted, func-tester-aarch64] steps: - name: Set envs run: | @@ -435,7 +435,7 @@ jobs: ############################################################################################## FunctionalStatefulTestRelease: needs: [BuilderDebRelease] - runs-on: [self-hosted, func-tester, on-demand, type-cpx51, ] + runs-on: [self-hosted, func-tester, on-demand, type-cpx51, image-x86-snapshot-docker_ipv6_x86] steps: - name: Set envs run: | From d7b1baf76545b0e88fcbb3a0bebb9a4691ac88e5 Mon Sep 17 00:00:00 2001 From: MyroTk <44327070+MyroTk@users.noreply.github.com> Date: Mon, 21 Aug 2023 21:12:39 -0700 Subject: [PATCH 30/72] Update release_branches.yml --- .github/workflows/release_branches.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/release_branches.yml b/.github/workflows/release_branches.yml index fa239c26b6e7..482c1fac4ce3 100644 --- a/.github/workflows/release_branches.yml +++ b/.github/workflows/release_branches.yml @@ -394,7 +394,7 @@ jobs: FunctionalStatelessTestAarch64: needs: [BuilderDebAarch64] - runs-on: [self-hosted, func-tester-aarch64] + runs-on: [self-hosted, func-tester, on-demand, type-ccx41, image-arm-snapshot-docker_ipv6_arm] steps: - name: Set envs run: | From e4844ea098fb77bdfccfa59ffbcf16595282d228 Mon Sep 17 00:00:00 2001 From: MyroTk <44327070+MyroTk@users.noreply.github.com> Date: Mon, 21 Aug 2023 21:44:14 -0700 Subject: [PATCH 31/72] Update release_branches.yml --- .github/workflows/release_branches.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/release_branches.yml b/.github/workflows/release_branches.yml index 482c1fac4ce3..007f29926364 100644 --- a/.github/workflows/release_branches.yml +++ b/.github/workflows/release_branches.yml @@ -28,7 +28,7 @@ on: # yamllint disable-line rule:truthy jobs: DockerHubPushAarch64: - runs-on: [self-hosted, style-checker, on-demand, type-cax41, image-arm-app-docker-ce] + runs-on: [self-hosted, style-checker, on-demand, type-cax41, in-fsn1, image-arm-app-docker-ce] steps: - name: Check out repository code uses: ClickHouse/checkout@v1 @@ -178,7 +178,7 @@ jobs: sudo rm -fr "$TEMP_PATH" "$CACHES_PATH" BuilderDebAarch64: needs: [DockerHubPush] - runs-on: [self-hosted, builder, on-demand, type-cax41, image-arm-app-docker-ce] + runs-on: [self-hosted, builder, on-demand, type-cax41, in-fsn1, image-arm-app-docker-ce] steps: - name: Set envs run: | @@ -394,7 +394,7 @@ jobs: FunctionalStatelessTestAarch64: needs: [BuilderDebAarch64] - runs-on: [self-hosted, func-tester, on-demand, type-ccx41, image-arm-snapshot-docker_ipv6_arm] + runs-on: [self-hosted, func-tester, on-demand, type-ccx41, in-fsn1, image-arm-snapshot-docker_ipv6_arm] steps: - name: Set envs run: | From 49db5266a15318cd0bd67edaef22734eaba111b3 Mon Sep 17 00:00:00 2001 From: MyroTk <44327070+MyroTk@users.noreply.github.com> Date: Tue, 22 Aug 2023 08:07:25 -0700 Subject: [PATCH 32/72] Update release_branches.yml --- .github/workflows/release_branches.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/release_branches.yml b/.github/workflows/release_branches.yml index 007f29926364..1f25d5c4ea76 100644 --- a/.github/workflows/release_branches.yml +++ b/.github/workflows/release_branches.yml @@ -394,7 +394,7 @@ jobs: FunctionalStatelessTestAarch64: needs: [BuilderDebAarch64] - runs-on: [self-hosted, func-tester, on-demand, type-ccx41, in-fsn1, image-arm-snapshot-docker_ipv6_arm] + runs-on: [self-hosted, func-tester, on-demand, type-cax41, in-fsn1, image-arm-snapshot-docker_ipv6_arm] steps: - name: Set envs run: | From 5fd4ba992dfa0799aade8fc3585f71f0a2453465 Mon Sep 17 00:00:00 2001 From: MyroTk <44327070+MyroTk@users.noreply.github.com> Date: Tue, 22 Aug 2023 16:46:53 -0700 Subject: [PATCH 33/72] Update release_branches.yml --- .github/workflows/release_branches.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/release_branches.yml b/.github/workflows/release_branches.yml index 1f25d5c4ea76..43a6d76483d9 100644 --- a/.github/workflows/release_branches.yml +++ b/.github/workflows/release_branches.yml @@ -178,7 +178,7 @@ jobs: sudo rm -fr "$TEMP_PATH" "$CACHES_PATH" BuilderDebAarch64: needs: [DockerHubPush] - runs-on: [self-hosted, builder, on-demand, type-cax41, in-fsn1, image-arm-app-docker-ce] + runs-on: [self-hosted, builder-aarch64, altinity-aarch64] steps: - name: Set envs run: | @@ -394,7 +394,7 @@ jobs: FunctionalStatelessTestAarch64: needs: [BuilderDebAarch64] - runs-on: [self-hosted, func-tester, on-demand, type-cax41, in-fsn1, image-arm-snapshot-docker_ipv6_arm] + runs-on: [self-hosted, func-tester-aarch64, altinity-aarch64] steps: - name: Set envs run: | From ed444a505c9c1b55409c454ad63120ffc3de106b Mon Sep 17 00:00:00 2001 From: MyroTk <44327070+MyroTk@users.noreply.github.com> Date: Wed, 23 Aug 2023 09:31:42 -0700 Subject: [PATCH 34/72] Update release_branches.yml --- .github/workflows/release_branches.yml | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/.github/workflows/release_branches.yml b/.github/workflows/release_branches.yml index 43a6d76483d9..08a2370bf036 100644 --- a/.github/workflows/release_branches.yml +++ b/.github/workflows/release_branches.yml @@ -396,6 +396,18 @@ jobs: needs: [BuilderDebAarch64] runs-on: [self-hosted, func-tester-aarch64, altinity-aarch64] steps: + - name: Setup + run: || + sudo touch /etc/docker/daemon.json + sudo chown ubuntu:ubuntu /etc/docker/daemon.json + sudo cat < /etc/docker/daemon.json + { + "ipv6": true, + "fixed-cidr-v6": "2001:db8:1::/64" + } + EOT + sudo chown root:root /etc/docker/daemon.json + sudo systemctl restart docker - name: Set envs run: | cat >> "$GITHUB_ENV" << 'EOF' From 27c32f10ef59b91f7765da49a3853925db779771 Mon Sep 17 00:00:00 2001 From: MyroTk <44327070+MyroTk@users.noreply.github.com> Date: Wed, 23 Aug 2023 09:38:17 -0700 Subject: [PATCH 35/72] Update release_branches.yml --- .github/workflows/release_branches.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/release_branches.yml b/.github/workflows/release_branches.yml index 08a2370bf036..bd4f151064cb 100644 --- a/.github/workflows/release_branches.yml +++ b/.github/workflows/release_branches.yml @@ -397,7 +397,7 @@ jobs: runs-on: [self-hosted, func-tester-aarch64, altinity-aarch64] steps: - name: Setup - run: || + run: | sudo touch /etc/docker/daemon.json sudo chown ubuntu:ubuntu /etc/docker/daemon.json sudo cat < /etc/docker/daemon.json From ebf364856e034f30df7a0b7717db324b4cd56472 Mon Sep 17 00:00:00 2001 From: MyroTk <44327070+MyroTk@users.noreply.github.com> Date: Wed, 23 Aug 2023 14:03:32 -0700 Subject: [PATCH 36/72] Update 02381_arrow_dict_to_lc.sh --- .../0_stateless/02381_arrow_dict_to_lc.sh | 20 +++++++++---------- 1 file changed, 9 insertions(+), 11 deletions(-) diff --git a/tests/queries/0_stateless/02381_arrow_dict_to_lc.sh b/tests/queries/0_stateless/02381_arrow_dict_to_lc.sh index 84f7867bac8d..316f82f1b880 100755 --- a/tests/queries/0_stateless/02381_arrow_dict_to_lc.sh +++ b/tests/queries/0_stateless/02381_arrow_dict_to_lc.sh @@ -25,14 +25,12 @@ CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) # cat data.arrow | gzip | base64 -ls -l $CLICKHOUSE_LOCAL 1>&2 - -# cat < Date: Wed, 23 Aug 2023 14:05:01 -0700 Subject: [PATCH 37/72] Update release_branches.yml --- .github/workflows/release_branches.yml | 72 +++++++++++++------------- 1 file changed, 36 insertions(+), 36 deletions(-) diff --git a/.github/workflows/release_branches.yml b/.github/workflows/release_branches.yml index bd4f151064cb..576bed49c9d0 100644 --- a/.github/workflows/release_branches.yml +++ b/.github/workflows/release_branches.yml @@ -481,42 +481,42 @@ jobs: # shellcheck disable=SC2046 docker rm -f $(docker ps -a -q) ||: sudo rm -fr "$TEMP_PATH" - # FunctionalStatefulTestAarch64: - # needs: [BuilderDebAarch64] - # runs-on: [self-hosted, func-tester-aarch64] - # steps: - # - name: Set envs - # run: | - # cat >> "$GITHUB_ENV" << 'EOF' - # TEMP_PATH=${{runner.temp}}/stateful_release - # REPORTS_PATH=${{runner.temp}}/reports_dir - # CHECK_NAME=Stateful tests (aarch64) - # REPO_COPY=${{runner.temp}}/stateful_release/ClickHouse - # KILL_TIMEOUT=3600 - # EOF - # - name: Download json reports - # uses: actions/download-artifact@v3 - # with: - # path: ${{ env.REPORTS_PATH }} - # - name: Check out repository code - # uses: ClickHouse/checkout@v1 - # with: - # clear-repository: true - # - name: Functional test - # run: | - # sudo rm -fr "$TEMP_PATH" - # mkdir -p "$TEMP_PATH" - # cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" - # cd "$REPO_COPY/tests/ci" - # python3 functional_test_check.py "$CHECK_NAME" "$KILL_TIMEOUT" - # - name: Cleanup - # if: always() - # run: | - # # shellcheck disable=SC2046 - # docker kill $(docker ps -q) ||: - # # shellcheck disable=SC2046 - # docker rm -f $(docker ps -a -q) ||: - # sudo rm -fr "$TEMP_PATH" + FunctionalStatefulTestAarch64: + needs: [BuilderDebAarch64] + runs-on: [self-hosted, func-tester-aarch64, altinity-aarch64] + steps: + - name: Set envs + run: | + cat >> "$GITHUB_ENV" << 'EOF' + TEMP_PATH=${{runner.temp}}/stateful_release + REPORTS_PATH=${{runner.temp}}/reports_dir + CHECK_NAME=Stateful tests (aarch64) + REPO_COPY=${{runner.temp}}/stateful_release/ClickHouse + KILL_TIMEOUT=3600 + EOF + - name: Download json reports + uses: actions/download-artifact@v3 + with: + path: ${{ env.REPORTS_PATH }} + - name: Check out repository code + uses: ClickHouse/checkout@v1 + with: + clear-repository: true + - name: Functional test + run: | + sudo rm -fr "$TEMP_PATH" + mkdir -p "$TEMP_PATH" + cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" + cd "$REPO_COPY/tests/ci" + python3 functional_test_check.py "$CHECK_NAME" "$KILL_TIMEOUT" + - name: Cleanup + if: always() + run: | + # shellcheck disable=SC2046 + docker kill $(docker ps -q) ||: + # shellcheck disable=SC2046 + docker rm -f $(docker ps -a -q) ||: + sudo rm -fr "$TEMP_PATH" ############################################################################################# ############################# INTEGRATION TESTS ############################################# ############################################################################################# From 290591486f9fa7ed34ae026638e1e0a0cf34f607 Mon Sep 17 00:00:00 2001 From: MyroTk <44327070+MyroTk@users.noreply.github.com> Date: Wed, 23 Aug 2023 18:52:27 -0700 Subject: [PATCH 38/72] Update release_branches.yml --- .github/workflows/release_branches.yml | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/.github/workflows/release_branches.yml b/.github/workflows/release_branches.yml index 576bed49c9d0..ba17e159f9a1 100644 --- a/.github/workflows/release_branches.yml +++ b/.github/workflows/release_branches.yml @@ -485,6 +485,18 @@ jobs: needs: [BuilderDebAarch64] runs-on: [self-hosted, func-tester-aarch64, altinity-aarch64] steps: + - name: Setup + run: | + sudo touch /etc/docker/daemon.json + sudo chown ubuntu:ubuntu /etc/docker/daemon.json + sudo cat < /etc/docker/daemon.json + { + "ipv6": true, + "fixed-cidr-v6": "2001:db8:1::/64" + } + EOT + sudo chown root:root /etc/docker/daemon.json + sudo systemctl restart docker - name: Set envs run: | cat >> "$GITHUB_ENV" << 'EOF' From b824798396addfefa8157d956c15202a2471f004 Mon Sep 17 00:00:00 2001 From: MyroTk <44327070+MyroTk@users.noreply.github.com> Date: Thu, 24 Aug 2023 08:17:28 -0700 Subject: [PATCH 39/72] Update run.sh --- docker/test/stateless/run.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docker/test/stateless/run.sh b/docker/test/stateless/run.sh index 4e7555ea3a4a..226aeb6c558f 100755 --- a/docker/test/stateless/run.sh +++ b/docker/test/stateless/run.sh @@ -109,7 +109,7 @@ function run_tests() # Too many tests fail for DatabaseReplicated in parallel. All other # configurations are OK. ADDITIONAL_OPTIONS+=('--jobs') - ADDITIONAL_OPTIONS+=('8') + ADDITIONAL_OPTIONS+=('2') fi if [[ -n "$RUN_BY_HASH_NUM" ]] && [[ -n "$RUN_BY_HASH_TOTAL" ]]; then From cbfbf52903963ffdc48ddb203eeba7e29a7569b4 Mon Sep 17 00:00:00 2001 From: MyroTk <44327070+MyroTk@users.noreply.github.com> Date: Thu, 24 Aug 2023 08:19:52 -0700 Subject: [PATCH 40/72] Update release_branches.yml --- .github/workflows/release_branches.yml | 154 ++++++++++++------------- 1 file changed, 77 insertions(+), 77 deletions(-) diff --git a/.github/workflows/release_branches.yml b/.github/workflows/release_branches.yml index ba17e159f9a1..ca438d2ab108 100644 --- a/.github/workflows/release_branches.yml +++ b/.github/workflows/release_branches.yml @@ -178,7 +178,7 @@ jobs: sudo rm -fr "$TEMP_PATH" "$CACHES_PATH" BuilderDebAarch64: needs: [DockerHubPush] - runs-on: [self-hosted, builder-aarch64, altinity-aarch64] + runs-on: [self-hosted, builder, on-demand, type-cax41, image-arm-app-docker-ce] steps: - name: Set envs run: | @@ -394,7 +394,7 @@ jobs: FunctionalStatelessTestAarch64: needs: [BuilderDebAarch64] - runs-on: [self-hosted, func-tester-aarch64, altinity-aarch64] + runs-on: [self-hosted, func-tester, on-demand, type-cax41, image-arm-snapshot-docker_ipv6_arm] steps: - name: Setup run: | @@ -483,7 +483,7 @@ jobs: sudo rm -fr "$TEMP_PATH" FunctionalStatefulTestAarch64: needs: [BuilderDebAarch64] - runs-on: [self-hosted, func-tester-aarch64, altinity-aarch64] + runs-on: [self-hosted, func-tester, on-demand, type-cax41, image-arm-snapshot-docker_ipv6_arm] steps: - name: Setup run: | @@ -532,80 +532,80 @@ jobs: ############################################################################################# ############################# INTEGRATION TESTS ############################################# ############################################################################################# - IntegrationTestsRelease0: - needs: [BuilderDebRelease] - runs-on: [self-hosted, stress-tester, on-demand, type-ccx52] - steps: - - name: Set envs - run: | - cat >> "$GITHUB_ENV" << 'EOF' - TEMP_PATH=${{runner.temp}}/integration_tests_release - REPORTS_PATH=${{runner.temp}}/reports_dir - CHECK_NAME=Integration tests (release) - REPO_COPY=${{runner.temp}}/integration_tests_release/ClickHouse - RUN_BY_HASH_NUM=0 - RUN_BY_HASH_TOTAL=2 - EOF - - name: Download json reports - uses: actions/download-artifact@v3 - with: - path: ${{ env.REPORTS_PATH }} - - name: Check out repository code - uses: ClickHouse/checkout@v1 - with: - clear-repository: true - - name: Integration test - run: | - sudo rm -fr "$TEMP_PATH" - mkdir -p "$TEMP_PATH" - cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" - cd "$REPO_COPY/tests/ci" - python3 integration_test_check.py "$CHECK_NAME" - - name: Cleanup - if: always() - run: | - # shellcheck disable=SC2046 - docker kill $(docker ps -q) ||: - # shellcheck disable=SC2046 - docker rm -f $(docker ps -a -q) ||: - sudo rm -fr "$TEMP_PATH" - IntegrationTestsRelease1: - needs: [BuilderDebRelease] - runs-on: [self-hosted, stress-tester, on-demand, type-ccx52] - steps: - - name: Set envs - run: | - cat >> "$GITHUB_ENV" << 'EOF' - TEMP_PATH=${{runner.temp}}/integration_tests_release - REPORTS_PATH=${{runner.temp}}/reports_dir - CHECK_NAME=Integration tests (release) - REPO_COPY=${{runner.temp}}/integration_tests_release/ClickHouse - RUN_BY_HASH_NUM=1 - RUN_BY_HASH_TOTAL=2 - EOF - - name: Download json reports - uses: actions/download-artifact@v3 - with: - path: ${{ env.REPORTS_PATH }} - - name: Check out repository code - uses: ClickHouse/checkout@v1 - with: - clear-repository: true - - name: Integration test - run: | - sudo rm -fr "$TEMP_PATH" - mkdir -p "$TEMP_PATH" - cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" - cd "$REPO_COPY/tests/ci" - python3 integration_test_check.py "$CHECK_NAME" - - name: Cleanup - if: always() - run: | - # shellcheck disable=SC2046 - docker kill $(docker ps -q) ||: - # shellcheck disable=SC2046 - docker rm -f $(docker ps -a -q) ||: - sudo rm -fr "$TEMP_PATH" + # IntegrationTestsRelease0: + # needs: [BuilderDebRelease] + # runs-on: [self-hosted, stress-tester, on-demand, type-ccx52] + # steps: + # - name: Set envs + # run: | + # cat >> "$GITHUB_ENV" << 'EOF' + # TEMP_PATH=${{runner.temp}}/integration_tests_release + # REPORTS_PATH=${{runner.temp}}/reports_dir + # CHECK_NAME=Integration tests (release) + # REPO_COPY=${{runner.temp}}/integration_tests_release/ClickHouse + # RUN_BY_HASH_NUM=0 + # RUN_BY_HASH_TOTAL=2 + # EOF + # - name: Download json reports + # uses: actions/download-artifact@v3 + # with: + # path: ${{ env.REPORTS_PATH }} + # - name: Check out repository code + # uses: ClickHouse/checkout@v1 + # with: + # clear-repository: true + # - name: Integration test + # run: | + # sudo rm -fr "$TEMP_PATH" + # mkdir -p "$TEMP_PATH" + # cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" + # cd "$REPO_COPY/tests/ci" + # python3 integration_test_check.py "$CHECK_NAME" + # - name: Cleanup + # if: always() + # run: | + # # shellcheck disable=SC2046 + # docker kill $(docker ps -q) ||: + # # shellcheck disable=SC2046 + # docker rm -f $(docker ps -a -q) ||: + # sudo rm -fr "$TEMP_PATH" + # IntegrationTestsRelease1: + # needs: [BuilderDebRelease] + # runs-on: [self-hosted, stress-tester, on-demand, type-ccx52] + # steps: + # - name: Set envs + # run: | + # cat >> "$GITHUB_ENV" << 'EOF' + # TEMP_PATH=${{runner.temp}}/integration_tests_release + # REPORTS_PATH=${{runner.temp}}/reports_dir + # CHECK_NAME=Integration tests (release) + # REPO_COPY=${{runner.temp}}/integration_tests_release/ClickHouse + # RUN_BY_HASH_NUM=1 + # RUN_BY_HASH_TOTAL=2 + # EOF + # - name: Download json reports + # uses: actions/download-artifact@v3 + # with: + # path: ${{ env.REPORTS_PATH }} + # - name: Check out repository code + # uses: ClickHouse/checkout@v1 + # with: + # clear-repository: true + # - name: Integration test + # run: | + # sudo rm -fr "$TEMP_PATH" + # mkdir -p "$TEMP_PATH" + # cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" + # cd "$REPO_COPY/tests/ci" + # python3 integration_test_check.py "$CHECK_NAME" + # - name: Cleanup + # if: always() + # run: | + # # shellcheck disable=SC2046 + # docker kill $(docker ps -q) ||: + # # shellcheck disable=SC2046 + # docker rm -f $(docker ps -a -q) ||: + # sudo rm -fr "$TEMP_PATH" ############################################################################################# ##################################### REGRESSION TESTS ###################################### ############################################################################################# From 0c42d3acef9e9c1e2c3b19bc9d3733ab9d6deb7f Mon Sep 17 00:00:00 2001 From: MyroTk <44327070+MyroTk@users.noreply.github.com> Date: Thu, 24 Aug 2023 08:30:05 -0700 Subject: [PATCH 41/72] Update release_branches.yml --- .github/workflows/release_branches.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/release_branches.yml b/.github/workflows/release_branches.yml index ca438d2ab108..61d1efd3c073 100644 --- a/.github/workflows/release_branches.yml +++ b/.github/workflows/release_branches.yml @@ -1071,8 +1071,8 @@ jobs: - FunctionalStatelessTestAarch64 - FunctionalStatefulTestRelease # - FunctionalStatefulTestAarch64 - - IntegrationTestsRelease0 - - IntegrationTestsRelease1 + # - IntegrationTestsRelease0 + # - IntegrationTestsRelease1 - CompatibilityCheck # - regression_common # - benchmark From fce67496c8ec5714c2849f77d58b99a5db7feec8 Mon Sep 17 00:00:00 2001 From: MyroTk <44327070+MyroTk@users.noreply.github.com> Date: Thu, 24 Aug 2023 10:08:50 -0700 Subject: [PATCH 42/72] Update run.sh --- docker/test/stateless/run.sh | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/docker/test/stateless/run.sh b/docker/test/stateless/run.sh index 226aeb6c558f..cc1a4a03ed20 100755 --- a/docker/test/stateless/run.sh +++ b/docker/test/stateless/run.sh @@ -2,6 +2,8 @@ # fail on errors, verbose and export all env variables set -e -x -a +echo "memory check" +free -h # Choose random timezone for this test run. TZ="$(grep -v '#' /usr/share/zoneinfo/zone.tab | awk '{print $3}' | shuf | head -n1)" @@ -109,7 +111,7 @@ function run_tests() # Too many tests fail for DatabaseReplicated in parallel. All other # configurations are OK. ADDITIONAL_OPTIONS+=('--jobs') - ADDITIONAL_OPTIONS+=('2') + ADDITIONAL_OPTIONS+=('1') fi if [[ -n "$RUN_BY_HASH_NUM" ]] && [[ -n "$RUN_BY_HASH_TOTAL" ]]; then From 4c036b2a1c2836349c24822cbf6319a1705f7660 Mon Sep 17 00:00:00 2001 From: MyroTk <44327070+MyroTk@users.noreply.github.com> Date: Wed, 30 Aug 2023 21:58:41 -0700 Subject: [PATCH 43/72] Update release_branches.yml --- .github/workflows/release_branches.yml | 1065 +++++++++++++----------- 1 file changed, 574 insertions(+), 491 deletions(-) diff --git a/.github/workflows/release_branches.yml b/.github/workflows/release_branches.yml index 61d1efd3c073..492941c281ad 100644 --- a/.github/workflows/release_branches.yml +++ b/.github/workflows/release_branches.yml @@ -532,496 +532,578 @@ jobs: ############################################################################################# ############################# INTEGRATION TESTS ############################################# ############################################################################################# - # IntegrationTestsRelease0: - # needs: [BuilderDebRelease] - # runs-on: [self-hosted, stress-tester, on-demand, type-ccx52] - # steps: - # - name: Set envs - # run: | - # cat >> "$GITHUB_ENV" << 'EOF' - # TEMP_PATH=${{runner.temp}}/integration_tests_release - # REPORTS_PATH=${{runner.temp}}/reports_dir - # CHECK_NAME=Integration tests (release) - # REPO_COPY=${{runner.temp}}/integration_tests_release/ClickHouse - # RUN_BY_HASH_NUM=0 - # RUN_BY_HASH_TOTAL=2 - # EOF - # - name: Download json reports - # uses: actions/download-artifact@v3 - # with: - # path: ${{ env.REPORTS_PATH }} - # - name: Check out repository code - # uses: ClickHouse/checkout@v1 - # with: - # clear-repository: true - # - name: Integration test - # run: | - # sudo rm -fr "$TEMP_PATH" - # mkdir -p "$TEMP_PATH" - # cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" - # cd "$REPO_COPY/tests/ci" - # python3 integration_test_check.py "$CHECK_NAME" - # - name: Cleanup - # if: always() - # run: | - # # shellcheck disable=SC2046 - # docker kill $(docker ps -q) ||: - # # shellcheck disable=SC2046 - # docker rm -f $(docker ps -a -q) ||: - # sudo rm -fr "$TEMP_PATH" - # IntegrationTestsRelease1: - # needs: [BuilderDebRelease] - # runs-on: [self-hosted, stress-tester, on-demand, type-ccx52] - # steps: - # - name: Set envs - # run: | - # cat >> "$GITHUB_ENV" << 'EOF' - # TEMP_PATH=${{runner.temp}}/integration_tests_release - # REPORTS_PATH=${{runner.temp}}/reports_dir - # CHECK_NAME=Integration tests (release) - # REPO_COPY=${{runner.temp}}/integration_tests_release/ClickHouse - # RUN_BY_HASH_NUM=1 - # RUN_BY_HASH_TOTAL=2 - # EOF - # - name: Download json reports - # uses: actions/download-artifact@v3 - # with: - # path: ${{ env.REPORTS_PATH }} - # - name: Check out repository code - # uses: ClickHouse/checkout@v1 - # with: - # clear-repository: true - # - name: Integration test - # run: | - # sudo rm -fr "$TEMP_PATH" - # mkdir -p "$TEMP_PATH" - # cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" - # cd "$REPO_COPY/tests/ci" - # python3 integration_test_check.py "$CHECK_NAME" - # - name: Cleanup - # if: always() - # run: | - # # shellcheck disable=SC2046 - # docker kill $(docker ps -q) ||: - # # shellcheck disable=SC2046 - # docker rm -f $(docker ps -a -q) ||: - # sudo rm -fr "$TEMP_PATH" + IntegrationTestsRelease0: + needs: [BuilderDebRelease] + runs-on: [self-hosted, stress-tester, func-tester] + steps: + - name: Set envs + run: | + cat >> "$GITHUB_ENV" << 'EOF' + TEMP_PATH=${{runner.temp}}/integration_tests_release + REPORTS_PATH=${{runner.temp}}/reports_dir + CHECK_NAME=Integration tests (release) + REPO_COPY=${{runner.temp}}/integration_tests_release/ClickHouse + RUN_BY_HASH_NUM=0 + RUN_BY_HASH_TOTAL=2 + EOF + - name: Download json reports + uses: actions/download-artifact@v3 + with: + path: ${{ env.REPORTS_PATH }} + - name: Check out repository code + uses: ClickHouse/checkout@v1 + with: + clear-repository: true + - name: Integration test + run: | + sudo rm -fr "$TEMP_PATH" + mkdir -p "$TEMP_PATH" + cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" + cd "$REPO_COPY/tests/ci" + python3 integration_test_check.py "$CHECK_NAME" + - name: Cleanup + if: always() + run: | + # shellcheck disable=SC2046 + docker kill $(docker ps -q) ||: + # shellcheck disable=SC2046 + docker rm -f $(docker ps -a -q) ||: + sudo rm -fr "$TEMP_PATH" + IntegrationTestsRelease1: + needs: [BuilderDebRelease] + runs-on: [self-hosted, stress-tester, func-tester] + steps: + - name: Set envs + run: | + cat >> "$GITHUB_ENV" << 'EOF' + TEMP_PATH=${{runner.temp}}/integration_tests_release + REPORTS_PATH=${{runner.temp}}/reports_dir + CHECK_NAME=Integration tests (release) + REPO_COPY=${{runner.temp}}/integration_tests_release/ClickHouse + RUN_BY_HASH_NUM=1 + RUN_BY_HASH_TOTAL=2 + EOF + - name: Download json reports + uses: actions/download-artifact@v3 + with: + path: ${{ env.REPORTS_PATH }} + - name: Check out repository code + uses: ClickHouse/checkout@v1 + with: + clear-repository: true + - name: Integration test + run: | + sudo rm -fr "$TEMP_PATH" + mkdir -p "$TEMP_PATH" + cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" + cd "$REPO_COPY/tests/ci" + python3 integration_test_check.py "$CHECK_NAME" + - name: Cleanup + if: always() + run: | + # shellcheck disable=SC2046 + docker kill $(docker ps -q) ||: + # shellcheck disable=SC2046 + docker rm -f $(docker ps -a -q) ||: + sudo rm -fr "$TEMP_PATH" ############################################################################################# ##################################### REGRESSION TESTS ###################################### ############################################################################################# -# regression_start: -# ## Not depending on the tests above since they can fail at any given moment. -# needs: [BuilderDebRelease] -# runs-on: ubuntu-latest -# steps: -# - run: true - -# regression_common: -# strategy: -# fail-fast: false -# matrix: -# SUITE: [aes_encryption, aggregate_functions, atomic_insert, base_58, clickhouse_keeper, datetime64_extended_range, disk_level_encryption, dns, example, extended_precision_data_types, kafka, kerberos, lightweight_delete, map_type, part_moves_between_shards, rbac, selects, ssl_server, tiered_storage, window_functions] -# needs: [regression_start] -# runs-on: [self-hosted, stress-tester] -# env: -# AWS_ACCESS_KEY_ID: ${{ secrets.AWS_REPORT_KEY_ID }} -# AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_REPORT_SECRET_ACCESS_KEY }} -# AWS_DEFAULT_REGION: ${{ secrets.AWS_REPORT_REGION }} -# steps: -# - name: Checkout regression repo -# uses: actions/checkout@v3 -# with: -# repository: Altinity/clickhouse-regression -# ref: releases -# - name: Set envs -# run: | -# cat >> "$GITHUB_ENV" << 'EOF' -# REPORTS_PATH=${{runner.temp}}/reports_dir -# SUITE=${{ matrix.SUITE }} -# artifacts=public -# EOF -# - name: Download json reports -# uses: actions/download-artifact@v3 -# with: -# path: ${{ env.REPORTS_PATH }} -# - name: Setup -# run: .github/setup.sh -# - name: Get deb url -# run: python3 .github/get-deb-url.py --reports-path ${{ env.REPORTS_PATH }} --github-env $GITHUB_ENV -# - name: Run ${{ env.SUITE }} suite -# run: python3 -# -u ${{ env.SUITE }}/regression.py -# --clickhouse-binary-path ${{ env.clickhouse_binary_path }} -# --test-to-end -# --local -# --collect-service-logs -# --output classic -# --parallel 1 -# --attr project="$GITHUB_REPOSITORY" project.id="$GITHUB_REPOSITORY_ID" package="${{ env.clickhouse_binary_path }}" version="${{ env.version }}" user.name="$GITHUB_ACTOR" repository="https://github.com/Altinity/clickhouse-regression" commit.hash="$(git rev-parse HEAD)" job.id="$GITHUB_RUN_ID" job.url="$GITHUB_SERVER_URL/$GITHUB_REPOSITORY/actions/runs/$GITHUB_RUN_ID" arch="x86_64" -# --log raw.log -# - name: Create and upload logs -# if: always() -# run: .github/create_and_upload_logs.sh 1 -# env: -# artifact_s3_dir: build/v${{ env.version }}/$GITHUB_SHA -# - uses: actions/upload-artifact@v3 -# if: always() -# with: -# name: ${{ env.SUITE }}-artifacts -# path: | -# ./report.html -# ./*.log.txt -# ./*.log -# ./*.html -# ./*/_instances/*.log -# ./*/_instances/*/logs/*.log -# ./*/*/_instances/*/logs/*.log -# ./*/*/_instances/*.log - -# benchmark: -# strategy: -# fail-fast: false -# matrix: -# STORAGE: [minio, aws_s3, gcs] -# needs: [regression_start] -# runs-on: [self-hosted, stress-tester] -# env: -# AWS_ACCESS_KEY_ID: ${{ secrets.AWS_REPORT_KEY_ID }} -# AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_REPORT_SECRET_ACCESS_KEY }} -# AWS_DEFAULT_REGION: ${{ secrets.AWS_REPORT_REGION }} -# steps: -# - name: Checkout regression repo -# uses: actions/checkout@v3 -# with: -# repository: Altinity/clickhouse-regression -# - name: Set envs -# run: | -# cat >> "$GITHUB_ENV" << 'EOF' -# REPORTS_PATH=${{runner.temp}}/reports_dir -# SUITE=ontime_benchmark -# STORAGE=/${{ matrix.STORAGE }} -# artifacts=public -# EOF -# - name: Download json reports -# uses: actions/download-artifact@v3 -# with: -# path: ${{ env.REPORTS_PATH }} -# - name: Setup -# run: .github/setup.sh -# - name: Get deb url -# run: python3 .github/get-deb-url.py --reports-path ${{ env.REPORTS_PATH }} --github-env $GITHUB_ENV -# - name: Run ${{ env.SUITE }} suite -# run: python3 -# -u ${{ env.SUITE }}/benchmark.py -# --clickhouse-binary-path ${{ env.clickhouse_binary_path }} -# --storage ${{ matrix.STORAGE }} -# --gcs-uri ${{ secrets.REGRESSION_GCS_URI }} -# --gcs-key-id ${{ secrets.REGRESSION_GCS_KEY_ID }} -# --gcs-key-secret ${{ secrets.REGRESSION_GCS_KEY_SECRET }} -# --aws-s3-bucket ${{ secrets.REGRESSION_AWS_S3_BUCKET }} -# --aws-s3-region ${{ secrets.REGRESSION_AWS_S3_REGION }} -# --aws-s3-key-id ${{ secrets.REGRESSION_AWS_S3_KEY_ID }} -# --aws-s3-access-key ${{ secrets.REGRESSION_AWS_S3_SECRET_ACCESS_KEY }} -# --test-to-end -# --local -# --collect-service-logs -# --output classic -# --parallel 1 -# --attr project="$GITHUB_REPOSITORY" project.id="$GITHUB_REPOSITORY_ID" package="${{ env.clickhouse_binary_path }}" version="${{ env.version }}" user.name="$GITHUB_ACTOR" repository="https://github.com/Altinity/clickhouse-regression" commit.hash="$(git rev-parse HEAD)" job.id="$GITHUB_RUN_ID" job.url="$GITHUB_SERVER_URL/$GITHUB_REPOSITORY/actions/runs/$GITHUB_RUN_ID" arch="x86_64" -# --log raw.log -# - name: Create and upload logs -# if: always() -# run: .github/create_and_upload_logs.sh 1 -# env: -# artifact_s3_dir: build/v${{ env.version }}/$GITHUB_SHA -# - uses: actions/upload-artifact@v3 -# if: always() -# with: -# name: ${{ env.SUITE }}-minio-artifacts -# path: | -# ./report.html -# ./*.log.txt -# ./*.log -# ./*.html -# ./*/_instances/*.log -# ./*/_instances/*/logs/*.log -# ./*/*/_instances/*/logs/*.log -# ./*/*/_instances/*.log - -# ldap: -# strategy: -# fail-fast: false -# matrix: -# SUITE: [authentication, external_user_directory, role_mapping] -# needs: [regression_start] -# runs-on: [self-hosted, stress-tester] -# env: -# AWS_ACCESS_KEY_ID: ${{ secrets.AWS_REPORT_KEY_ID }} -# AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_REPORT_SECRET_ACCESS_KEY }} -# AWS_DEFAULT_REGION: ${{ secrets.AWS_REPORT_REGION }} -# steps: -# - name: Checkout regression repo -# uses: actions/checkout@v3 -# with: -# repository: Altinity/clickhouse-regression -# ref: releases -# - name: Set envs -# run: | -# cat >> "$GITHUB_ENV" << 'EOF' -# REPORTS_PATH=${{runner.temp}}/reports_dir -# SUITE=ldap/${{ matrix.SUITE }} -# artifacts=public -# EOF -# - name: Download json reports -# uses: actions/download-artifact@v3 -# with: -# path: ${{ env.REPORTS_PATH }} -# - name: Setup -# run: .github/setup.sh -# - name: Get deb url -# run: python3 .github/get-deb-url.py --reports-path ${{ env.REPORTS_PATH }} --github-env $GITHUB_ENV -# - name: Run ${{ env.SUITE }} suite -# run: python3 -# -u ${{ env.SUITE }}/regression.py -# --clickhouse-binary-path ${{ env.clickhouse_binary_path }} -# --test-to-end -# --local -# --collect-service-logs -# --output classic -# --parallel 1 -# --attr project="$GITHUB_REPOSITORY" project.id="$GITHUB_REPOSITORY_ID" package="${{ env.clickhouse_binary_path }}" version="${{ env.version }}" user.name="$GITHUB_ACTOR" repository="https://github.com/Altinity/clickhouse-regression" commit.hash="$(git rev-parse HEAD)" job.id="$GITHUB_RUN_ID" job.url="$GITHUB_SERVER_URL/$GITHUB_REPOSITORY/actions/runs/$GITHUB_RUN_ID" arch="x86_64" -# --log raw.log -# - name: Create and upload logs -# if: always() -# run: .github/create_and_upload_logs.sh 1 -# env: -# artifact_s3_dir: build/v${{ env.version }}/$GITHUB_SHA -# - uses: actions/upload-artifact@v3 -# if: always() -# with: -# name: ldap-authentication-artifacts -# path: | -# ./report.html -# ./*.log.txt -# ./*.log -# ./*.html -# ./*/_instances/*.log -# ./*/_instances/*/logs/*.log -# ./*/*/_instances/*/logs/*.log -# ./*/*/_instances/*.log - -# parquet: -# needs: [regression_start] -# runs-on: [self-hosted, stress-tester] -# env: -# AWS_ACCESS_KEY_ID: ${{ secrets.AWS_REPORT_KEY_ID }} -# AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_REPORT_SECRET_ACCESS_KEY }} -# AWS_DEFAULT_REGION: ${{ secrets.AWS_REPORT_REGION }} -# steps: -# - name: Checkout regression repo -# uses: actions/checkout@v3 -# with: -# repository: Altinity/clickhouse-regression -# ref: releases -# - name: Set envs -# run: | -# cat >> "$GITHUB_ENV" << 'EOF' -# REPORTS_PATH=${{runner.temp}}/reports_dir -# SUITE=parquet -# artifacts=public -# EOF -# - name: Download json reports -# uses: actions/download-artifact@v3 -# with: -# path: ${{ env.REPORTS_PATH }} -# - name: Setup -# run: .github/setup.sh -# - name: Get deb url -# run: python3 .github/get-deb-url.py --reports-path ${{ env.REPORTS_PATH }} --github-env $GITHUB_ENV -# - name: Run ${{ env.SUITE }} suite -# run: python3 -# -u ${{ env.SUITE }}/regression.py -# --clickhouse-binary-path ${{ env.clickhouse_binary_path }} -# --test-to-end -# --local -# --collect-service-logs -# --output classic -# --parallel 1 -# --attr project="$GITHUB_REPOSITORY" project.id="$GITHUB_REPOSITORY_ID" package="${{ env.clickhouse_binary_path }}" version="${{ env.version }}" user.name="$GITHUB_ACTOR" repository="https://github.com/Altinity/clickhouse-regression" commit.hash="$(git rev-parse HEAD)" job.id="$GITHUB_RUN_ID" job.url="$GITHUB_SERVER_URL/$GITHUB_REPOSITORY/actions/runs/$GITHUB_RUN_ID" arch="x86_64" -# --log raw.log -# --storage minio -# --storage aws_s3 -# --aws-s3-bucket ${{ secrets.REGRESSION_AWS_S3_BUCKET }} -# --aws-s3-region ${{ secrets.REGRESSION_AWS_S3_REGION }} -# --aws-s3-key-id ${{ secrets.REGRESSION_AWS_S3_KEY_ID }} -# --aws-s3-access-key ${{ secrets.REGRESSION_AWS_S3_SECRET_ACCESS_KEY }} -# --storage gcs -# --gcs-uri ${{ secrets.REGRESSION_GCS_URI }} -# --gcs-key-id ${{ secrets.REGRESSION_GCS_KEY_ID }} -# --gcs-key-secret ${{ secrets.REGRESSION_GCS_KEY_SECRET }} -# - name: Create and upload logs -# if: always() -# run: .github/create_and_upload_logs.sh 1 -# env: -# artifact_s3_dir: build/v${{ env.version }}/$GITHUB_SHA -# - uses: actions/upload-artifact@v3 -# if: always() -# with: -# name: ${{ env.SUITE }}-artifacts -# path: | -# ./report.html -# ./*.log.txt -# ./*.log -# ./*.html -# ./*/_instances/*.log -# ./*/_instances/*/logs/*.log -# ./*/*/_instances/*/logs/*.log -# ./*/*/_instances/*.log + RegressionStart: + ## Not depending on the tests above since they can fail at any given moment. + needs: [BuilderDebRelease] + runs-on: ubuntu-latest + steps: + - run: true + + RegressionCommonAmd64: + strategy: + fail-fast: false + matrix: + SUITE: [aes_encryption, aggregate_functions, atomic_insert, base_58, clickhouse_keeper, datetime64_extended_range, disk_level_encryption, dns, example, extended_precision_data_types, kafka, kerberos, lightweight_delete, map_type, part_moves_between_shards, rbac, selects, ssl_server, tiered_storage, window_functions] + needs: [RegressionStart] + runs-on: [self-hosted, stress-tester] + env: + AWS_ACCESS_KEY_ID: ${{ secrets.AWS_REPORT_KEY_ID }} + AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_REPORT_SECRET_ACCESS_KEY }} + AWS_DEFAULT_REGION: ${{ secrets.AWS_REPORT_REGION }} + steps: + - name: Checkout regression repo + uses: actions/checkout@v3 + with: + repository: Altinity/clickhouse-regression + ref: releases + - name: Set envs + run: | + cat >> "$GITHUB_ENV" << 'EOF' + REPORTS_PATH=${{runner.temp}}/reports_dir + SUITE=${{ matrix.SUITE }} + artifacts=public + EOF + - name: Download json reports + uses: actions/download-artifact@v3 + with: + path: ${{ env.REPORTS_PATH }} + - name: Setup + run: .github/setup.sh + - name: Get deb url + run: python3 .github/get-deb-url.py --reports-path ${{ env.REPORTS_PATH }} --github-env $GITHUB_ENV + - name: Run ${{ env.SUITE }} suite + run: python3 + -u ${{ env.SUITE }}/regression.py + --clickhouse-binary-path ${{ env.clickhouse_binary_path }} + --test-to-end + --local + --collect-service-logs + --output classic + --parallel 1 + --attr project="$GITHUB_REPOSITORY" project.id="$GITHUB_REPOSITORY_ID" package="${{ env.clickhouse_binary_path }}" version="${{ env.version }}" user.name="$GITHUB_ACTOR" repository="https://github.com/Altinity/clickhouse-regression" commit.hash="$(git rev-parse HEAD)" job.id="$GITHUB_RUN_ID" job.url="$GITHUB_SERVER_URL/$GITHUB_REPOSITORY/actions/runs/$GITHUB_RUN_ID" arch="x86_64" + --log raw.log + - name: Create and upload logs + if: always() + run: .github/create_and_upload_logs.sh 1 + env: + artifact_s3_dir: build/v${{ env.version }}/$GITHUB_SHA + - uses: actions/upload-artifact@v3 + if: always() + with: + name: ${{ env.SUITE }}-amd64-artifacts + path: | + ./report.html + ./*.log.txt + ./*.log + ./*.html + ./*/_instances/*.log + ./*/_instances/*/logs/*.log + ./*/*/_instances/*/logs/*.log + ./*/*/_instances/*.log -# s3: -# strategy: -# fail-fast: false -# matrix: -# STORAGE: [minio, aws_s3, gcs] -# needs: [regression_start] -# runs-on: [self-hosted, stress-tester] -# env: -# AWS_ACCESS_KEY_ID: ${{ secrets.AWS_REPORT_KEY_ID }} -# AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_REPORT_SECRET_ACCESS_KEY }} -# AWS_DEFAULT_REGION: ${{ secrets.AWS_REPORT_REGION }} -# steps: -# - name: Checkout regression repo -# uses: actions/checkout@v3 -# with: -# repository: Altinity/clickhouse-regression -# ref: releases -# - name: Set envs -# run: | -# cat >> "$GITHUB_ENV" << 'EOF' -# REPORTS_PATH=${{runner.temp}}/reports_dir -# SUITE=s3 -# STORAGE=/${{ matrix.STORAGE }} -# artifacts=public -# EOF -# - name: Download json reports -# uses: actions/download-artifact@v3 -# with: -# path: ${{ env.REPORTS_PATH }} -# - name: Setup -# run: .github/setup.sh -# - name: Get deb url -# run: python3 .github/get-deb-url.py --reports-path ${{ env.REPORTS_PATH }} --github-env $GITHUB_ENV -# - name: Run ${{ env.SUITE }} suite -# run: python3 -# -u ${{ env.SUITE }}/regression.py -# --clickhouse-binary-path ${{ env.clickhouse_binary_path }} -# --test-to-end -# --local -# --collect-service-logs -# --output classic -# --parallel 1 -# --attr project="$GITHUB_REPOSITORY" project.id="$GITHUB_REPOSITORY_ID" package="${{ env.clickhouse_binary_path }}" version="${{ env.version }}" user.name="$GITHUB_ACTOR" repository="https://github.com/Altinity/clickhouse-regression" commit.hash="$(git rev-parse HEAD)" job.id="$GITHUB_RUN_ID" job.url="$GITHUB_SERVER_URL/$GITHUB_REPOSITORY/actions/runs/$GITHUB_RUN_ID" arch="x86_64" -# --log raw.log -# --storage ${{ matrix.STORAGE }} -# --gcs-uri ${{ secrets.REGRESSION_GCS_URI }} -# --gcs-key-id ${{ secrets.REGRESSION_GCS_KEY_ID }} -# --gcs-key-secret ${{ secrets.REGRESSION_GCS_KEY_SECRET }} -# --aws-s3-bucket ${{ secrets.REGRESSION_AWS_S3_BUCKET }} -# --aws-s3-region ${{ secrets.REGRESSION_AWS_S3_REGION }} -# --aws-s3-key-id ${{ secrets.REGRESSION_AWS_S3_KEY_ID }} -# --aws-s3-access-key ${{ secrets.REGRESSION_AWS_S3_SECRET_ACCESS_KEY }} -# - name: Create and upload logs -# if: always() -# run: .github/create_and_upload_logs.sh 1 -# env: -# artifact_s3_dir: build/v${{ env.version }}/$GITHUB_SHA -# - uses: actions/upload-artifact@v3 -# if: always() -# with: -# name: ${{ env.SUITE }}-${{ matrix.STORAGE }}-artifacts -# path: | -# ./report.html -# ./*.log.txt -# ./*.log -# ./*.html -# ./*/_instances/*.log -# ./*/_instances/*/logs/*.log -# ./*/*/_instances/*/logs/*.log -# ./*/*/_instances/*.log + RegressionCommonAarch64: + strategy: + fail-fast: false + matrix: + SUITE: [aes_encryption, base_58, datetime64_extended_range, disk_level_encryption, example, extended_precision_data_types, kafka, lightweight_delete, map_type, part_moves_between_shards, rbac, selects, tiered_storage] + needs: [RegressionStart] + runs-on: [self-hosted, regression-tester, on-demand, type-cax41, image-arm-app-docker-ce] + env: + AWS_ACCESS_KEY_ID: ${{ secrets.AWS_REPORT_KEY_ID }} + AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_REPORT_SECRET_ACCESS_KEY }} + AWS_DEFAULT_REGION: ${{ secrets.AWS_REPORT_REGION }} + steps: + - name: Checkout regression repo + uses: actions/checkout@v3 + with: + repository: Altinity/clickhouse-regression + ref: releases + - name: Set envs + run: | + cat >> "$GITHUB_ENV" << 'EOF' + REPORTS_PATH=${{runner.temp}}/reports_dir + SUITE=${{ matrix.SUITE }} + artifacts=public + EOF + - name: Download json reports + uses: actions/download-artifact@v3 + with: + path: ${{ env.REPORTS_PATH }} + - name: Setup + run: .github/setup.sh + - name: Get deb url + run: python3 .github/get-deb-url.py --reports-path ${{ env.REPORTS_PATH }} --github-env $GITHUB_ENV + - name: Run ${{ env.SUITE }} suite + run: python3 + -u ${{ env.SUITE }}/regression.py + --clickhouse-binary-path ${{ env.clickhouse_binary_path }} + --test-to-end + --local + --collect-service-logs + --output classic + --parallel 1 + --attr project="$GITHUB_REPOSITORY" project.id="$GITHUB_REPOSITORY_ID" package="${{ env.clickhouse_binary_path }}" version="${{ env.version }}" user.name="$GITHUB_ACTOR" repository="https://github.com/Altinity/clickhouse-regression" commit.hash="$(git rev-parse HEAD)" job.id="$GITHUB_RUN_ID" job.url="$GITHUB_SERVER_URL/$GITHUB_REPOSITORY/actions/runs/$GITHUB_RUN_ID" arch="x86_64" + --log raw.log + - name: Create and upload logs + if: always() + run: .github/create_and_upload_logs.sh 1 + env: + artifact_s3_dir: build/v${{ env.version }}/$GITHUB_SHA + - uses: actions/upload-artifact@v3 + if: always() + with: + name: ${{ env.SUITE }}-aarch64-artifacts + path: | + ./report.html + ./*.log.txt + ./*.log + ./*.html + ./*/_instances/*.log + ./*/_instances/*/logs/*.log + ./*/*/_instances/*/logs/*.log + ./*/*/_instances/*.log + + RegressionBenchmark: + strategy: + fail-fast: false + matrix: + STORAGE: [minio, aws_s3, gcs] + include: + - ARCH: arm + SERVER: cax41 + - ARCH: x86 + SERVER: cpx51 + needs: [RegressionStart] + runs-on: [self-hosted, regression-tester, on-demand, 'type-${{ matrix.SERVER }}', 'image-${{ matrix.ARCH }}-app-docker-ce'] + env: + AWS_ACCESS_KEY_ID: ${{ secrets.AWS_REPORT_KEY_ID }} + AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_REPORT_SECRET_ACCESS_KEY }} + AWS_DEFAULT_REGION: ${{ secrets.AWS_REPORT_REGION }} + steps: + - name: Checkout regression repo + uses: actions/checkout@v3 + with: + repository: Altinity/clickhouse-regression + - name: Set envs + run: | + cat >> "$GITHUB_ENV" << 'EOF' + REPORTS_PATH=${{runner.temp}}/reports_dir + SUITE=ontime_benchmark + STORAGE=/${{ matrix.STORAGE }} + artifacts=public + EOF + - name: Download json reports + uses: actions/download-artifact@v3 + with: + path: ${{ env.REPORTS_PATH }} + - name: Setup + run: .github/setup.sh + - name: Get deb url + run: python3 .github/get-deb-url.py --reports-path ${{ env.REPORTS_PATH }} --github-env $GITHUB_ENV + - name: Run ${{ env.SUITE }} suite + run: python3 + -u ${{ env.SUITE }}/benchmark.py + --clickhouse-binary-path ${{ env.clickhouse_binary_path }} + --storage ${{ matrix.STORAGE }} + --gcs-uri ${{ secrets.REGRESSION_GCS_URI }} + --gcs-key-id ${{ secrets.REGRESSION_GCS_KEY_ID }} + --gcs-key-secret ${{ secrets.REGRESSION_GCS_KEY_SECRET }} + --aws-s3-bucket ${{ secrets.REGRESSION_AWS_S3_BUCKET }} + --aws-s3-region ${{ secrets.REGRESSION_AWS_S3_REGION }} + --aws-s3-key-id ${{ secrets.REGRESSION_AWS_S3_KEY_ID }} + --aws-s3-access-key ${{ secrets.REGRESSION_AWS_S3_SECRET_ACCESS_KEY }} + --test-to-end + --local + --collect-service-logs + --output classic + --parallel 1 + --attr project="$GITHUB_REPOSITORY" project.id="$GITHUB_REPOSITORY_ID" package="${{ env.clickhouse_binary_path }}" version="${{ env.version }}" user.name="$GITHUB_ACTOR" repository="https://github.com/Altinity/clickhouse-regression" commit.hash="$(git rev-parse HEAD)" job.id="$GITHUB_RUN_ID" job.url="$GITHUB_SERVER_URL/$GITHUB_REPOSITORY/actions/runs/$GITHUB_RUN_ID" arch="x86_64" + --log raw.log + - name: Create and upload logs + if: always() + run: .github/create_and_upload_logs.sh 1 + env: + artifact_s3_dir: build/v${{ env.version }}/$GITHUB_SHA + - uses: actions/upload-artifact@v3 + if: always() + with: + name: ${{ env.SUITE }}-minio-artifacts + path: | + ./report.html + ./*.log.txt + ./*.log + ./*.html + ./*/_instances/*.log + ./*/_instances/*/logs/*.log + ./*/*/_instances/*/logs/*.log + ./*/*/_instances/*.log + + RegressionLDAP: + strategy: + fail-fast: false + matrix: + SUITE: [authentication, external_user_directory, role_mapping] + include: + - ARCH: arm + SERVER: cax41 + - ARCH: x86 + SERVER: cpx51 + needs: [RegressionStart] + runs-on: [self-hosted, regression-tester, on-demand, 'type-${{ matrix.SERVER }}', 'image-${{ matrix.ARCH }}-app-docker-ce'] + env: + AWS_ACCESS_KEY_ID: ${{ secrets.AWS_REPORT_KEY_ID }} + AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_REPORT_SECRET_ACCESS_KEY }} + AWS_DEFAULT_REGION: ${{ secrets.AWS_REPORT_REGION }} + steps: + - name: Checkout regression repo + uses: actions/checkout@v3 + with: + repository: Altinity/clickhouse-regression + ref: releases + - name: Set envs + run: | + cat >> "$GITHUB_ENV" << 'EOF' + REPORTS_PATH=${{runner.temp}}/reports_dir + SUITE=ldap/${{ matrix.SUITE }} + artifacts=public + EOF + - name: Download json reports + uses: actions/download-artifact@v3 + with: + path: ${{ env.REPORTS_PATH }} + - name: Setup + run: .github/setup.sh + - name: Get deb url + run: python3 .github/get-deb-url.py --reports-path ${{ env.REPORTS_PATH }} --github-env $GITHUB_ENV + - name: Run ${{ env.SUITE }} suite + run: python3 + -u ${{ env.SUITE }}/regression.py + --clickhouse-binary-path ${{ env.clickhouse_binary_path }} + --test-to-end + --local + --collect-service-logs + --output classic + --parallel 1 + --attr project="$GITHUB_REPOSITORY" project.id="$GITHUB_REPOSITORY_ID" package="${{ env.clickhouse_binary_path }}" version="${{ env.version }}" user.name="$GITHUB_ACTOR" repository="https://github.com/Altinity/clickhouse-regression" commit.hash="$(git rev-parse HEAD)" job.id="$GITHUB_RUN_ID" job.url="$GITHUB_SERVER_URL/$GITHUB_REPOSITORY/actions/runs/$GITHUB_RUN_ID" arch="x86_64" + --log raw.log + - name: Create and upload logs + if: always() + run: .github/create_and_upload_logs.sh 1 + env: + artifact_s3_dir: build/v${{ env.version }}/$GITHUB_SHA + - uses: actions/upload-artifact@v3 + if: always() + with: + name: ldap-authentication-artifacts + path: | + ./report.html + ./*.log.txt + ./*.log + ./*.html + ./*/_instances/*.log + ./*/_instances/*/logs/*.log + ./*/*/_instances/*/logs/*.log + ./*/*/_instances/*.log + + RegressionParquetAmd64: + needs: [RegressionStart] + runs-on: [self-hosted, stress-tester] + env: + AWS_ACCESS_KEY_ID: ${{ secrets.AWS_REPORT_KEY_ID }} + AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_REPORT_SECRET_ACCESS_KEY }} + AWS_DEFAULT_REGION: ${{ secrets.AWS_REPORT_REGION }} + steps: + - name: Checkout regression repo + uses: actions/checkout@v3 + with: + repository: Altinity/clickhouse-regression + ref: releases + - name: Set envs + run: | + cat >> "$GITHUB_ENV" << 'EOF' + REPORTS_PATH=${{runner.temp}}/reports_dir + SUITE=parquet + artifacts=public + EOF + - name: Download json reports + uses: actions/download-artifact@v3 + with: + path: ${{ env.REPORTS_PATH }} + - name: Setup + run: .github/setup.sh + - name: Get deb url + run: python3 .github/get-deb-url.py --reports-path ${{ env.REPORTS_PATH }} --github-env $GITHUB_ENV + - name: Run ${{ env.SUITE }} suite + run: python3 + -u ${{ env.SUITE }}/regression.py + --clickhouse-binary-path ${{ env.clickhouse_binary_path }} + --test-to-end + --local + --collect-service-logs + --output classic + --parallel 1 + --attr project="$GITHUB_REPOSITORY" project.id="$GITHUB_REPOSITORY_ID" package="${{ env.clickhouse_binary_path }}" version="${{ env.version }}" user.name="$GITHUB_ACTOR" repository="https://github.com/Altinity/clickhouse-regression" commit.hash="$(git rev-parse HEAD)" job.id="$GITHUB_RUN_ID" job.url="$GITHUB_SERVER_URL/$GITHUB_REPOSITORY/actions/runs/$GITHUB_RUN_ID" arch="x86_64" + --log raw.log + --storage minio + --storage aws_s3 + --aws-s3-bucket ${{ secrets.REGRESSION_AWS_S3_BUCKET }} + --aws-s3-region ${{ secrets.REGRESSION_AWS_S3_REGION }} + --aws-s3-key-id ${{ secrets.REGRESSION_AWS_S3_KEY_ID }} + --aws-s3-access-key ${{ secrets.REGRESSION_AWS_S3_SECRET_ACCESS_KEY }} + --storage gcs + --gcs-uri ${{ secrets.REGRESSION_GCS_URI }} + --gcs-key-id ${{ secrets.REGRESSION_GCS_KEY_ID }} + --gcs-key-secret ${{ secrets.REGRESSION_GCS_KEY_SECRET }} + - name: Create and upload logs + if: always() + run: .github/create_and_upload_logs.sh 1 + env: + artifact_s3_dir: build/v${{ env.version }}/$GITHUB_SHA + - uses: actions/upload-artifact@v3 + if: always() + with: + name: ${{ env.SUITE }}-artifacts + path: | + ./report.html + ./*.log.txt + ./*.log + ./*.html + ./*/_instances/*.log + ./*/_instances/*/logs/*.log + ./*/*/_instances/*/logs/*.log + ./*/*/_instances/*.log + + RegressionS3: + strategy: + fail-fast: false + matrix: + STORAGE: [minio, aws_s3, gcs] + include: + - ARCH: arm + SERVER: cax41 + - ARCH: x86 + SERVER: cpx51 + needs: [RegressionStart] + runs-on: [self-hosted, regression-tester, on-demand, 'type-${{ matrix.SERVER }}', 'image-${{ matrix.ARCH }}-app-docker-ce'] + env: + AWS_ACCESS_KEY_ID: ${{ secrets.AWS_REPORT_KEY_ID }} + AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_REPORT_SECRET_ACCESS_KEY }} + AWS_DEFAULT_REGION: ${{ secrets.AWS_REPORT_REGION }} + steps: + - name: Checkout regression repo + uses: actions/checkout@v3 + with: + repository: Altinity/clickhouse-regression + ref: releases + - name: Set envs + run: | + cat >> "$GITHUB_ENV" << 'EOF' + REPORTS_PATH=${{runner.temp}}/reports_dir + SUITE=s3 + STORAGE=/${{ matrix.STORAGE }} + artifacts=public + EOF + - name: Download json reports + uses: actions/download-artifact@v3 + with: + path: ${{ env.REPORTS_PATH }} + - name: Setup + run: .github/setup.sh + - name: Get deb url + run: python3 .github/get-deb-url.py --reports-path ${{ env.REPORTS_PATH }} --github-env $GITHUB_ENV + - name: Run ${{ env.SUITE }} suite + run: python3 + -u ${{ env.SUITE }}/regression.py + --clickhouse-binary-path ${{ env.clickhouse_binary_path }} + --test-to-end + --local + --collect-service-logs + --output classic + --parallel 1 + --attr project="$GITHUB_REPOSITORY" project.id="$GITHUB_REPOSITORY_ID" package="${{ env.clickhouse_binary_path }}" version="${{ env.version }}" user.name="$GITHUB_ACTOR" repository="https://github.com/Altinity/clickhouse-regression" commit.hash="$(git rev-parse HEAD)" job.id="$GITHUB_RUN_ID" job.url="$GITHUB_SERVER_URL/$GITHUB_REPOSITORY/actions/runs/$GITHUB_RUN_ID" arch="x86_64" + --log raw.log + --storage ${{ matrix.STORAGE }} + --gcs-uri ${{ secrets.REGRESSION_GCS_URI }} + --gcs-key-id ${{ secrets.REGRESSION_GCS_KEY_ID }} + --gcs-key-secret ${{ secrets.REGRESSION_GCS_KEY_SECRET }} + --aws-s3-bucket ${{ secrets.REGRESSION_AWS_S3_BUCKET }} + --aws-s3-region ${{ secrets.REGRESSION_AWS_S3_REGION }} + --aws-s3-key-id ${{ secrets.REGRESSION_AWS_S3_KEY_ID }} + --aws-s3-access-key ${{ secrets.REGRESSION_AWS_S3_SECRET_ACCESS_KEY }} + - name: Create and upload logs + if: always() + run: .github/create_and_upload_logs.sh 1 + env: + artifact_s3_dir: build/v${{ env.version }}/$GITHUB_SHA + - uses: actions/upload-artifact@v3 + if: always() + with: + name: ${{ env.SUITE }}-${{ matrix.STORAGE }}-${{ matrix.OS }}-artifacts + path: | + ./report.html + ./*.log.txt + ./*.log + ./*.html + ./*/_instances/*.log + ./*/_instances/*/logs/*.log + ./*/*/_instances/*/logs/*.log + ./*/*/_instances/*.log -# tiered_storage_s3: -# strategy: -# fail-fast: false -# matrix: -# STORAGE: [minio, s3amazon, s3gcs] -# needs: [regression_start] -# runs-on: [self-hosted, stress-tester] -# env: -# AWS_ACCESS_KEY_ID: ${{ secrets.AWS_REPORT_KEY_ID }} -# AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_REPORT_SECRET_ACCESS_KEY }} -# AWS_DEFAULT_REGION: ${{ secrets.AWS_REPORT_REGION }} -# steps: -# - name: Checkout regression repo -# uses: actions/checkout@v3 -# with: -# repository: Altinity/clickhouse-regression -# ref: releases -# - name: Set envs -# run: | -# cat >> "$GITHUB_ENV" << 'EOF' -# REPORTS_PATH=${{runner.temp}}/reports_dir -# SUITE=tiered_storage -# STORAGE=/${{ matrix.STORAGE }} -# artifacts=public -# EOF -# - name: Download json reports -# uses: actions/download-artifact@v3 -# with: -# path: ${{ env.REPORTS_PATH }} -# - name: Setup -# run: .github/setup.sh -# - name: Get deb url -# run: python3 .github/get-deb-url.py --reports-path ${{ env.REPORTS_PATH }} --github-env $GITHUB_ENV -# - name: Run ${{ env.SUITE }} suite -# run: python3 -# -u ${{ env.SUITE }}/regression.py -# --clickhouse-binary-path ${{ env.clickhouse_binary_path }} -# --test-to-end -# --local -# --collect-service-logs -# --output classic -# --parallel 1 -# --attr project="$GITHUB_REPOSITORY" project.id="$GITHUB_REPOSITORY_ID" package="${{ env.clickhouse_binary_path }}" version="${{ env.version }}" user.name="$GITHUB_ACTOR" repository="https://github.com/Altinity/clickhouse-regression" commit.hash="$(git rev-parse HEAD)" job.id="$GITHUB_RUN_ID" job.url="$GITHUB_SERVER_URL/$GITHUB_REPOSITORY/actions/runs/$GITHUB_RUN_ID" arch="x86_64" -# --log raw.log -# --aws-s3-access-key ${{ secrets.REGRESSION_AWS_S3_SECRET_ACCESS_KEY }} -# --aws-s3-key-id ${{ secrets.REGRESSION_AWS_S3_KEY_ID }} -# --aws-s3-uri https://s3.${{ secrets.REGRESSION_AWS_S3_REGION}}.amazonaws.com/${{ secrets.REGRESSION_AWS_S3_BUCKET }}/data/ -# --gcs-key-id ${{ secrets.REGRESSION_GCS_KEY_ID }} -# --gcs-key-secret ${{ secrets.REGRESSION_GCS_KEY_SECRET }} -# --gcs-uri ${{ secrets.REGRESSION_GCS_URI }} -# --with-${{ matrix.STORAGE }} -# - name: Create and upload logs -# if: always() -# run: .github/create_and_upload_logs.sh 1 -# env: -# artifact_s3_dir: build/v${{ env.version }}/$GITHUB_SHA -# - uses: actions/upload-artifact@v3 -# if: always() -# with: -# name: ${{ env.SUITE }}-${{ matrix.STORAGE }}-artifacts -# path: | -# ./report.html -# ./*.log.txt -# ./*.log -# ./*.html -# ./*/_instances/*.log -# ./*/_instances/*/logs/*.log -# ./*/*/_instances/*/logs/*.log -# ./*/*/_instances/*.log + RegressionTieredStorageS3: + strategy: + fail-fast: false + matrix: + STORAGE: [minio, aws_s3, gcs] + include: + - ARCH: arm + SERVER: cax41 + - ARCH: x86 + SERVER: cpx51 + needs: [RegressionStart] + runs-on: [self-hosted, regression-tester, on-demand, 'type-${{ matrix.SERVER }}', 'image-${{ matrix.ARCH }}-app-docker-ce'] + env: + AWS_ACCESS_KEY_ID: ${{ secrets.AWS_REPORT_KEY_ID }} + AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_REPORT_SECRET_ACCESS_KEY }} + AWS_DEFAULT_REGION: ${{ secrets.AWS_REPORT_REGION }} + steps: + - name: Checkout regression repo + uses: actions/checkout@v3 + with: + repository: Altinity/clickhouse-regression + ref: releases + - name: Set envs + run: | + cat >> "$GITHUB_ENV" << 'EOF' + REPORTS_PATH=${{runner.temp}}/reports_dir + SUITE=tiered_storage + STORAGE=/${{ matrix.STORAGE }} + artifacts=public + EOF + - name: Download json reports + uses: actions/download-artifact@v3 + with: + path: ${{ env.REPORTS_PATH }} + - name: Setup + run: .github/setup.sh + - name: Get deb url + run: python3 .github/get-deb-url.py --reports-path ${{ env.REPORTS_PATH }} --github-env $GITHUB_ENV + - name: Run ${{ env.SUITE }} suite + run: python3 + -u ${{ env.SUITE }}/regression.py + --clickhouse-binary-path ${{ env.clickhouse_binary_path }} + --test-to-end + --local + --collect-service-logs + --output classic + --parallel 1 + --attr project="$GITHUB_REPOSITORY" project.id="$GITHUB_REPOSITORY_ID" package="${{ env.clickhouse_binary_path }}" version="${{ env.version }}" user.name="$GITHUB_ACTOR" repository="https://github.com/Altinity/clickhouse-regression" commit.hash="$(git rev-parse HEAD)" job.id="$GITHUB_RUN_ID" job.url="$GITHUB_SERVER_URL/$GITHUB_REPOSITORY/actions/runs/$GITHUB_RUN_ID" arch="x86_64" + --log raw.log + --aws-s3-access-key ${{ secrets.REGRESSION_AWS_S3_SECRET_ACCESS_KEY }} + --aws-s3-key-id ${{ secrets.REGRESSION_AWS_S3_KEY_ID }} + --aws-s3-uri https://s3.${{ secrets.REGRESSION_AWS_S3_REGION}}.amazonaws.com/${{ secrets.REGRESSION_AWS_S3_BUCKET }}/data/ + --gcs-key-id ${{ secrets.REGRESSION_GCS_KEY_ID }} + --gcs-key-secret ${{ secrets.REGRESSION_GCS_KEY_SECRET }} + --gcs-uri ${{ secrets.REGRESSION_GCS_URI }} + --with-${{ matrix.STORAGE }} + - name: Create and upload logs + if: always() + run: .github/create_and_upload_logs.sh 1 + env: + artifact_s3_dir: build/v${{ env.version }}/$GITHUB_SHA + - uses: actions/upload-artifact@v3 + if: always() + with: + name: ${{ env.SUITE }}-${{ matrix.STORAGE }}-${{ matrix.OS }}-artifacts + path: | + ./report.html + ./*.log.txt + ./*.log + ./*.html + ./*/_instances/*.log + ./*/_instances/*/logs/*.log + ./*/*/_instances/*/logs/*.log + ./*/*/_instances/*.log SignRelease: needs: [BuilderDebRelease] - runs-on: [ self-hosted, on-demand, type-cpx41, image-x86-app-docker-ce] + runs-on: [self-hosted, on-demand, type-cpx41, image-x86-app-docker-ce] steps: - name: Set envs run: | @@ -1070,16 +1152,17 @@ jobs: - FunctionalStatelessTestRelease - FunctionalStatelessTestAarch64 - FunctionalStatefulTestRelease - # - FunctionalStatefulTestAarch64 - # - IntegrationTestsRelease0 - # - IntegrationTestsRelease1 + - FunctionalStatefulTestAarch64 + - IntegrationTestsRelease0 + - IntegrationTestsRelease1 - CompatibilityCheck -# - regression_common -# - benchmark -# - ldap -# - parquet -# - s3 -# - tiered_storage_s3 + - RegressionCommonAmd64 + - RegressionCommonAarch64 + - RegressionBenchmark + - RegressionLDAP + - RegressionParquetAmd64 + - RegressionS3 + - RegressionTieredStorageS3 - SignRelease runs-on: [self-hosted, style-checker, on-demand, type-cpx31, image-x86-app-docker-ce] steps: From 029fea1ddb657a415355bf975216d2838420ae43 Mon Sep 17 00:00:00 2001 From: MyroTk <44327070+MyroTk@users.noreply.github.com> Date: Thu, 31 Aug 2023 07:19:19 -0700 Subject: [PATCH 44/72] Update release_branches.yml --- .github/workflows/release_branches.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/release_branches.yml b/.github/workflows/release_branches.yml index 492941c281ad..7200f4716549 100644 --- a/.github/workflows/release_branches.yml +++ b/.github/workflows/release_branches.yml @@ -356,7 +356,7 @@ jobs: ############################################################################################## FunctionalStatelessTestRelease: needs: [BuilderDebRelease] - runs-on: [self-hosted, func-tester, on-demand, type-cpx51, image-x86-snapshot-docker_ipv6_x86] + runs-on: [self-hosted, altinity-aarch64, func-tester-aarch64] steps: - name: Set envs run: | From b473bf18d17909bded3f0a8eda3dd3c90ab37f50 Mon Sep 17 00:00:00 2001 From: Arthur Passos Date: Fri, 16 Dec 2022 12:06:01 -0300 Subject: [PATCH 45/72] first test --- .github/workflows/release_branches.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/release_branches.yml b/.github/workflows/release_branches.yml index 7200f4716549..e8282c676935 100644 --- a/.github/workflows/release_branches.yml +++ b/.github/workflows/release_branches.yml @@ -1174,3 +1174,4 @@ jobs: run: | cd "$GITHUB_WORKSPACE/tests/ci" python3 finish_check.py + From b1c2791218fa286ba2bfcd73bfff8ddf81e70d38 Mon Sep 17 00:00:00 2001 From: Arthur Passos Date: Fri, 16 Dec 2022 12:46:57 -0300 Subject: [PATCH 46/72] cancel dhpa64 for now From aa0148c746cffe644a07a1e03d1a9772ce91f7a4 Mon Sep 17 00:00:00 2001 From: Arthur Passos Date: Mon, 19 Dec 2022 08:26:38 -0300 Subject: [PATCH 47/72] a --- .github/workflows/release_branches.yml | 46 ++++++++++++++++++++++++++ 1 file changed, 46 insertions(+) diff --git a/.github/workflows/release_branches.yml b/.github/workflows/release_branches.yml index e8282c676935..676d3e58c52b 100644 --- a/.github/workflows/release_branches.yml +++ b/.github/workflows/release_branches.yml @@ -176,6 +176,7 @@ jobs: # shellcheck disable=SC2046 docker rm -f $(docker ps -a -q) ||: sudo rm -fr "$TEMP_PATH" "$CACHES_PATH" +<<<<<<< HEAD BuilderDebAarch64: needs: [DockerHubPush] runs-on: [self-hosted, builder, on-demand, type-cax41, image-arm-app-docker-ce] @@ -222,6 +223,51 @@ jobs: # shellcheck disable=SC2046 docker rm -f $(docker ps -a -q) ||: sudo rm -fr "$TEMP_PATH" "$CACHES_PATH" +======= + # BuilderDebAarch64: + # needs: [DockerHubPush] + # runs-on: [self-hosted, builder] + # steps: + # - name: Set envs + # run: | + # cat >> "$GITHUB_ENV" << 'EOF' + # TEMP_PATH=${{runner.temp}}/build_check + # IMAGES_PATH=${{runner.temp}}/images_path + # REPO_COPY=${{runner.temp}}/build_check/ClickHouse + # CACHES_PATH=${{runner.temp}}/../ccaches + # BUILD_NAME=package_aarch64 + # EOF + # - name: Download changed images + # uses: actions/download-artifact@v2 + # with: + # name: changed_images + # path: ${{ runner.temp }}/images_path + # - name: Check out repository code + # uses: ClickHouse/checkout@v1 + # with: + # fetch-depth: 0 # otherwise we will have no info about contributors + # - name: Build + # run: | + # git -C "$GITHUB_WORKSPACE" submodule sync + # git -C "$GITHUB_WORKSPACE" submodule update --depth=1 --recursive --init --jobs=10 + # sudo rm -fr "$TEMP_PATH" + # mkdir -p "$TEMP_PATH" + # cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" + # cd "$REPO_COPY/tests/ci" && python3 build_check.py "$BUILD_NAME" + # - name: Upload build URLs to artifacts + # uses: actions/upload-artifact@v2 + # with: + # name: ${{ env.BUILD_URLS }} + # path: ${{ runner.temp }}/build_check/${{ env.BUILD_URLS }}.json + # - name: Cleanup + # if: always() + # run: | + # # shellcheck disable=SC2046 + # docker kill $(docker ps -q) ||: + # # shellcheck disable=SC2046 + # docker rm -f $(docker ps -a -q) ||: + # sudo rm -fr "$TEMP_PATH" "$CACHES_PATH" +>>>>>>> ac5f1a0f318 (a) ############################################################################################ ##################################### Docker images ####################################### ############################################################################################ From 1a5a8f617c0c63c9872addaf9db4edd0f75e0243 Mon Sep 17 00:00:00 2001 From: Arthur Passos Date: Mon, 19 Dec 2022 10:49:51 -0300 Subject: [PATCH 48/72] a --- .github/workflows/release_branches.yml | 46 -------------------------- 1 file changed, 46 deletions(-) diff --git a/.github/workflows/release_branches.yml b/.github/workflows/release_branches.yml index 676d3e58c52b..e8282c676935 100644 --- a/.github/workflows/release_branches.yml +++ b/.github/workflows/release_branches.yml @@ -176,7 +176,6 @@ jobs: # shellcheck disable=SC2046 docker rm -f $(docker ps -a -q) ||: sudo rm -fr "$TEMP_PATH" "$CACHES_PATH" -<<<<<<< HEAD BuilderDebAarch64: needs: [DockerHubPush] runs-on: [self-hosted, builder, on-demand, type-cax41, image-arm-app-docker-ce] @@ -223,51 +222,6 @@ jobs: # shellcheck disable=SC2046 docker rm -f $(docker ps -a -q) ||: sudo rm -fr "$TEMP_PATH" "$CACHES_PATH" -======= - # BuilderDebAarch64: - # needs: [DockerHubPush] - # runs-on: [self-hosted, builder] - # steps: - # - name: Set envs - # run: | - # cat >> "$GITHUB_ENV" << 'EOF' - # TEMP_PATH=${{runner.temp}}/build_check - # IMAGES_PATH=${{runner.temp}}/images_path - # REPO_COPY=${{runner.temp}}/build_check/ClickHouse - # CACHES_PATH=${{runner.temp}}/../ccaches - # BUILD_NAME=package_aarch64 - # EOF - # - name: Download changed images - # uses: actions/download-artifact@v2 - # with: - # name: changed_images - # path: ${{ runner.temp }}/images_path - # - name: Check out repository code - # uses: ClickHouse/checkout@v1 - # with: - # fetch-depth: 0 # otherwise we will have no info about contributors - # - name: Build - # run: | - # git -C "$GITHUB_WORKSPACE" submodule sync - # git -C "$GITHUB_WORKSPACE" submodule update --depth=1 --recursive --init --jobs=10 - # sudo rm -fr "$TEMP_PATH" - # mkdir -p "$TEMP_PATH" - # cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" - # cd "$REPO_COPY/tests/ci" && python3 build_check.py "$BUILD_NAME" - # - name: Upload build URLs to artifacts - # uses: actions/upload-artifact@v2 - # with: - # name: ${{ env.BUILD_URLS }} - # path: ${{ runner.temp }}/build_check/${{ env.BUILD_URLS }}.json - # - name: Cleanup - # if: always() - # run: | - # # shellcheck disable=SC2046 - # docker kill $(docker ps -q) ||: - # # shellcheck disable=SC2046 - # docker rm -f $(docker ps -a -q) ||: - # sudo rm -fr "$TEMP_PATH" "$CACHES_PATH" ->>>>>>> ac5f1a0f318 (a) ############################################################################################ ##################################### Docker images ####################################### ############################################################################################ From c133a143341f4944e2fa14d1e3b1dcaf1ea784d3 Mon Sep 17 00:00:00 2001 From: Arthur Passos Date: Mon, 19 Dec 2022 10:57:47 -0300 Subject: [PATCH 49/72] a --- .github/workflows/release_branches.yml | 3 +++ tests/ci/sign_release.py | 23 ++++++----------------- 2 files changed, 9 insertions(+), 17 deletions(-) diff --git a/.github/workflows/release_branches.yml b/.github/workflows/release_branches.yml index e8282c676935..32cdcc1549cb 100644 --- a/.github/workflows/release_branches.yml +++ b/.github/workflows/release_branches.yml @@ -1116,10 +1116,13 @@ jobs: sudo rm -fr "$GITHUB_WORKSPACE" && mkdir "$GITHUB_WORKSPACE" - name: Check out repository code uses: actions/checkout@v2 +<<<<<<< HEAD - name: Download json reports uses: actions/download-artifact@v2 with: path: ${{ env.REPORTS_PATH }} +======= +>>>>>>> e6f24eca112 (a) - name: Sign release env: GPG_BINARY_SIGNING_KEY: ${{ secrets.GPG_BINARY_SIGNING_KEY }} diff --git a/tests/ci/sign_release.py b/tests/ci/sign_release.py index 2c44e0a928f2..ad1ecee6a84a 100644 --- a/tests/ci/sign_release.py +++ b/tests/ci/sign_release.py @@ -72,23 +72,12 @@ def main(): s3_helper.upload_build_file_to_s3(signed_file_path, s3_path) print(f'Uploaded file {signed_file_path} to {s3_path}') - # Signed hashes are: - # clickhouse-client_22.3.15.2.altinitystable_amd64.deb.sha512.gpg clickhouse-keeper_22.3.15.2.altinitystable_x86_64.apk.sha512.gpg - # clickhouse-client-22.3.15.2.altinitystable-amd64.tgz.sha512.gpg clickhouse-keeper-22.3.15.2.altinitystable.x86_64.rpm.sha512.gpg - # clickhouse-client_22.3.15.2.altinitystable_x86_64.apk.sha512.gpg clickhouse-keeper-dbg_22.3.15.2.altinitystable_amd64.deb.sha512.gpg - # clickhouse-client-22.3.15.2.altinitystable.x86_64.rpm.sha512.gpg clickhouse-keeper-dbg-22.3.15.2.altinitystable-amd64.tgz.sha512.gpg - # clickhouse-common-static_22.3.15.2.altinitystable_amd64.deb.sha512.gpg clickhouse-keeper-dbg_22.3.15.2.altinitystable_x86_64.apk.sha512.gpg - # clickhouse-common-static-22.3.15.2.altinitystable-amd64.tgz.sha512.gpg clickhouse-keeper-dbg-22.3.15.2.altinitystable.x86_64.rpm.sha512.gpg - # clickhouse-common-static_22.3.15.2.altinitystable_x86_64.apk.sha512.gpg clickhouse-keeper.sha512.gpg - # clickhouse-common-static-22.3.15.2.altinitystable.x86_64.rpm.sha512.gpg clickhouse-library-bridge.sha512.gpg - # clickhouse-common-static-dbg_22.3.15.2.altinitystable_amd64.deb.sha512.gpg clickhouse-odbc-bridge.sha512.gpg - # clickhouse-common-static-dbg-22.3.15.2.altinitystable-amd64.tgz.sha512.gpg clickhouse-server_22.3.15.2.altinitystable_amd64.deb.sha512.gpg - # clickhouse-common-static-dbg_22.3.15.2.altinitystable_x86_64.apk.sha512.gpg clickhouse-server-22.3.15.2.altinitystable-amd64.tgz.sha512.gpg - # clickhouse-common-static-dbg-22.3.15.2.altinitystable.x86_64.rpm.sha512.gpg clickhouse-server_22.3.15.2.altinitystable_x86_64.apk.sha512.gpg - # clickhouse-keeper_22.3.15.2.altinitystable_amd64.deb.sha512.gpg clickhouse-server-22.3.15.2.altinitystable.x86_64.rpm.sha512.gpg - # clickhouse-keeper-22.3.15.2.altinitystable-amd64.tgz.sha512.gpg clickhouse.sha512.gpg - - sys.exit(0) + for f in os.listdir(packages_path): + full_path = os.path.join(packages_path, f) + subprocess.check_call( + f"dpkg -x {full_path} {packages_path} && rm {full_path}", shell=True + ) + print(f"aaa: {full_path}") if __name__ == "__main__": main() From d5d5b8e68660538e91941cff64c218d986017eb0 Mon Sep 17 00:00:00 2001 From: Arthur Passos Date: Tue, 3 Jan 2023 08:12:44 -0300 Subject: [PATCH 50/72] a --- .github/workflows/release_branches.yml | 3 --- 1 file changed, 3 deletions(-) diff --git a/.github/workflows/release_branches.yml b/.github/workflows/release_branches.yml index 32cdcc1549cb..e8282c676935 100644 --- a/.github/workflows/release_branches.yml +++ b/.github/workflows/release_branches.yml @@ -1116,13 +1116,10 @@ jobs: sudo rm -fr "$GITHUB_WORKSPACE" && mkdir "$GITHUB_WORKSPACE" - name: Check out repository code uses: actions/checkout@v2 -<<<<<<< HEAD - name: Download json reports uses: actions/download-artifact@v2 with: path: ${{ env.REPORTS_PATH }} -======= ->>>>>>> e6f24eca112 (a) - name: Sign release env: GPG_BINARY_SIGNING_KEY: ${{ secrets.GPG_BINARY_SIGNING_KEY }} From d23632b0c0cf31a9647b99c666503dff1052a466 Mon Sep 17 00:00:00 2001 From: Arthur Passos Date: Tue, 3 Jan 2023 10:29:42 -0300 Subject: [PATCH 51/72] a From 93bd60c7740402356421503b9661979413d585ae Mon Sep 17 00:00:00 2001 From: Arthur Passos Date: Wed, 4 Jan 2023 17:32:58 -0300 Subject: [PATCH 52/72] a --- tests/ci/sign_release.py | 24 ++++++++++++++++++------ 1 file changed, 18 insertions(+), 6 deletions(-) diff --git a/tests/ci/sign_release.py b/tests/ci/sign_release.py index ad1ecee6a84a..531b9b8a74be 100644 --- a/tests/ci/sign_release.py +++ b/tests/ci/sign_release.py @@ -72,12 +72,24 @@ def main(): s3_helper.upload_build_file_to_s3(signed_file_path, s3_path) print(f'Uploaded file {signed_file_path} to {s3_path}') - for f in os.listdir(packages_path): - full_path = os.path.join(packages_path, f) - subprocess.check_call( - f"dpkg -x {full_path} {packages_path} && rm {full_path}", shell=True - ) - print(f"aaa: {full_path}") + # Signed hashes are: + # clickhouse-client_22.3.15.2.altinitystable_amd64.deb.sha512.gpg clickhouse-keeper_22.3.15.2.altinitystable_x86_64.apk.sha512.gpg + # clickhouse-client-22.3.15.2.altinitystable-amd64.tgz.sha512.gpg clickhouse-keeper-22.3.15.2.altinitystable.x86_64.rpm.sha512.gpg + # clickhouse-client_22.3.15.2.altinitystable_x86_64.apk.sha512.gpg clickhouse-keeper-dbg_22.3.15.2.altinitystable_amd64.deb.sha512.gpg + # clickhouse-client-22.3.15.2.altinitystable.x86_64.rpm.sha512.gpg clickhouse-keeper-dbg-22.3.15.2.altinitystable-amd64.tgz.sha512.gpg + # clickhouse-common-static_22.3.15.2.altinitystable_amd64.deb.sha512.gpg clickhouse-keeper-dbg_22.3.15.2.altinitystable_x86_64.apk.sha512.gpg + # clickhouse-common-static-22.3.15.2.altinitystable-amd64.tgz.sha512.gpg clickhouse-keeper-dbg-22.3.15.2.altinitystable.x86_64.rpm.sha512.gpg + # clickhouse-common-static_22.3.15.2.altinitystable_x86_64.apk.sha512.gpg clickhouse-keeper.sha512.gpg + # clickhouse-common-static-22.3.15.2.altinitystable.x86_64.rpm.sha512.gpg clickhouse-library-bridge.sha512.gpg + # clickhouse-common-static-dbg_22.3.15.2.altinitystable_amd64.deb.sha512.gpg clickhouse-odbc-bridge.sha512.gpg + # clickhouse-common-static-dbg-22.3.15.2.altinitystable-amd64.tgz.sha512.gpg clickhouse-server_22.3.15.2.altinitystable_amd64.deb.sha512.gpg + # clickhouse-common-static-dbg_22.3.15.2.altinitystable_x86_64.apk.sha512.gpg clickhouse-server-22.3.15.2.altinitystable-amd64.tgz.sha512.gpg + # clickhouse-common-static-dbg-22.3.15.2.altinitystable.x86_64.rpm.sha512.gpg clickhouse-server_22.3.15.2.altinitystable_x86_64.apk.sha512.gpg + # clickhouse-keeper_22.3.15.2.altinitystable_amd64.deb.sha512.gpg clickhouse-server-22.3.15.2.altinitystable.x86_64.rpm.sha512.gpg + # clickhouse-keeper-22.3.15.2.altinitystable-amd64.tgz.sha512.gpg clickhouse.sha512.gpg + + sys.exit(0) if __name__ == "__main__": main() + From a4a6cc3a4fdff573f126c9152f5242200e4c44c3 Mon Sep 17 00:00:00 2001 From: Arthur Passos Date: Fri, 6 Jan 2023 15:32:45 -0300 Subject: [PATCH 53/72] a --- aform | 1 + 1 file changed, 1 insertion(+) create mode 100644 aform diff --git a/aform b/aform new file mode 100644 index 000000000000..5f2ec319ac09 --- /dev/null +++ b/aform @@ -0,0 +1 @@ +skodn asmdap skldma From 288c691e1f2d6e13f7e2c1109a4dd270d13078f9 Mon Sep 17 00:00:00 2001 From: Arthur Passos Date: Fri, 6 Jan 2023 19:50:41 -0300 Subject: [PATCH 54/72] working as epxected, just a cleanup --- .github/workflows/release_branches.yml | 300 +++++++++++++++++++++++++ aform | 1 - 2 files changed, 300 insertions(+), 1 deletion(-) delete mode 100644 aform diff --git a/.github/workflows/release_branches.yml b/.github/workflows/release_branches.yml index e8282c676935..d9206d243c3f 100644 --- a/.github/workflows/release_branches.yml +++ b/.github/workflows/release_branches.yml @@ -1101,6 +1101,296 @@ jobs: ./*/*/_instances/*/logs/*.log ./*/*/_instances/*.log +<<<<<<< HEAD +======= + ############################################################################################ + ##################################### Docker images ####################################### + ############################################################################################ + DockerServerImages: + needs: + - BuilderDebRelease + # - BuilderDebAarch64 - currently we do not build aarch images + runs-on: [self-hosted, style-checker] + steps: + - name: Clear repository + run: | + sudo rm -fr "$GITHUB_WORKSPACE" && mkdir "$GITHUB_WORKSPACE" + - name: Check out repository code + uses: actions/checkout@v2 + with: + fetch-depth: 0 # It MUST BE THE SAME for all dependencies and the job itself + - name: Check docker clickhouse/clickhouse-server building + run: | + cd "$GITHUB_WORKSPACE/tests/ci" + python3 docker_server.py --release-type head --no-push + python3 docker_server.py --release-type head --no-push --no-ubuntu \ + --image-repo clickhouse/clickhouse-keeper --image-path docker/keeper + - name: Cleanup + if: always() + run: | + # shellcheck disable=SC2046 + docker kill $(docker ps -q) ||: + # shellcheck disable=SC2046 + docker rm -f $(docker ps -a -q) ||: + sudo rm -fr "$TEMP_PATH" + ############################################################################################ + ##################################### BUILD REPORTER ####################################### + ############################################################################################ + BuilderReport: + needs: + - BuilderDebRelease + runs-on: [self-hosted, style-checker] + if: ${{ success() || failure() }} + steps: + - name: Set envs + run: | + cat >> "$GITHUB_ENV" << 'EOF' + CHECK_NAME=ClickHouse build check (actions) + REPORTS_PATH=${{runner.temp}}/reports_dir + REPORTS_PATH=${{runner.temp}}/reports_dir + TEMP_PATH=${{runner.temp}}/report_check + NEEDS_DATA_PATH=${{runner.temp}}/needs.json + EOF + - name: Download json reports + uses: actions/download-artifact@v2 + with: + path: ${{ env.REPORTS_PATH }} + - name: Clear repository + run: | + sudo rm -fr "$GITHUB_WORKSPACE" && mkdir "$GITHUB_WORKSPACE" + - name: Check out repository code + uses: actions/checkout@v2 + - name: Report Builder + run: | + sudo rm -fr "$TEMP_PATH" + mkdir -p "$TEMP_PATH" + cat > "$NEEDS_DATA_PATH" << 'EOF' + ${{ toJSON(needs) }} + EOF + cd "$GITHUB_WORKSPACE/tests/ci" + python3 build_report_check.py "$CHECK_NAME" + - name: Cleanup + if: always() + run: | + # shellcheck disable=SC2046 + docker kill $(docker ps -q) ||: + # shellcheck disable=SC2046 + docker rm -f $(docker ps -a -q) ||: + sudo rm -fr "$TEMP_PATH" + MarkReleaseReady: + needs: + - BuilderDebRelease + # - BuilderDebAarch64 + runs-on: [self-hosted, style-checker] + steps: + - name: Clear repository + run: | + sudo rm -fr "$GITHUB_WORKSPACE" && mkdir "$GITHUB_WORKSPACE" + - name: Check out repository code + uses: actions/checkout@v2 + - name: Mark Commit Release Ready + run: | + cd "$GITHUB_WORKSPACE/tests/ci" + python3 mark_release_ready.py + ############################################################################################## + ########################### FUNCTIONAl STATELESS TESTS ####################################### + ############################################################################################## + FunctionalStatelessTestRelease: + needs: [BuilderDebRelease] + runs-on: [self-hosted, func-tester] + steps: + - name: Set envs + run: | + cat >> "$GITHUB_ENV" << 'EOF' + TEMP_PATH=${{runner.temp}}/stateless_debug + REPORTS_PATH=${{runner.temp}}/reports_dir + CHECK_NAME=Stateless tests (release, actions) + REPO_COPY=${{runner.temp}}/stateless_debug/ClickHouse + KILL_TIMEOUT=10800 + EOF + - name: Download json reports + uses: actions/download-artifact@v2 + with: + path: ${{ env.REPORTS_PATH }} + - name: Clear repository + run: | + sudo rm -fr "$GITHUB_WORKSPACE" && mkdir "$GITHUB_WORKSPACE" + - name: Check out repository code + uses: actions/checkout@v2 + - name: Functional test + run: | + sudo rm -fr "$TEMP_PATH" + mkdir -p "$TEMP_PATH" + cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" + cd "$REPO_COPY/tests/ci" + python3 functional_test_check.py "$CHECK_NAME" "$KILL_TIMEOUT" + - name: Cleanup + if: always() + run: | + # shellcheck disable=SC2046 + docker kill $(docker ps -q) ||: + # shellcheck disable=SC2046 + docker rm -f $(docker ps -a -q) ||: + sudo rm -fr "$TEMP_PATH" + ############################################################################################## + ############################ FUNCTIONAl STATEFUL TESTS ####################################### + ############################################################################################## + FunctionalStatefulTestRelease: + needs: [BuilderDebRelease] + runs-on: [self-hosted, func-tester] + steps: + - name: Set envs + run: | + cat >> "$GITHUB_ENV" << 'EOF' + TEMP_PATH=${{runner.temp}}/stateful_debug + REPORTS_PATH=${{runner.temp}}/reports_dir + CHECK_NAME=Stateful tests (release, actions) + REPO_COPY=${{runner.temp}}/stateful_debug/ClickHouse + KILL_TIMEOUT=3600 + EOF + - name: Download json reports + uses: actions/download-artifact@v2 + with: + path: ${{ env.REPORTS_PATH }} + - name: Clear repository + run: | + sudo rm -fr "$GITHUB_WORKSPACE" && mkdir "$GITHUB_WORKSPACE" + - name: Check out repository code + uses: actions/checkout@v2 + - name: Functional test + run: | + sudo rm -fr "$TEMP_PATH" + mkdir -p "$TEMP_PATH" + cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" + cd "$REPO_COPY/tests/ci" + python3 functional_test_check.py "$CHECK_NAME" "$KILL_TIMEOUT" + - name: Cleanup + if: always() + run: | + # shellcheck disable=SC2046 + docker kill $(docker ps -q) ||: + # shellcheck disable=SC2046 + docker rm -f $(docker ps -a -q) ||: + sudo rm -fr "$TEMP_PATH" + ############################################################################################# + ############################# INTEGRATION TESTS ############################################# + ############################################################################################# + IntegrationTestsRelease0: + needs: [BuilderDebRelease] + runs-on: [self-hosted, stress-tester] + steps: + - name: Set envs + run: | + cat >> "$GITHUB_ENV" << 'EOF' + TEMP_PATH=${{runner.temp}}/integration_tests_release + REPORTS_PATH=${{runner.temp}}/reports_dir + CHECK_NAME=Integration tests (release, actions) + REPO_COPY=${{runner.temp}}/integration_tests_release/ClickHouse + RUN_BY_HASH_NUM=0 + RUN_BY_HASH_TOTAL=2 + EOF + - name: Download json reports + uses: actions/download-artifact@v2 + with: + path: ${{ env.REPORTS_PATH }} + - name: Clear repository + run: | + sudo rm -fr "$GITHUB_WORKSPACE" && mkdir "$GITHUB_WORKSPACE" + - name: Check out repository code + uses: actions/checkout@v2 + - name: Integration test + run: | + sudo rm -fr "$TEMP_PATH" + mkdir -p "$TEMP_PATH" + cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" + cd "$REPO_COPY/tests/ci" + python3 integration_test_check.py "$CHECK_NAME" + - name: Cleanup + if: always() + run: | + # shellcheck disable=SC2046 + docker kill $(docker ps -q) ||: + # shellcheck disable=SC2046 + docker rm -f $(docker ps -a -q) ||: + sudo rm -fr "$TEMP_PATH" + IntegrationTestsRelease1: + needs: [BuilderDebRelease] + runs-on: [self-hosted, stress-tester] + steps: + - name: Set envs + run: | + cat >> "$GITHUB_ENV" << 'EOF' + TEMP_PATH=${{runner.temp}}/integration_tests_release + REPORTS_PATH=${{runner.temp}}/reports_dir + CHECK_NAME=Integration tests (release, actions) + REPO_COPY=${{runner.temp}}/integration_tests_release/ClickHouse + RUN_BY_HASH_NUM=1 + RUN_BY_HASH_TOTAL=2 + EOF + - name: Download json reports + uses: actions/download-artifact@v2 + with: + path: ${{ env.REPORTS_PATH }} + - name: Clear repository + run: | + sudo rm -fr "$GITHUB_WORKSPACE" && mkdir "$GITHUB_WORKSPACE" + - name: Check out repository code + uses: actions/checkout@v2 + - name: Integration test + run: | + sudo rm -fr "$TEMP_PATH" + mkdir -p "$TEMP_PATH" + cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" + cd "$REPO_COPY/tests/ci" + python3 integration_test_check.py "$CHECK_NAME" + - name: Cleanup + if: always() + run: | + # shellcheck disable=SC2046 + docker kill $(docker ps -q) ||: + # shellcheck disable=SC2046 + docker rm -f $(docker ps -a -q) ||: + sudo rm -fr "$TEMP_PATH" + ############################################################################################# + ############################### TESTFLOWS TESTS ############################################# + ############################################################################################# + TestFlowsTestsRelease: + needs: [BuilderDebRelease] + runs-on: [self-hosted, stress-tester] + steps: + - name: Set envs + run: | + cat >> "$GITHUB_ENV" << 'EOF' + TEMP_PATH=${{runner.temp}}/testflows_tests_release + REPORTS_PATH=${{runner.temp}}/reports_dir + CHECK_NAME=TestFlows tests (release, actions) + REPO_COPY=${{runner.temp}}/testflows_tests_release/ClickHouse + EOF + - name: Download json reports + uses: actions/download-artifact@v2 + with: + path: ${{ env.REPORTS_PATH }} + - name: Clear repository + run: | + sudo rm -fr "$GITHUB_WORKSPACE" && mkdir "$GITHUB_WORKSPACE" + - name: Check out repository code + uses: actions/checkout@v2 + - name: TestFlows test + run: | + sudo rm -fr "$TEMP_PATH" + mkdir -p "$TEMP_PATH" + cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" + cd "$REPO_COPY/tests/ci" + python3 testflows_test_check.py "$CHECK_NAME" + - name: Cleanup + if: always() + run: | + # shellcheck disable=SC2046 + docker kill $(docker ps -q) ||: + # shellcheck disable=SC2046 + docker rm -f $(docker ps -a -q) ||: + sudo rm -fr "$TEMP_PATH" +>>>>>>> 983e7addba (working as epxected, just a cleanup) SignRelease: needs: [BuilderDebRelease] runs-on: [self-hosted, on-demand, type-cpx41, image-x86-app-docker-ce] @@ -1147,6 +1437,7 @@ jobs: - DockerHubPush - DockerServerImages - BuilderReport +<<<<<<< HEAD # - BuilderSpecialReport - MarkReleaseReady - FunctionalStatelessTestRelease @@ -1155,6 +1446,15 @@ jobs: - FunctionalStatefulTestAarch64 - IntegrationTestsRelease0 - IntegrationTestsRelease1 +======= + - MarkReleaseReady + - FunctionalStatelessTestRelease + - FunctionalStatefulTestRelease + - IntegrationTestsRelease0 + - IntegrationTestsRelease1 + - TestFlowsTestsRelease + - SignRelease +>>>>>>> 983e7addba (working as epxected, just a cleanup) - CompatibilityCheck - RegressionCommonAmd64 - RegressionCommonAarch64 diff --git a/aform b/aform deleted file mode 100644 index 5f2ec319ac09..000000000000 --- a/aform +++ /dev/null @@ -1 +0,0 @@ -skodn asmdap skldma From 33279993b68d98f1fc3485c307c21ed351465236 Mon Sep 17 00:00:00 2001 From: MyroTk Date: Wed, 31 May 2023 09:57:28 -0700 Subject: [PATCH 55/72] merge fix --- .github/workflows/release_branches.yml | 300 ------------------------- 1 file changed, 300 deletions(-) diff --git a/.github/workflows/release_branches.yml b/.github/workflows/release_branches.yml index d9206d243c3f..e8282c676935 100644 --- a/.github/workflows/release_branches.yml +++ b/.github/workflows/release_branches.yml @@ -1101,296 +1101,6 @@ jobs: ./*/*/_instances/*/logs/*.log ./*/*/_instances/*.log -<<<<<<< HEAD -======= - ############################################################################################ - ##################################### Docker images ####################################### - ############################################################################################ - DockerServerImages: - needs: - - BuilderDebRelease - # - BuilderDebAarch64 - currently we do not build aarch images - runs-on: [self-hosted, style-checker] - steps: - - name: Clear repository - run: | - sudo rm -fr "$GITHUB_WORKSPACE" && mkdir "$GITHUB_WORKSPACE" - - name: Check out repository code - uses: actions/checkout@v2 - with: - fetch-depth: 0 # It MUST BE THE SAME for all dependencies and the job itself - - name: Check docker clickhouse/clickhouse-server building - run: | - cd "$GITHUB_WORKSPACE/tests/ci" - python3 docker_server.py --release-type head --no-push - python3 docker_server.py --release-type head --no-push --no-ubuntu \ - --image-repo clickhouse/clickhouse-keeper --image-path docker/keeper - - name: Cleanup - if: always() - run: | - # shellcheck disable=SC2046 - docker kill $(docker ps -q) ||: - # shellcheck disable=SC2046 - docker rm -f $(docker ps -a -q) ||: - sudo rm -fr "$TEMP_PATH" - ############################################################################################ - ##################################### BUILD REPORTER ####################################### - ############################################################################################ - BuilderReport: - needs: - - BuilderDebRelease - runs-on: [self-hosted, style-checker] - if: ${{ success() || failure() }} - steps: - - name: Set envs - run: | - cat >> "$GITHUB_ENV" << 'EOF' - CHECK_NAME=ClickHouse build check (actions) - REPORTS_PATH=${{runner.temp}}/reports_dir - REPORTS_PATH=${{runner.temp}}/reports_dir - TEMP_PATH=${{runner.temp}}/report_check - NEEDS_DATA_PATH=${{runner.temp}}/needs.json - EOF - - name: Download json reports - uses: actions/download-artifact@v2 - with: - path: ${{ env.REPORTS_PATH }} - - name: Clear repository - run: | - sudo rm -fr "$GITHUB_WORKSPACE" && mkdir "$GITHUB_WORKSPACE" - - name: Check out repository code - uses: actions/checkout@v2 - - name: Report Builder - run: | - sudo rm -fr "$TEMP_PATH" - mkdir -p "$TEMP_PATH" - cat > "$NEEDS_DATA_PATH" << 'EOF' - ${{ toJSON(needs) }} - EOF - cd "$GITHUB_WORKSPACE/tests/ci" - python3 build_report_check.py "$CHECK_NAME" - - name: Cleanup - if: always() - run: | - # shellcheck disable=SC2046 - docker kill $(docker ps -q) ||: - # shellcheck disable=SC2046 - docker rm -f $(docker ps -a -q) ||: - sudo rm -fr "$TEMP_PATH" - MarkReleaseReady: - needs: - - BuilderDebRelease - # - BuilderDebAarch64 - runs-on: [self-hosted, style-checker] - steps: - - name: Clear repository - run: | - sudo rm -fr "$GITHUB_WORKSPACE" && mkdir "$GITHUB_WORKSPACE" - - name: Check out repository code - uses: actions/checkout@v2 - - name: Mark Commit Release Ready - run: | - cd "$GITHUB_WORKSPACE/tests/ci" - python3 mark_release_ready.py - ############################################################################################## - ########################### FUNCTIONAl STATELESS TESTS ####################################### - ############################################################################################## - FunctionalStatelessTestRelease: - needs: [BuilderDebRelease] - runs-on: [self-hosted, func-tester] - steps: - - name: Set envs - run: | - cat >> "$GITHUB_ENV" << 'EOF' - TEMP_PATH=${{runner.temp}}/stateless_debug - REPORTS_PATH=${{runner.temp}}/reports_dir - CHECK_NAME=Stateless tests (release, actions) - REPO_COPY=${{runner.temp}}/stateless_debug/ClickHouse - KILL_TIMEOUT=10800 - EOF - - name: Download json reports - uses: actions/download-artifact@v2 - with: - path: ${{ env.REPORTS_PATH }} - - name: Clear repository - run: | - sudo rm -fr "$GITHUB_WORKSPACE" && mkdir "$GITHUB_WORKSPACE" - - name: Check out repository code - uses: actions/checkout@v2 - - name: Functional test - run: | - sudo rm -fr "$TEMP_PATH" - mkdir -p "$TEMP_PATH" - cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" - cd "$REPO_COPY/tests/ci" - python3 functional_test_check.py "$CHECK_NAME" "$KILL_TIMEOUT" - - name: Cleanup - if: always() - run: | - # shellcheck disable=SC2046 - docker kill $(docker ps -q) ||: - # shellcheck disable=SC2046 - docker rm -f $(docker ps -a -q) ||: - sudo rm -fr "$TEMP_PATH" - ############################################################################################## - ############################ FUNCTIONAl STATEFUL TESTS ####################################### - ############################################################################################## - FunctionalStatefulTestRelease: - needs: [BuilderDebRelease] - runs-on: [self-hosted, func-tester] - steps: - - name: Set envs - run: | - cat >> "$GITHUB_ENV" << 'EOF' - TEMP_PATH=${{runner.temp}}/stateful_debug - REPORTS_PATH=${{runner.temp}}/reports_dir - CHECK_NAME=Stateful tests (release, actions) - REPO_COPY=${{runner.temp}}/stateful_debug/ClickHouse - KILL_TIMEOUT=3600 - EOF - - name: Download json reports - uses: actions/download-artifact@v2 - with: - path: ${{ env.REPORTS_PATH }} - - name: Clear repository - run: | - sudo rm -fr "$GITHUB_WORKSPACE" && mkdir "$GITHUB_WORKSPACE" - - name: Check out repository code - uses: actions/checkout@v2 - - name: Functional test - run: | - sudo rm -fr "$TEMP_PATH" - mkdir -p "$TEMP_PATH" - cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" - cd "$REPO_COPY/tests/ci" - python3 functional_test_check.py "$CHECK_NAME" "$KILL_TIMEOUT" - - name: Cleanup - if: always() - run: | - # shellcheck disable=SC2046 - docker kill $(docker ps -q) ||: - # shellcheck disable=SC2046 - docker rm -f $(docker ps -a -q) ||: - sudo rm -fr "$TEMP_PATH" - ############################################################################################# - ############################# INTEGRATION TESTS ############################################# - ############################################################################################# - IntegrationTestsRelease0: - needs: [BuilderDebRelease] - runs-on: [self-hosted, stress-tester] - steps: - - name: Set envs - run: | - cat >> "$GITHUB_ENV" << 'EOF' - TEMP_PATH=${{runner.temp}}/integration_tests_release - REPORTS_PATH=${{runner.temp}}/reports_dir - CHECK_NAME=Integration tests (release, actions) - REPO_COPY=${{runner.temp}}/integration_tests_release/ClickHouse - RUN_BY_HASH_NUM=0 - RUN_BY_HASH_TOTAL=2 - EOF - - name: Download json reports - uses: actions/download-artifact@v2 - with: - path: ${{ env.REPORTS_PATH }} - - name: Clear repository - run: | - sudo rm -fr "$GITHUB_WORKSPACE" && mkdir "$GITHUB_WORKSPACE" - - name: Check out repository code - uses: actions/checkout@v2 - - name: Integration test - run: | - sudo rm -fr "$TEMP_PATH" - mkdir -p "$TEMP_PATH" - cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" - cd "$REPO_COPY/tests/ci" - python3 integration_test_check.py "$CHECK_NAME" - - name: Cleanup - if: always() - run: | - # shellcheck disable=SC2046 - docker kill $(docker ps -q) ||: - # shellcheck disable=SC2046 - docker rm -f $(docker ps -a -q) ||: - sudo rm -fr "$TEMP_PATH" - IntegrationTestsRelease1: - needs: [BuilderDebRelease] - runs-on: [self-hosted, stress-tester] - steps: - - name: Set envs - run: | - cat >> "$GITHUB_ENV" << 'EOF' - TEMP_PATH=${{runner.temp}}/integration_tests_release - REPORTS_PATH=${{runner.temp}}/reports_dir - CHECK_NAME=Integration tests (release, actions) - REPO_COPY=${{runner.temp}}/integration_tests_release/ClickHouse - RUN_BY_HASH_NUM=1 - RUN_BY_HASH_TOTAL=2 - EOF - - name: Download json reports - uses: actions/download-artifact@v2 - with: - path: ${{ env.REPORTS_PATH }} - - name: Clear repository - run: | - sudo rm -fr "$GITHUB_WORKSPACE" && mkdir "$GITHUB_WORKSPACE" - - name: Check out repository code - uses: actions/checkout@v2 - - name: Integration test - run: | - sudo rm -fr "$TEMP_PATH" - mkdir -p "$TEMP_PATH" - cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" - cd "$REPO_COPY/tests/ci" - python3 integration_test_check.py "$CHECK_NAME" - - name: Cleanup - if: always() - run: | - # shellcheck disable=SC2046 - docker kill $(docker ps -q) ||: - # shellcheck disable=SC2046 - docker rm -f $(docker ps -a -q) ||: - sudo rm -fr "$TEMP_PATH" - ############################################################################################# - ############################### TESTFLOWS TESTS ############################################# - ############################################################################################# - TestFlowsTestsRelease: - needs: [BuilderDebRelease] - runs-on: [self-hosted, stress-tester] - steps: - - name: Set envs - run: | - cat >> "$GITHUB_ENV" << 'EOF' - TEMP_PATH=${{runner.temp}}/testflows_tests_release - REPORTS_PATH=${{runner.temp}}/reports_dir - CHECK_NAME=TestFlows tests (release, actions) - REPO_COPY=${{runner.temp}}/testflows_tests_release/ClickHouse - EOF - - name: Download json reports - uses: actions/download-artifact@v2 - with: - path: ${{ env.REPORTS_PATH }} - - name: Clear repository - run: | - sudo rm -fr "$GITHUB_WORKSPACE" && mkdir "$GITHUB_WORKSPACE" - - name: Check out repository code - uses: actions/checkout@v2 - - name: TestFlows test - run: | - sudo rm -fr "$TEMP_PATH" - mkdir -p "$TEMP_PATH" - cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" - cd "$REPO_COPY/tests/ci" - python3 testflows_test_check.py "$CHECK_NAME" - - name: Cleanup - if: always() - run: | - # shellcheck disable=SC2046 - docker kill $(docker ps -q) ||: - # shellcheck disable=SC2046 - docker rm -f $(docker ps -a -q) ||: - sudo rm -fr "$TEMP_PATH" ->>>>>>> 983e7addba (working as epxected, just a cleanup) SignRelease: needs: [BuilderDebRelease] runs-on: [self-hosted, on-demand, type-cpx41, image-x86-app-docker-ce] @@ -1437,7 +1147,6 @@ jobs: - DockerHubPush - DockerServerImages - BuilderReport -<<<<<<< HEAD # - BuilderSpecialReport - MarkReleaseReady - FunctionalStatelessTestRelease @@ -1446,15 +1155,6 @@ jobs: - FunctionalStatefulTestAarch64 - IntegrationTestsRelease0 - IntegrationTestsRelease1 -======= - - MarkReleaseReady - - FunctionalStatelessTestRelease - - FunctionalStatefulTestRelease - - IntegrationTestsRelease0 - - IntegrationTestsRelease1 - - TestFlowsTestsRelease - - SignRelease ->>>>>>> 983e7addba (working as epxected, just a cleanup) - CompatibilityCheck - RegressionCommonAmd64 - RegressionCommonAarch64 From c7f6ab465cbdec8f66916bdd5c100fae680720bb Mon Sep 17 00:00:00 2001 From: MyroTk Date: Thu, 31 Aug 2023 08:41:05 -0700 Subject: [PATCH 56/72] removing comment --- .github/workflows/release_branches.yml | 6 ------ 1 file changed, 6 deletions(-) diff --git a/.github/workflows/release_branches.yml b/.github/workflows/release_branches.yml index e8282c676935..4b259e6e961e 100644 --- a/.github/workflows/release_branches.yml +++ b/.github/workflows/release_branches.yml @@ -43,7 +43,6 @@ jobs: with: name: changed_images_aarch64 path: ${{ runner.temp }}/docker_images_check/changed_images_aarch64.json - # temporary DockerHubPush: DockerHubPushAmd64: runs-on: [self-hosted, style-checker, on-demand, type-cpx51, image-x86-app-docker-ce] steps: @@ -55,11 +54,6 @@ jobs: run: | cd "$GITHUB_WORKSPACE/tests/ci" python3 docker_images_check.py --suffix amd64 - # TODO(vnemkov): remove this step if you uncomment DockerHubPushAarch64 and DockerHubPush below. - # The rest of the pipeline expects changed_images.json, which was generated by previous version of DockerHubPush. - # - name: Rename artifact - # run: | - # mv ${{ runner.temp }}/docker_images_check/changed_images_amd64.json ${{ runner.temp }}/docker_images_check/changed_images.json - name: Upload images files to artifacts uses: actions/upload-artifact@v3 with: From dd8e59ddfdca045c3649cf3690fd911d78899a92 Mon Sep 17 00:00:00 2001 From: MyroTk <44327070+MyroTk@users.noreply.github.com> Date: Thu, 31 Aug 2023 08:49:35 -0700 Subject: [PATCH 57/72] Update sign_release.py --- tests/ci/sign_release.py | 1 - 1 file changed, 1 deletion(-) diff --git a/tests/ci/sign_release.py b/tests/ci/sign_release.py index 531b9b8a74be..2c44e0a928f2 100644 --- a/tests/ci/sign_release.py +++ b/tests/ci/sign_release.py @@ -92,4 +92,3 @@ def main(): if __name__ == "__main__": main() - From 0e2f9923d1b00a5fb5febc56f86390c4d6c15128 Mon Sep 17 00:00:00 2001 From: MyroTk <44327070+MyroTk@users.noreply.github.com> Date: Thu, 31 Aug 2023 08:49:51 -0700 Subject: [PATCH 58/72] Update 02381_arrow_dict_to_lc.sh From 92e2404fef7fbbcc956b0fc7283ccc70255bead9 Mon Sep 17 00:00:00 2001 From: MyroTk <44327070+MyroTk@users.noreply.github.com> Date: Thu, 31 Aug 2023 08:50:44 -0700 Subject: [PATCH 59/72] Update 02381_arrow_dict_to_lc.sh From 40aae5698b3c16ea8c1485db46a7a7e7606e91ec Mon Sep 17 00:00:00 2001 From: MyroTk <44327070+MyroTk@users.noreply.github.com> Date: Thu, 31 Aug 2023 15:55:23 -0700 Subject: [PATCH 60/72] Update release_branches.yml --- .github/workflows/release_branches.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/release_branches.yml b/.github/workflows/release_branches.yml index 4b259e6e961e..4d3d5fefe0a1 100644 --- a/.github/workflows/release_branches.yml +++ b/.github/workflows/release_branches.yml @@ -350,7 +350,7 @@ jobs: ############################################################################################## FunctionalStatelessTestRelease: needs: [BuilderDebRelease] - runs-on: [self-hosted, altinity-aarch64, func-tester-aarch64] + runs-on: [self-hosted, func-tester, on-demand, type-cax41, image-arm-snapshot-docker_ipv6_arm] steps: - name: Set envs run: | From 6e0eafec692dbc763e7702a26e379ed08e734956 Mon Sep 17 00:00:00 2001 From: MyroTk <44327070+MyroTk@users.noreply.github.com> Date: Fri, 1 Sep 2023 08:23:54 -0700 Subject: [PATCH 61/72] Update run.sh --- docker/test/stateful/run.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docker/test/stateful/run.sh b/docker/test/stateful/run.sh index 45a4601e0287..c75871383fc7 100755 --- a/docker/test/stateful/run.sh +++ b/docker/test/stateful/run.sh @@ -152,7 +152,7 @@ if [[ -n "$USE_DATABASE_REPLICATED" ]] && [[ "$USE_DATABASE_REPLICATED" -eq 1 ]] sudo clickhouse stop --pid-path /var/run/clickhouse-server2 ||: fi -grep -Fa "Fatal" /var/log/clickhouse-server/clickhouse-server.log ||: +rg -Fa "" /var/log/clickhouse-server/clickhouse-server.log ||: pigz < /var/log/clickhouse-server/clickhouse-server.log > /test_output/clickhouse-server.log.gz ||: # FIXME: remove once only github actions will be left From 032ca64fe94230598da3b57dc037d623f472b756 Mon Sep 17 00:00:00 2001 From: MyroTk <44327070+MyroTk@users.noreply.github.com> Date: Fri, 1 Sep 2023 08:25:23 -0700 Subject: [PATCH 62/72] Update run.sh --- docker/test/stateless/run.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docker/test/stateless/run.sh b/docker/test/stateless/run.sh index cc1a4a03ed20..4cb71d38f8f4 100755 --- a/docker/test/stateless/run.sh +++ b/docker/test/stateless/run.sh @@ -162,7 +162,7 @@ if [[ -n "$USE_DATABASE_REPLICATED" ]] && [[ "$USE_DATABASE_REPLICATED" -eq 1 ]] sudo clickhouse stop --pid-path /var/run/clickhouse-server2 ||: fi -grep -Fa "Fatal" /var/log/clickhouse-server/clickhouse-server.log ||: +rg -Fa "" /var/log/clickhouse-server/clickhouse-server.log ||: pigz < /var/log/clickhouse-server/clickhouse-server.log > /test_output/clickhouse-server.log.gz & # Compress tables. From 55cd2a12c4a805996fb1c39bc11dacc7c62f1fcf Mon Sep 17 00:00:00 2001 From: MyroTk <44327070+MyroTk@users.noreply.github.com> Date: Fri, 1 Sep 2023 12:49:50 -0700 Subject: [PATCH 63/72] Update release_branches.yml --- .github/workflows/release_branches.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/release_branches.yml b/.github/workflows/release_branches.yml index 4d3d5fefe0a1..7ded5e010bd3 100644 --- a/.github/workflows/release_branches.yml +++ b/.github/workflows/release_branches.yml @@ -350,7 +350,7 @@ jobs: ############################################################################################## FunctionalStatelessTestRelease: needs: [BuilderDebRelease] - runs-on: [self-hosted, func-tester, on-demand, type-cax41, image-arm-snapshot-docker_ipv6_arm] + runs-on: [self-hosted, func-tester, on-demand, type-cpx51, image-x86-snapshot-docker_ipv6_x86] steps: - name: Set envs run: | @@ -605,7 +605,7 @@ jobs: ############################################################################################# RegressionStart: ## Not depending on the tests above since they can fail at any given moment. - needs: [BuilderDebRelease] + needs: [BuilderDebRelease, BuilderDebAarch64] runs-on: ubuntu-latest steps: - run: true @@ -616,7 +616,7 @@ jobs: matrix: SUITE: [aes_encryption, aggregate_functions, atomic_insert, base_58, clickhouse_keeper, datetime64_extended_range, disk_level_encryption, dns, example, extended_precision_data_types, kafka, kerberos, lightweight_delete, map_type, part_moves_between_shards, rbac, selects, ssl_server, tiered_storage, window_functions] needs: [RegressionStart] - runs-on: [self-hosted, stress-tester] + runs-on: [self-hosted, regression-tester, on-demand, type-cpx51, image-x86-app-docker-ce] env: AWS_ACCESS_KEY_ID: ${{ secrets.AWS_REPORT_KEY_ID }} AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_REPORT_SECRET_ACCESS_KEY }} From d17d8d57281c8f36bc0e17be07c172187ed4302b Mon Sep 17 00:00:00 2001 From: MyroTk <44327070+MyroTk@users.noreply.github.com> Date: Wed, 6 Sep 2023 09:22:57 -0700 Subject: [PATCH 64/72] Update release_branches.yml --- .github/workflows/release_branches.yml | 6 ------ 1 file changed, 6 deletions(-) diff --git a/.github/workflows/release_branches.yml b/.github/workflows/release_branches.yml index 7ded5e010bd3..6940b256f8be 100644 --- a/.github/workflows/release_branches.yml +++ b/.github/workflows/release_branches.yml @@ -626,7 +626,6 @@ jobs: uses: actions/checkout@v3 with: repository: Altinity/clickhouse-regression - ref: releases - name: Set envs run: | cat >> "$GITHUB_ENV" << 'EOF' @@ -688,7 +687,6 @@ jobs: uses: actions/checkout@v3 with: repository: Altinity/clickhouse-regression - ref: releases - name: Set envs run: | cat >> "$GITHUB_ENV" << 'EOF' @@ -830,7 +828,6 @@ jobs: uses: actions/checkout@v3 with: repository: Altinity/clickhouse-regression - ref: releases - name: Set envs run: | cat >> "$GITHUB_ENV" << 'EOF' @@ -888,7 +885,6 @@ jobs: uses: actions/checkout@v3 with: repository: Altinity/clickhouse-regression - ref: releases - name: Set envs run: | cat >> "$GITHUB_ENV" << 'EOF' @@ -965,7 +961,6 @@ jobs: uses: actions/checkout@v3 with: repository: Altinity/clickhouse-regression - ref: releases - name: Set envs run: | cat >> "$GITHUB_ENV" << 'EOF' @@ -1041,7 +1036,6 @@ jobs: uses: actions/checkout@v3 with: repository: Altinity/clickhouse-regression - ref: releases - name: Set envs run: | cat >> "$GITHUB_ENV" << 'EOF' From fc964cc4aefbd44181f8380d84654517b6b723ab Mon Sep 17 00:00:00 2001 From: Ilya Golshtein Date: Thu, 14 Sep 2023 13:12:29 +0000 Subject: [PATCH 65/72] arm_22.8.15: no global stop merges --- .../02286_drop_filesystem_cache.queries | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/queries/0_stateless/filesystem_cache_queries/02286_drop_filesystem_cache.queries b/tests/queries/0_stateless/filesystem_cache_queries/02286_drop_filesystem_cache.queries index 96774db32edc..dc069c94a533 100644 --- a/tests/queries/0_stateless/filesystem_cache_queries/02286_drop_filesystem_cache.queries +++ b/tests/queries/0_stateless/filesystem_cache_queries/02286_drop_filesystem_cache.queries @@ -9,7 +9,7 @@ Engine=MergeTree() ORDER BY key SETTINGS storage_policy='_storagePolicy', min_bytes_for_wide_part = 10485760; -SYSTEM STOP MERGES; +SYSTEM STOP MERGES test; SYSTEM DROP FILESYSTEM CACHE; SELECT count() FROM system.filesystem_cache; From a028c29d351a145a57721e998c88830fcdc03a15 Mon Sep 17 00:00:00 2001 From: Ilya Golshtein Date: Thu, 14 Sep 2023 22:56:40 +0000 Subject: [PATCH 66/72] arm_22.8.15: other IPv6 addr, explicit 8.8.8.8 dns --- .github/workflows/release_branches.yml | 39 +++++++++++++++---- tests/ci/functional_test_check.py | 1 + .../02286_drop_filesystem_cache.reference | 5 +-- 3 files changed, 34 insertions(+), 11 deletions(-) diff --git a/.github/workflows/release_branches.yml b/.github/workflows/release_branches.yml index 6940b256f8be..52951c35f42b 100644 --- a/.github/workflows/release_branches.yml +++ b/.github/workflows/release_branches.yml @@ -352,6 +352,18 @@ jobs: needs: [BuilderDebRelease] runs-on: [self-hosted, func-tester, on-demand, type-cpx51, image-x86-snapshot-docker_ipv6_x86] steps: + - name: Setup + run: | + sudo touch /etc/docker/daemon.json + sudo chown ubuntu:ubuntu /etc/docker/daemon.json + sudo cat < /etc/docker/daemon.json + { + "ipv6": true, + "fixed-cidr-v6": "2001:3984:3989::/64" + } + EOT + sudo chown root:root /etc/docker/daemon.json + sudo systemctl restart docker - name: Set envs run: | cat >> "$GITHUB_ENV" << 'EOF' @@ -397,7 +409,7 @@ jobs: sudo cat < /etc/docker/daemon.json { "ipv6": true, - "fixed-cidr-v6": "2001:db8:1::/64" + "fixed-cidr-v6": "2001:3984:3989::/64" } EOT sudo chown root:root /etc/docker/daemon.json @@ -443,6 +455,18 @@ jobs: needs: [BuilderDebRelease] runs-on: [self-hosted, func-tester, on-demand, type-cpx51, image-x86-snapshot-docker_ipv6_x86] steps: + - name: Setup + run: | + sudo touch /etc/docker/daemon.json + sudo chown ubuntu:ubuntu /etc/docker/daemon.json + sudo cat < /etc/docker/daemon.json + { + "ipv6": true, + "fixed-cidr-v6": "2001:3984:3989::/64" + } + EOT + sudo chown root:root /etc/docker/daemon.json + sudo systemctl restart docker - name: Set envs run: | cat >> "$GITHUB_ENV" << 'EOF' @@ -486,7 +510,7 @@ jobs: sudo cat < /etc/docker/daemon.json { "ipv6": true, - "fixed-cidr-v6": "2001:db8:1::/64" + "fixed-cidr-v6": "2001:3984:3989::/64" } EOT sudo chown root:root /etc/docker/daemon.json @@ -609,7 +633,7 @@ jobs: runs-on: ubuntu-latest steps: - run: true - + RegressionCommonAmd64: strategy: fail-fast: false @@ -731,7 +755,7 @@ jobs: ./*/_instances/*/logs/*.log ./*/*/_instances/*/logs/*.log ./*/*/_instances/*.log - + RegressionBenchmark: strategy: fail-fast: false @@ -806,7 +830,7 @@ jobs: ./*/_instances/*/logs/*.log ./*/*/_instances/*/logs/*.log ./*/*/_instances/*.log - + RegressionLDAP: strategy: fail-fast: false @@ -872,7 +896,7 @@ jobs: ./*/_instances/*/logs/*.log ./*/*/_instances/*/logs/*.log ./*/*/_instances/*.log - + RegressionParquetAmd64: needs: [RegressionStart] runs-on: [self-hosted, stress-tester] @@ -939,7 +963,7 @@ jobs: ./*/_instances/*/logs/*.log ./*/*/_instances/*/logs/*.log ./*/*/_instances/*.log - + RegressionS3: strategy: fail-fast: false @@ -1162,4 +1186,3 @@ jobs: run: | cd "$GITHUB_WORKSPACE/tests/ci" python3 finish_check.py - diff --git a/tests/ci/functional_test_check.py b/tests/ci/functional_test_check.py index bb0b17a59aa0..019087e27fa5 100644 --- a/tests/ci/functional_test_check.py +++ b/tests/ci/functional_test_check.py @@ -101,6 +101,7 @@ def get_run_command( return ( f"docker run --volume={builds_path}:/package_folder " + f"--dns=8.8.8.8 " f"--volume={repo_tests_path}:/usr/share/clickhouse-test " f"--volume={result_path}:/test_output --volume={server_log_path}:/var/log/clickhouse-server " f"--cap-add=SYS_PTRACE {env_str} {additional_options_str} {image}" diff --git a/tests/queries/0_stateless/02286_drop_filesystem_cache.reference b/tests/queries/0_stateless/02286_drop_filesystem_cache.reference index b37f87afc28e..4d9218dcaaf5 100644 --- a/tests/queries/0_stateless/02286_drop_filesystem_cache.reference +++ b/tests/queries/0_stateless/02286_drop_filesystem_cache.reference @@ -7,7 +7,7 @@ CREATE TABLE test (key UInt32, value String) Engine=MergeTree() ORDER BY key SETTINGS storage_policy='s3_cache', min_bytes_for_wide_part = 10485760; -SYSTEM STOP MERGES; +SYSTEM STOP MERGES test; SYSTEM DROP FILESYSTEM CACHE; SELECT count() FROM system.filesystem_cache; 0 @@ -76,7 +76,7 @@ CREATE TABLE test (key UInt32, value String) Engine=MergeTree() ORDER BY key SETTINGS storage_policy='local_cache', min_bytes_for_wide_part = 10485760; -SYSTEM STOP MERGES; +SYSTEM STOP MERGES test; SYSTEM DROP FILESYSTEM CACHE; SELECT count() FROM system.filesystem_cache; 0 @@ -135,4 +135,3 @@ SELECT count() FROM system.filesystem_cache; SYSTEM DROP FILESYSTEM CACHE 'local_cache_2/'; SELECT count() FROM system.filesystem_cache; 0 - From f25117d41254b8d1d21907f4940b435859dab6b3 Mon Sep 17 00:00:00 2001 From: Ilya Golshtein Date: Fri, 15 Sep 2023 08:29:24 +0000 Subject: [PATCH 67/72] arm_22.8.15: blank trailing line is required --- tests/queries/0_stateless/02286_drop_filesystem_cache.reference | 1 + 1 file changed, 1 insertion(+) diff --git a/tests/queries/0_stateless/02286_drop_filesystem_cache.reference b/tests/queries/0_stateless/02286_drop_filesystem_cache.reference index 4d9218dcaaf5..2f7b8820e9c3 100644 --- a/tests/queries/0_stateless/02286_drop_filesystem_cache.reference +++ b/tests/queries/0_stateless/02286_drop_filesystem_cache.reference @@ -135,3 +135,4 @@ SELECT count() FROM system.filesystem_cache; SYSTEM DROP FILESYSTEM CACHE 'local_cache_2/'; SELECT count() FROM system.filesystem_cache; 0 + From 69fa532f9a5d9cc135e834100b3499d8fab30a17 Mon Sep 17 00:00:00 2001 From: MyroTk <44327070+MyroTk@users.noreply.github.com> Date: Fri, 15 Sep 2023 11:18:56 -0700 Subject: [PATCH 68/72] Update release_branches.yml --- .github/workflows/release_branches.yml | 334 ++++++++++++++++++++++--- 1 file changed, 294 insertions(+), 40 deletions(-) diff --git a/.github/workflows/release_branches.yml b/.github/workflows/release_branches.yml index 52951c35f42b..dd89642e964b 100644 --- a/.github/workflows/release_branches.yml +++ b/.github/workflows/release_branches.yml @@ -674,7 +674,7 @@ jobs: --collect-service-logs --output classic --parallel 1 - --attr project="$GITHUB_REPOSITORY" project.id="$GITHUB_REPOSITORY_ID" package="${{ env.clickhouse_binary_path }}" version="${{ env.version }}" user.name="$GITHUB_ACTOR" repository="https://github.com/Altinity/clickhouse-regression" commit.hash="$(git rev-parse HEAD)" job.id="$GITHUB_RUN_ID" job.url="$GITHUB_SERVER_URL/$GITHUB_REPOSITORY/actions/runs/$GITHUB_RUN_ID" arch="x86_64" + --attr project="$GITHUB_REPOSITORY" project.id="$GITHUB_REPOSITORY_ID" package="${{ env.clickhouse_binary_path }}" version="${{ env.version }}" user.name="$GITHUB_ACTOR" repository="https://github.com/Altinity/clickhouse-regression" commit.hash="$(git rev-parse HEAD)" job.id="$GITHUB_RUN_ID" job.url="$GITHUB_SERVER_URL/$GITHUB_REPOSITORY/actions/runs/$GITHUB_RUN_ID" arch="${uname -i}" --log raw.log - name: Create and upload logs if: always() @@ -735,7 +735,7 @@ jobs: --collect-service-logs --output classic --parallel 1 - --attr project="$GITHUB_REPOSITORY" project.id="$GITHUB_REPOSITORY_ID" package="${{ env.clickhouse_binary_path }}" version="${{ env.version }}" user.name="$GITHUB_ACTOR" repository="https://github.com/Altinity/clickhouse-regression" commit.hash="$(git rev-parse HEAD)" job.id="$GITHUB_RUN_ID" job.url="$GITHUB_SERVER_URL/$GITHUB_REPOSITORY/actions/runs/$GITHUB_RUN_ID" arch="x86_64" + --attr project="$GITHUB_REPOSITORY" project.id="$GITHUB_REPOSITORY_ID" package="${{ env.clickhouse_binary_path }}" version="${{ env.version }}" user.name="$GITHUB_ACTOR" repository="https://github.com/Altinity/clickhouse-regression" commit.hash="$(git rev-parse HEAD)" job.id="$GITHUB_RUN_ID" job.url="$GITHUB_SERVER_URL/$GITHUB_REPOSITORY/actions/runs/$GITHUB_RUN_ID" arch="${uname -i}" --log raw.log - name: Create and upload logs if: always() @@ -756,18 +756,83 @@ jobs: ./*/*/_instances/*/logs/*.log ./*/*/_instances/*.log - RegressionBenchmark: + RegressionBenchmarkAmd64: strategy: fail-fast: false matrix: STORAGE: [minio, aws_s3, gcs] - include: - - ARCH: arm - SERVER: cax41 - - ARCH: x86 - SERVER: cpx51 needs: [RegressionStart] - runs-on: [self-hosted, regression-tester, on-demand, 'type-${{ matrix.SERVER }}', 'image-${{ matrix.ARCH }}-app-docker-ce'] + runs-on: [self-hosted, regression-tester, on-demand, type-cpx51, image-x86-app-docker-ce] + env: + AWS_ACCESS_KEY_ID: ${{ secrets.AWS_REPORT_KEY_ID }} + AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_REPORT_SECRET_ACCESS_KEY }} + AWS_DEFAULT_REGION: ${{ secrets.AWS_REPORT_REGION }} + steps: + - name: Checkout regression repo + uses: actions/checkout@v3 + with: + repository: Altinity/clickhouse-regression + - name: Set envs + run: | + cat >> "$GITHUB_ENV" << 'EOF' + REPORTS_PATH=${{runner.temp}}/reports_dir + SUITE=ontime_benchmark + STORAGE=/${{ matrix.STORAGE }} + artifacts=public + EOF + - name: Download json reports + uses: actions/download-artifact@v3 + with: + path: ${{ env.REPORTS_PATH }} + - name: Setup + run: .github/setup.sh + - name: Get deb url + run: python3 .github/get-deb-url.py --reports-path ${{ env.REPORTS_PATH }} --github-env $GITHUB_ENV + - name: Run ${{ env.SUITE }} suite + run: python3 + -u ${{ env.SUITE }}/benchmark.py + --clickhouse-binary-path ${{ env.clickhouse_binary_path }} + --storage ${{ matrix.STORAGE }} + --gcs-uri ${{ secrets.REGRESSION_GCS_URI }} + --gcs-key-id ${{ secrets.REGRESSION_GCS_KEY_ID }} + --gcs-key-secret ${{ secrets.REGRESSION_GCS_KEY_SECRET }} + --aws-s3-bucket ${{ secrets.REGRESSION_AWS_S3_BUCKET }} + --aws-s3-region ${{ secrets.REGRESSION_AWS_S3_REGION }} + --aws-s3-key-id ${{ secrets.REGRESSION_AWS_S3_KEY_ID }} + --aws-s3-access-key ${{ secrets.REGRESSION_AWS_S3_SECRET_ACCESS_KEY }} + --test-to-end + --local + --collect-service-logs + --output classic + --parallel 1 + --attr project="$GITHUB_REPOSITORY" project.id="$GITHUB_REPOSITORY_ID" package="${{ env.clickhouse_binary_path }}" version="${{ env.version }}" user.name="$GITHUB_ACTOR" repository="https://github.com/Altinity/clickhouse-regression" commit.hash="$(git rev-parse HEAD)" job.id="$GITHUB_RUN_ID" job.url="$GITHUB_SERVER_URL/$GITHUB_REPOSITORY/actions/runs/$GITHUB_RUN_ID" arch="${uname -i}" + --log raw.log + - name: Create and upload logs + if: always() + run: .github/create_and_upload_logs.sh 1 + env: + artifact_s3_dir: build/v${{ env.version }}/$GITHUB_SHA + - uses: actions/upload-artifact@v3 + if: always() + with: + name: ${{ env.SUITE }}-minio-artifacts + path: | + ./report.html + ./*.log.txt + ./*.log + ./*.html + ./*/_instances/*.log + ./*/_instances/*/logs/*.log + ./*/*/_instances/*/logs/*.log + ./*/*/_instances/*.log + + RegressionBenchmarkAarch64: + strategy: + fail-fast: false + matrix: + STORAGE: [minio, aws_s3, gcs] + needs: [RegressionStart] + runs-on: [self-hosted, regression-tester, on-demand, type-cax41, image-arm-app-docker-ce] env: AWS_ACCESS_KEY_ID: ${{ secrets.AWS_REPORT_KEY_ID }} AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_REPORT_SECRET_ACCESS_KEY }} @@ -810,7 +875,7 @@ jobs: --collect-service-logs --output classic --parallel 1 - --attr project="$GITHUB_REPOSITORY" project.id="$GITHUB_REPOSITORY_ID" package="${{ env.clickhouse_binary_path }}" version="${{ env.version }}" user.name="$GITHUB_ACTOR" repository="https://github.com/Altinity/clickhouse-regression" commit.hash="$(git rev-parse HEAD)" job.id="$GITHUB_RUN_ID" job.url="$GITHUB_SERVER_URL/$GITHUB_REPOSITORY/actions/runs/$GITHUB_RUN_ID" arch="x86_64" + --attr project="$GITHUB_REPOSITORY" project.id="$GITHUB_REPOSITORY_ID" package="${{ env.clickhouse_binary_path }}" version="${{ env.version }}" user.name="$GITHUB_ACTOR" repository="https://github.com/Altinity/clickhouse-regression" commit.hash="$(git rev-parse HEAD)" job.id="$GITHUB_RUN_ID" job.url="$GITHUB_SERVER_URL/$GITHUB_REPOSITORY/actions/runs/$GITHUB_RUN_ID" arch="${uname -i}" --log raw.log - name: Create and upload logs if: always() @@ -831,18 +896,13 @@ jobs: ./*/*/_instances/*/logs/*.log ./*/*/_instances/*.log - RegressionLDAP: + RegressionLDAPAmd64: strategy: fail-fast: false matrix: SUITE: [authentication, external_user_directory, role_mapping] - include: - - ARCH: arm - SERVER: cax41 - - ARCH: x86 - SERVER: cpx51 needs: [RegressionStart] - runs-on: [self-hosted, regression-tester, on-demand, 'type-${{ matrix.SERVER }}', 'image-${{ matrix.ARCH }}-app-docker-ce'] + runs-on: [self-hosted, regression-tester, on-demand, type-cpx51, image-x86-app-docker-ce] env: AWS_ACCESS_KEY_ID: ${{ secrets.AWS_REPORT_KEY_ID }} AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_REPORT_SECRET_ACCESS_KEY }} @@ -876,7 +936,68 @@ jobs: --collect-service-logs --output classic --parallel 1 - --attr project="$GITHUB_REPOSITORY" project.id="$GITHUB_REPOSITORY_ID" package="${{ env.clickhouse_binary_path }}" version="${{ env.version }}" user.name="$GITHUB_ACTOR" repository="https://github.com/Altinity/clickhouse-regression" commit.hash="$(git rev-parse HEAD)" job.id="$GITHUB_RUN_ID" job.url="$GITHUB_SERVER_URL/$GITHUB_REPOSITORY/actions/runs/$GITHUB_RUN_ID" arch="x86_64" + --attr project="$GITHUB_REPOSITORY" project.id="$GITHUB_REPOSITORY_ID" package="${{ env.clickhouse_binary_path }}" version="${{ env.version }}" user.name="$GITHUB_ACTOR" repository="https://github.com/Altinity/clickhouse-regression" commit.hash="$(git rev-parse HEAD)" job.id="$GITHUB_RUN_ID" job.url="$GITHUB_SERVER_URL/$GITHUB_REPOSITORY/actions/runs/$GITHUB_RUN_ID" arch="${uname -i}" + --log raw.log + - name: Create and upload logs + if: always() + run: .github/create_and_upload_logs.sh 1 + env: + artifact_s3_dir: build/v${{ env.version }}/$GITHUB_SHA + - uses: actions/upload-artifact@v3 + if: always() + with: + name: ldap-authentication-artifacts + path: | + ./report.html + ./*.log.txt + ./*.log + ./*.html + ./*/_instances/*.log + ./*/_instances/*/logs/*.log + ./*/*/_instances/*/logs/*.log + ./*/*/_instances/*.log + + RegressionLDAPAarch64: + strategy: + fail-fast: false + matrix: + SUITE: [authentication, external_user_directory, role_mapping] + needs: [RegressionStart] + runs-on: [self-hosted, regression-tester, on-demand, type-cax41, image-arm-app-docker-ce] + env: + AWS_ACCESS_KEY_ID: ${{ secrets.AWS_REPORT_KEY_ID }} + AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_REPORT_SECRET_ACCESS_KEY }} + AWS_DEFAULT_REGION: ${{ secrets.AWS_REPORT_REGION }} + steps: + - name: Checkout regression repo + uses: actions/checkout@v3 + with: + repository: Altinity/clickhouse-regression + - name: Set envs + run: | + cat >> "$GITHUB_ENV" << 'EOF' + REPORTS_PATH=${{runner.temp}}/reports_dir + SUITE=ldap/${{ matrix.SUITE }} + artifacts=public + EOF + - name: Download json reports + uses: actions/download-artifact@v3 + with: + path: ${{ env.REPORTS_PATH }} + - name: Setup + run: .github/setup.sh + - name: Get deb url + run: python3 .github/get-deb-url.py --reports-path ${{ env.REPORTS_PATH }} --github-env $GITHUB_ENV + - name: Run ${{ env.SUITE }} suite + run: python3 + -u ${{ env.SUITE }}/regression.py + --clickhouse-binary-path ${{ env.clickhouse_binary_path }} + --test-to-end + --local + --collect-service-logs + --output classic + --parallel 1 + --attr project="$GITHUB_REPOSITORY" project.id="$GITHUB_REPOSITORY_ID" package="${{ env.clickhouse_binary_path }}" version="${{ env.version }}" user.name="$GITHUB_ACTOR" repository="https://github.com/Altinity/clickhouse-regression" commit.hash="$(git rev-parse HEAD)" job.id="$GITHUB_RUN_ID" job.url="$GITHUB_SERVER_URL/$GITHUB_REPOSITORY/actions/runs/$GITHUB_RUN_ID" arch="${uname -i}" --log raw.log - name: Create and upload logs if: always() @@ -899,7 +1020,7 @@ jobs: RegressionParquetAmd64: needs: [RegressionStart] - runs-on: [self-hosted, stress-tester] + runs-on: [self-hosted, regression-tester, on-demand, type-cpx51, image-x86-app-docker-ce] env: AWS_ACCESS_KEY_ID: ${{ secrets.AWS_REPORT_KEY_ID }} AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_REPORT_SECRET_ACCESS_KEY }} @@ -933,7 +1054,7 @@ jobs: --collect-service-logs --output classic --parallel 1 - --attr project="$GITHUB_REPOSITORY" project.id="$GITHUB_REPOSITORY_ID" package="${{ env.clickhouse_binary_path }}" version="${{ env.version }}" user.name="$GITHUB_ACTOR" repository="https://github.com/Altinity/clickhouse-regression" commit.hash="$(git rev-parse HEAD)" job.id="$GITHUB_RUN_ID" job.url="$GITHUB_SERVER_URL/$GITHUB_REPOSITORY/actions/runs/$GITHUB_RUN_ID" arch="x86_64" + --attr project="$GITHUB_REPOSITORY" project.id="$GITHUB_REPOSITORY_ID" package="${{ env.clickhouse_binary_path }}" version="${{ env.version }}" user.name="$GITHUB_ACTOR" repository="https://github.com/Altinity/clickhouse-regression" commit.hash="$(git rev-parse HEAD)" job.id="$GITHUB_RUN_ID" job.url="$GITHUB_SERVER_URL/$GITHUB_REPOSITORY/actions/runs/$GITHUB_RUN_ID" arch="${uname -i}" --log raw.log --storage minio --storage aws_s3 @@ -964,18 +1085,13 @@ jobs: ./*/*/_instances/*/logs/*.log ./*/*/_instances/*.log - RegressionS3: + RegressionS3Amd64: strategy: fail-fast: false matrix: STORAGE: [minio, aws_s3, gcs] - include: - - ARCH: arm - SERVER: cax41 - - ARCH: x86 - SERVER: cpx51 needs: [RegressionStart] - runs-on: [self-hosted, regression-tester, on-demand, 'type-${{ matrix.SERVER }}', 'image-${{ matrix.ARCH }}-app-docker-ce'] + runs-on: [self-hosted, regression-tester, on-demand, type-cpx51, image-x86-app-docker-ce] env: AWS_ACCESS_KEY_ID: ${{ secrets.AWS_REPORT_KEY_ID }} AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_REPORT_SECRET_ACCESS_KEY }} @@ -1010,7 +1126,7 @@ jobs: --collect-service-logs --output classic --parallel 1 - --attr project="$GITHUB_REPOSITORY" project.id="$GITHUB_REPOSITORY_ID" package="${{ env.clickhouse_binary_path }}" version="${{ env.version }}" user.name="$GITHUB_ACTOR" repository="https://github.com/Altinity/clickhouse-regression" commit.hash="$(git rev-parse HEAD)" job.id="$GITHUB_RUN_ID" job.url="$GITHUB_SERVER_URL/$GITHUB_REPOSITORY/actions/runs/$GITHUB_RUN_ID" arch="x86_64" + --attr project="$GITHUB_REPOSITORY" project.id="$GITHUB_REPOSITORY_ID" package="${{ env.clickhouse_binary_path }}" version="${{ env.version }}" user.name="$GITHUB_ACTOR" repository="https://github.com/Altinity/clickhouse-regression" commit.hash="$(git rev-parse HEAD)" job.id="$GITHUB_RUN_ID" job.url="$GITHUB_SERVER_URL/$GITHUB_REPOSITORY/actions/runs/$GITHUB_RUN_ID" arch="${uname -i}" --log raw.log --storage ${{ matrix.STORAGE }} --gcs-uri ${{ secrets.REGRESSION_GCS_URI }} @@ -1039,18 +1155,152 @@ jobs: ./*/*/_instances/*/logs/*.log ./*/*/_instances/*.log - RegressionTieredStorageS3: + RegressionS3Aarch64: strategy: fail-fast: false matrix: STORAGE: [minio, aws_s3, gcs] - include: - - ARCH: arm - SERVER: cax41 - - ARCH: x86 - SERVER: cpx51 needs: [RegressionStart] - runs-on: [self-hosted, regression-tester, on-demand, 'type-${{ matrix.SERVER }}', 'image-${{ matrix.ARCH }}-app-docker-ce'] + runs-on: [self-hosted, regression-tester, on-demand, type-cax41, image-arm-app-docker-ce] + env: + AWS_ACCESS_KEY_ID: ${{ secrets.AWS_REPORT_KEY_ID }} + AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_REPORT_SECRET_ACCESS_KEY }} + AWS_DEFAULT_REGION: ${{ secrets.AWS_REPORT_REGION }} + steps: + - name: Checkout regression repo + uses: actions/checkout@v3 + with: + repository: Altinity/clickhouse-regression + - name: Set envs + run: | + cat >> "$GITHUB_ENV" << 'EOF' + REPORTS_PATH=${{runner.temp}}/reports_dir + SUITE=s3 + STORAGE=/${{ matrix.STORAGE }} + artifacts=public + EOF + - name: Download json reports + uses: actions/download-artifact@v3 + with: + path: ${{ env.REPORTS_PATH }} + - name: Setup + run: .github/setup.sh + - name: Get deb url + run: python3 .github/get-deb-url.py --reports-path ${{ env.REPORTS_PATH }} --github-env $GITHUB_ENV + - name: Run ${{ env.SUITE }} suite + run: python3 + -u ${{ env.SUITE }}/regression.py + --clickhouse-binary-path ${{ env.clickhouse_binary_path }} + --test-to-end + --local + --collect-service-logs + --output classic + --parallel 1 + --attr project="$GITHUB_REPOSITORY" project.id="$GITHUB_REPOSITORY_ID" package="${{ env.clickhouse_binary_path }}" version="${{ env.version }}" user.name="$GITHUB_ACTOR" repository="https://github.com/Altinity/clickhouse-regression" commit.hash="$(git rev-parse HEAD)" job.id="$GITHUB_RUN_ID" job.url="$GITHUB_SERVER_URL/$GITHUB_REPOSITORY/actions/runs/$GITHUB_RUN_ID" arch="${uname -i}" + --log raw.log + --storage ${{ matrix.STORAGE }} + --gcs-uri ${{ secrets.REGRESSION_GCS_URI }} + --gcs-key-id ${{ secrets.REGRESSION_GCS_KEY_ID }} + --gcs-key-secret ${{ secrets.REGRESSION_GCS_KEY_SECRET }} + --aws-s3-bucket ${{ secrets.REGRESSION_AWS_S3_BUCKET }} + --aws-s3-region ${{ secrets.REGRESSION_AWS_S3_REGION }} + --aws-s3-key-id ${{ secrets.REGRESSION_AWS_S3_KEY_ID }} + --aws-s3-access-key ${{ secrets.REGRESSION_AWS_S3_SECRET_ACCESS_KEY }} + - name: Create and upload logs + if: always() + run: .github/create_and_upload_logs.sh 1 + env: + artifact_s3_dir: build/v${{ env.version }}/$GITHUB_SHA + - uses: actions/upload-artifact@v3 + if: always() + with: + name: ${{ env.SUITE }}-${{ matrix.STORAGE }}-${{ matrix.OS }}-artifacts + path: | + ./report.html + ./*.log.txt + ./*.log + ./*.html + ./*/_instances/*.log + ./*/_instances/*/logs/*.log + ./*/*/_instances/*/logs/*.log + ./*/*/_instances/*.log + + RegressionTieredStorageS3Amd64: + strategy: + fail-fast: false + matrix: + STORAGE: [minio, aws_s3, gcs] + needs: [RegressionStart] + runs-on: [self-hosted, regression-tester, on-demand, type-cpx51, image-x86-app-docker-ce] + env: + AWS_ACCESS_KEY_ID: ${{ secrets.AWS_REPORT_KEY_ID }} + AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_REPORT_SECRET_ACCESS_KEY }} + AWS_DEFAULT_REGION: ${{ secrets.AWS_REPORT_REGION }} + steps: + - name: Checkout regression repo + uses: actions/checkout@v3 + with: + repository: Altinity/clickhouse-regression + - name: Set envs + run: | + cat >> "$GITHUB_ENV" << 'EOF' + REPORTS_PATH=${{runner.temp}}/reports_dir + SUITE=tiered_storage + STORAGE=/${{ matrix.STORAGE }} + artifacts=public + EOF + - name: Download json reports + uses: actions/download-artifact@v3 + with: + path: ${{ env.REPORTS_PATH }} + - name: Setup + run: .github/setup.sh + - name: Get deb url + run: python3 .github/get-deb-url.py --reports-path ${{ env.REPORTS_PATH }} --github-env $GITHUB_ENV + - name: Run ${{ env.SUITE }} suite + run: python3 + -u ${{ env.SUITE }}/regression.py + --clickhouse-binary-path ${{ env.clickhouse_binary_path }} + --test-to-end + --local + --collect-service-logs + --output classic + --parallel 1 + --attr project="$GITHUB_REPOSITORY" project.id="$GITHUB_REPOSITORY_ID" package="${{ env.clickhouse_binary_path }}" version="${{ env.version }}" user.name="$GITHUB_ACTOR" repository="https://github.com/Altinity/clickhouse-regression" commit.hash="$(git rev-parse HEAD)" job.id="$GITHUB_RUN_ID" job.url="$GITHUB_SERVER_URL/$GITHUB_REPOSITORY/actions/runs/$GITHUB_RUN_ID" arch="${uname -i}" + --log raw.log + --aws-s3-access-key ${{ secrets.REGRESSION_AWS_S3_SECRET_ACCESS_KEY }} + --aws-s3-key-id ${{ secrets.REGRESSION_AWS_S3_KEY_ID }} + --aws-s3-uri https://s3.${{ secrets.REGRESSION_AWS_S3_REGION}}.amazonaws.com/${{ secrets.REGRESSION_AWS_S3_BUCKET }}/data/ + --gcs-key-id ${{ secrets.REGRESSION_GCS_KEY_ID }} + --gcs-key-secret ${{ secrets.REGRESSION_GCS_KEY_SECRET }} + --gcs-uri ${{ secrets.REGRESSION_GCS_URI }} + --with-${{ matrix.STORAGE }} + - name: Create and upload logs + if: always() + run: .github/create_and_upload_logs.sh 1 + env: + artifact_s3_dir: build/v${{ env.version }}/$GITHUB_SHA + - uses: actions/upload-artifact@v3 + if: always() + with: + name: ${{ env.SUITE }}-${{ matrix.STORAGE }}-${{ matrix.OS }}-artifacts + path: | + ./report.html + ./*.log.txt + ./*.log + ./*.html + ./*/_instances/*.log + ./*/_instances/*/logs/*.log + ./*/*/_instances/*/logs/*.log + ./*/*/_instances/*.log + + RegressionTieredStorageS3Aarch64: + strategy: + fail-fast: false + matrix: + STORAGE: [minio, aws_s3, gcs] + needs: [RegressionStart] + runs-on: [self-hosted, regression-tester, on-demand, type-cax41, image-arm-app-docker-ce] env: AWS_ACCESS_KEY_ID: ${{ secrets.AWS_REPORT_KEY_ID }} AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_REPORT_SECRET_ACCESS_KEY }} @@ -1085,7 +1335,7 @@ jobs: --collect-service-logs --output classic --parallel 1 - --attr project="$GITHUB_REPOSITORY" project.id="$GITHUB_REPOSITORY_ID" package="${{ env.clickhouse_binary_path }}" version="${{ env.version }}" user.name="$GITHUB_ACTOR" repository="https://github.com/Altinity/clickhouse-regression" commit.hash="$(git rev-parse HEAD)" job.id="$GITHUB_RUN_ID" job.url="$GITHUB_SERVER_URL/$GITHUB_REPOSITORY/actions/runs/$GITHUB_RUN_ID" arch="x86_64" + --attr project="$GITHUB_REPOSITORY" project.id="$GITHUB_REPOSITORY_ID" package="${{ env.clickhouse_binary_path }}" version="${{ env.version }}" user.name="$GITHUB_ACTOR" repository="https://github.com/Altinity/clickhouse-regression" commit.hash="$(git rev-parse HEAD)" job.id="$GITHUB_RUN_ID" job.url="$GITHUB_SERVER_URL/$GITHUB_REPOSITORY/actions/runs/$GITHUB_RUN_ID" arch="${uname -i}" --log raw.log --aws-s3-access-key ${{ secrets.REGRESSION_AWS_S3_SECRET_ACCESS_KEY }} --aws-s3-key-id ${{ secrets.REGRESSION_AWS_S3_KEY_ID }} @@ -1170,11 +1420,15 @@ jobs: - CompatibilityCheck - RegressionCommonAmd64 - RegressionCommonAarch64 - - RegressionBenchmark - - RegressionLDAP + - RegressionBenchmarkAmd64 + - RegressionBenchmarkAarch64 + - RegressionLDAPAmd64 + - RegressionLDAPAarch64 - RegressionParquetAmd64 - - RegressionS3 - - RegressionTieredStorageS3 + - RegressionS3Amd64 + - RegressionS3Aarch64 + - RegressionTieredStorageS3Amd64 + - RegressionTieredStorageS3Aarch64 - SignRelease runs-on: [self-hosted, style-checker, on-demand, type-cpx31, image-x86-app-docker-ce] steps: From 989e89792023bb93a44521f96b7400b9e0c10d73 Mon Sep 17 00:00:00 2001 From: MyroTk <44327070+MyroTk@users.noreply.github.com> Date: Mon, 18 Sep 2023 08:03:09 -0700 Subject: [PATCH 69/72] Update release_branches.yml --- .github/workflows/release_branches.yml | 22 +++++++++++----------- 1 file changed, 11 insertions(+), 11 deletions(-) diff --git a/.github/workflows/release_branches.yml b/.github/workflows/release_branches.yml index dd89642e964b..3dca3c0762a1 100644 --- a/.github/workflows/release_branches.yml +++ b/.github/workflows/release_branches.yml @@ -674,7 +674,7 @@ jobs: --collect-service-logs --output classic --parallel 1 - --attr project="$GITHUB_REPOSITORY" project.id="$GITHUB_REPOSITORY_ID" package="${{ env.clickhouse_binary_path }}" version="${{ env.version }}" user.name="$GITHUB_ACTOR" repository="https://github.com/Altinity/clickhouse-regression" commit.hash="$(git rev-parse HEAD)" job.id="$GITHUB_RUN_ID" job.url="$GITHUB_SERVER_URL/$GITHUB_REPOSITORY/actions/runs/$GITHUB_RUN_ID" arch="${uname -i}" + --attr project="$GITHUB_REPOSITORY" project.id="$GITHUB_REPOSITORY_ID" package="${{ env.clickhouse_binary_path }}" version="${{ env.version }}" user.name="$GITHUB_ACTOR" repository="https://github.com/Altinity/clickhouse-regression" commit.hash="$(git rev-parse HEAD)" job.id="$GITHUB_RUN_ID" job.url="$GITHUB_SERVER_URL/$GITHUB_REPOSITORY/actions/runs/$GITHUB_RUN_ID" arch="$(uname -i)" --log raw.log - name: Create and upload logs if: always() @@ -735,7 +735,7 @@ jobs: --collect-service-logs --output classic --parallel 1 - --attr project="$GITHUB_REPOSITORY" project.id="$GITHUB_REPOSITORY_ID" package="${{ env.clickhouse_binary_path }}" version="${{ env.version }}" user.name="$GITHUB_ACTOR" repository="https://github.com/Altinity/clickhouse-regression" commit.hash="$(git rev-parse HEAD)" job.id="$GITHUB_RUN_ID" job.url="$GITHUB_SERVER_URL/$GITHUB_REPOSITORY/actions/runs/$GITHUB_RUN_ID" arch="${uname -i}" + --attr project="$GITHUB_REPOSITORY" project.id="$GITHUB_REPOSITORY_ID" package="${{ env.clickhouse_binary_path }}" version="${{ env.version }}" user.name="$GITHUB_ACTOR" repository="https://github.com/Altinity/clickhouse-regression" commit.hash="$(git rev-parse HEAD)" job.id="$GITHUB_RUN_ID" job.url="$GITHUB_SERVER_URL/$GITHUB_REPOSITORY/actions/runs/$GITHUB_RUN_ID" arch="$(uname -i)" --log raw.log - name: Create and upload logs if: always() @@ -805,7 +805,7 @@ jobs: --collect-service-logs --output classic --parallel 1 - --attr project="$GITHUB_REPOSITORY" project.id="$GITHUB_REPOSITORY_ID" package="${{ env.clickhouse_binary_path }}" version="${{ env.version }}" user.name="$GITHUB_ACTOR" repository="https://github.com/Altinity/clickhouse-regression" commit.hash="$(git rev-parse HEAD)" job.id="$GITHUB_RUN_ID" job.url="$GITHUB_SERVER_URL/$GITHUB_REPOSITORY/actions/runs/$GITHUB_RUN_ID" arch="${uname -i}" + --attr project="$GITHUB_REPOSITORY" project.id="$GITHUB_REPOSITORY_ID" package="${{ env.clickhouse_binary_path }}" version="${{ env.version }}" user.name="$GITHUB_ACTOR" repository="https://github.com/Altinity/clickhouse-regression" commit.hash="$(git rev-parse HEAD)" job.id="$GITHUB_RUN_ID" job.url="$GITHUB_SERVER_URL/$GITHUB_REPOSITORY/actions/runs/$GITHUB_RUN_ID" arch="$(uname -i)" --log raw.log - name: Create and upload logs if: always() @@ -875,7 +875,7 @@ jobs: --collect-service-logs --output classic --parallel 1 - --attr project="$GITHUB_REPOSITORY" project.id="$GITHUB_REPOSITORY_ID" package="${{ env.clickhouse_binary_path }}" version="${{ env.version }}" user.name="$GITHUB_ACTOR" repository="https://github.com/Altinity/clickhouse-regression" commit.hash="$(git rev-parse HEAD)" job.id="$GITHUB_RUN_ID" job.url="$GITHUB_SERVER_URL/$GITHUB_REPOSITORY/actions/runs/$GITHUB_RUN_ID" arch="${uname -i}" + --attr project="$GITHUB_REPOSITORY" project.id="$GITHUB_REPOSITORY_ID" package="${{ env.clickhouse_binary_path }}" version="${{ env.version }}" user.name="$GITHUB_ACTOR" repository="https://github.com/Altinity/clickhouse-regression" commit.hash="$(git rev-parse HEAD)" job.id="$GITHUB_RUN_ID" job.url="$GITHUB_SERVER_URL/$GITHUB_REPOSITORY/actions/runs/$GITHUB_RUN_ID" arch="$(uname -i)" --log raw.log - name: Create and upload logs if: always() @@ -936,7 +936,7 @@ jobs: --collect-service-logs --output classic --parallel 1 - --attr project="$GITHUB_REPOSITORY" project.id="$GITHUB_REPOSITORY_ID" package="${{ env.clickhouse_binary_path }}" version="${{ env.version }}" user.name="$GITHUB_ACTOR" repository="https://github.com/Altinity/clickhouse-regression" commit.hash="$(git rev-parse HEAD)" job.id="$GITHUB_RUN_ID" job.url="$GITHUB_SERVER_URL/$GITHUB_REPOSITORY/actions/runs/$GITHUB_RUN_ID" arch="${uname -i}" + --attr project="$GITHUB_REPOSITORY" project.id="$GITHUB_REPOSITORY_ID" package="${{ env.clickhouse_binary_path }}" version="${{ env.version }}" user.name="$GITHUB_ACTOR" repository="https://github.com/Altinity/clickhouse-regression" commit.hash="$(git rev-parse HEAD)" job.id="$GITHUB_RUN_ID" job.url="$GITHUB_SERVER_URL/$GITHUB_REPOSITORY/actions/runs/$GITHUB_RUN_ID" arch="$(uname -i)" --log raw.log - name: Create and upload logs if: always() @@ -997,7 +997,7 @@ jobs: --collect-service-logs --output classic --parallel 1 - --attr project="$GITHUB_REPOSITORY" project.id="$GITHUB_REPOSITORY_ID" package="${{ env.clickhouse_binary_path }}" version="${{ env.version }}" user.name="$GITHUB_ACTOR" repository="https://github.com/Altinity/clickhouse-regression" commit.hash="$(git rev-parse HEAD)" job.id="$GITHUB_RUN_ID" job.url="$GITHUB_SERVER_URL/$GITHUB_REPOSITORY/actions/runs/$GITHUB_RUN_ID" arch="${uname -i}" + --attr project="$GITHUB_REPOSITORY" project.id="$GITHUB_REPOSITORY_ID" package="${{ env.clickhouse_binary_path }}" version="${{ env.version }}" user.name="$GITHUB_ACTOR" repository="https://github.com/Altinity/clickhouse-regression" commit.hash="$(git rev-parse HEAD)" job.id="$GITHUB_RUN_ID" job.url="$GITHUB_SERVER_URL/$GITHUB_REPOSITORY/actions/runs/$GITHUB_RUN_ID" arch="$(uname -i)" --log raw.log - name: Create and upload logs if: always() @@ -1054,7 +1054,7 @@ jobs: --collect-service-logs --output classic --parallel 1 - --attr project="$GITHUB_REPOSITORY" project.id="$GITHUB_REPOSITORY_ID" package="${{ env.clickhouse_binary_path }}" version="${{ env.version }}" user.name="$GITHUB_ACTOR" repository="https://github.com/Altinity/clickhouse-regression" commit.hash="$(git rev-parse HEAD)" job.id="$GITHUB_RUN_ID" job.url="$GITHUB_SERVER_URL/$GITHUB_REPOSITORY/actions/runs/$GITHUB_RUN_ID" arch="${uname -i}" + --attr project="$GITHUB_REPOSITORY" project.id="$GITHUB_REPOSITORY_ID" package="${{ env.clickhouse_binary_path }}" version="${{ env.version }}" user.name="$GITHUB_ACTOR" repository="https://github.com/Altinity/clickhouse-regression" commit.hash="$(git rev-parse HEAD)" job.id="$GITHUB_RUN_ID" job.url="$GITHUB_SERVER_URL/$GITHUB_REPOSITORY/actions/runs/$GITHUB_RUN_ID" arch="$(uname -i)" --log raw.log --storage minio --storage aws_s3 @@ -1126,7 +1126,7 @@ jobs: --collect-service-logs --output classic --parallel 1 - --attr project="$GITHUB_REPOSITORY" project.id="$GITHUB_REPOSITORY_ID" package="${{ env.clickhouse_binary_path }}" version="${{ env.version }}" user.name="$GITHUB_ACTOR" repository="https://github.com/Altinity/clickhouse-regression" commit.hash="$(git rev-parse HEAD)" job.id="$GITHUB_RUN_ID" job.url="$GITHUB_SERVER_URL/$GITHUB_REPOSITORY/actions/runs/$GITHUB_RUN_ID" arch="${uname -i}" + --attr project="$GITHUB_REPOSITORY" project.id="$GITHUB_REPOSITORY_ID" package="${{ env.clickhouse_binary_path }}" version="${{ env.version }}" user.name="$GITHUB_ACTOR" repository="https://github.com/Altinity/clickhouse-regression" commit.hash="$(git rev-parse HEAD)" job.id="$GITHUB_RUN_ID" job.url="$GITHUB_SERVER_URL/$GITHUB_REPOSITORY/actions/runs/$GITHUB_RUN_ID" arch="$(uname -i)" --log raw.log --storage ${{ matrix.STORAGE }} --gcs-uri ${{ secrets.REGRESSION_GCS_URI }} @@ -1196,7 +1196,7 @@ jobs: --collect-service-logs --output classic --parallel 1 - --attr project="$GITHUB_REPOSITORY" project.id="$GITHUB_REPOSITORY_ID" package="${{ env.clickhouse_binary_path }}" version="${{ env.version }}" user.name="$GITHUB_ACTOR" repository="https://github.com/Altinity/clickhouse-regression" commit.hash="$(git rev-parse HEAD)" job.id="$GITHUB_RUN_ID" job.url="$GITHUB_SERVER_URL/$GITHUB_REPOSITORY/actions/runs/$GITHUB_RUN_ID" arch="${uname -i}" + --attr project="$GITHUB_REPOSITORY" project.id="$GITHUB_REPOSITORY_ID" package="${{ env.clickhouse_binary_path }}" version="${{ env.version }}" user.name="$GITHUB_ACTOR" repository="https://github.com/Altinity/clickhouse-regression" commit.hash="$(git rev-parse HEAD)" job.id="$GITHUB_RUN_ID" job.url="$GITHUB_SERVER_URL/$GITHUB_REPOSITORY/actions/runs/$GITHUB_RUN_ID" arch="$(uname -i)" --log raw.log --storage ${{ matrix.STORAGE }} --gcs-uri ${{ secrets.REGRESSION_GCS_URI }} @@ -1266,7 +1266,7 @@ jobs: --collect-service-logs --output classic --parallel 1 - --attr project="$GITHUB_REPOSITORY" project.id="$GITHUB_REPOSITORY_ID" package="${{ env.clickhouse_binary_path }}" version="${{ env.version }}" user.name="$GITHUB_ACTOR" repository="https://github.com/Altinity/clickhouse-regression" commit.hash="$(git rev-parse HEAD)" job.id="$GITHUB_RUN_ID" job.url="$GITHUB_SERVER_URL/$GITHUB_REPOSITORY/actions/runs/$GITHUB_RUN_ID" arch="${uname -i}" + --attr project="$GITHUB_REPOSITORY" project.id="$GITHUB_REPOSITORY_ID" package="${{ env.clickhouse_binary_path }}" version="${{ env.version }}" user.name="$GITHUB_ACTOR" repository="https://github.com/Altinity/clickhouse-regression" commit.hash="$(git rev-parse HEAD)" job.id="$GITHUB_RUN_ID" job.url="$GITHUB_SERVER_URL/$GITHUB_REPOSITORY/actions/runs/$GITHUB_RUN_ID" arch="$(uname -i)" --log raw.log --aws-s3-access-key ${{ secrets.REGRESSION_AWS_S3_SECRET_ACCESS_KEY }} --aws-s3-key-id ${{ secrets.REGRESSION_AWS_S3_KEY_ID }} @@ -1335,7 +1335,7 @@ jobs: --collect-service-logs --output classic --parallel 1 - --attr project="$GITHUB_REPOSITORY" project.id="$GITHUB_REPOSITORY_ID" package="${{ env.clickhouse_binary_path }}" version="${{ env.version }}" user.name="$GITHUB_ACTOR" repository="https://github.com/Altinity/clickhouse-regression" commit.hash="$(git rev-parse HEAD)" job.id="$GITHUB_RUN_ID" job.url="$GITHUB_SERVER_URL/$GITHUB_REPOSITORY/actions/runs/$GITHUB_RUN_ID" arch="${uname -i}" + --attr project="$GITHUB_REPOSITORY" project.id="$GITHUB_REPOSITORY_ID" package="${{ env.clickhouse_binary_path }}" version="${{ env.version }}" user.name="$GITHUB_ACTOR" repository="https://github.com/Altinity/clickhouse-regression" commit.hash="$(git rev-parse HEAD)" job.id="$GITHUB_RUN_ID" job.url="$GITHUB_SERVER_URL/$GITHUB_REPOSITORY/actions/runs/$GITHUB_RUN_ID" arch="$(uname -i)" --log raw.log --aws-s3-access-key ${{ secrets.REGRESSION_AWS_S3_SECRET_ACCESS_KEY }} --aws-s3-key-id ${{ secrets.REGRESSION_AWS_S3_KEY_ID }} From 60b289ad6ce69f6d7d3ed5931481b24947142237 Mon Sep 17 00:00:00 2001 From: MyroTk <44327070+MyroTk@users.noreply.github.com> Date: Mon, 18 Sep 2023 17:00:22 -0700 Subject: [PATCH 70/72] Update release_branches.yml --- .github/workflows/release_branches.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/release_branches.yml b/.github/workflows/release_branches.yml index 3dca3c0762a1..1dee4cd44c8c 100644 --- a/.github/workflows/release_branches.yml +++ b/.github/workflows/release_branches.yml @@ -699,7 +699,7 @@ jobs: strategy: fail-fast: false matrix: - SUITE: [aes_encryption, base_58, datetime64_extended_range, disk_level_encryption, example, extended_precision_data_types, kafka, lightweight_delete, map_type, part_moves_between_shards, rbac, selects, tiered_storage] + SUITE: [aes_encryption, base_58, datetime64_extended_range, disk_level_encryption, example, extended_precision_data_types, kafka, lightweight_delete, data_types, part_moves_between_shards, rbac, selects, tiered_storage, windows_functions] needs: [RegressionStart] runs-on: [self-hosted, regression-tester, on-demand, type-cax41, image-arm-app-docker-ce] env: @@ -1229,7 +1229,7 @@ jobs: strategy: fail-fast: false matrix: - STORAGE: [minio, aws_s3, gcs] + STORAGE: [minio, s3amazon, s3gcs] needs: [RegressionStart] runs-on: [self-hosted, regression-tester, on-demand, type-cpx51, image-x86-app-docker-ce] env: @@ -1298,7 +1298,7 @@ jobs: strategy: fail-fast: false matrix: - STORAGE: [minio, aws_s3, gcs] + STORAGE: [minio, s3amazon, s3gcs] needs: [RegressionStart] runs-on: [self-hosted, regression-tester, on-demand, type-cax41, image-arm-app-docker-ce] env: From 6b52da82b6ca003e8ebd34ead5f4eabbc45d1cd6 Mon Sep 17 00:00:00 2001 From: MyroTk <44327070+MyroTk@users.noreply.github.com> Date: Mon, 18 Sep 2023 21:33:51 -0700 Subject: [PATCH 71/72] Update release_branches.yml --- .github/workflows/release_branches.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/release_branches.yml b/.github/workflows/release_branches.yml index 1dee4cd44c8c..f7cc00f75fef 100644 --- a/.github/workflows/release_branches.yml +++ b/.github/workflows/release_branches.yml @@ -638,7 +638,7 @@ jobs: strategy: fail-fast: false matrix: - SUITE: [aes_encryption, aggregate_functions, atomic_insert, base_58, clickhouse_keeper, datetime64_extended_range, disk_level_encryption, dns, example, extended_precision_data_types, kafka, kerberos, lightweight_delete, map_type, part_moves_between_shards, rbac, selects, ssl_server, tiered_storage, window_functions] + SUITE: [aes_encryption, aggregate_functions, atomic_insert, base_58, clickhouse_keeper, datetime64_extended_range, disk_level_encryption, dns, example, extended_precision_data_types, kafka, kerberos, lightweight_delete, data_types, part_moves_between_shards, rbac, selects, ssl_server, tiered_storage, window_functions] needs: [RegressionStart] runs-on: [self-hosted, regression-tester, on-demand, type-cpx51, image-x86-app-docker-ce] env: @@ -699,7 +699,7 @@ jobs: strategy: fail-fast: false matrix: - SUITE: [aes_encryption, base_58, datetime64_extended_range, disk_level_encryption, example, extended_precision_data_types, kafka, lightweight_delete, data_types, part_moves_between_shards, rbac, selects, tiered_storage, windows_functions] + SUITE: [aes_encryption, base_58, datetime64_extended_range, disk_level_encryption, example, extended_precision_data_types, kafka, lightweight_delete, data_types, part_moves_between_shards, rbac, selects, tiered_storage, window_functions] needs: [RegressionStart] runs-on: [self-hosted, regression-tester, on-demand, type-cax41, image-arm-app-docker-ce] env: From 227b78a228b6ee63ea9133e122449eb5cfd8efd6 Mon Sep 17 00:00:00 2001 From: MyroTk <44327070+MyroTk@users.noreply.github.com> Date: Thu, 21 Sep 2023 08:23:12 -0700 Subject: [PATCH 72/72] Enabling latest ARM regression tests --- .github/workflows/release_branches.yml | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/.github/workflows/release_branches.yml b/.github/workflows/release_branches.yml index f7cc00f75fef..954284228bf5 100644 --- a/.github/workflows/release_branches.yml +++ b/.github/workflows/release_branches.yml @@ -7,6 +7,7 @@ env: AWS_DEFAULT_REGION: ${{ secrets.AWS_DEFAULT_REGION }} AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }} REGRESSION_RESULTS_URL: altinity-build-artifacts/${{github.event.number}}/$GITHUB_SHA + REGRESSION_ARM_COMMIT: e948952d4910b4bc9d95ddedc439903a0f795778 on: # yamllint disable-line rule:truthy @@ -699,7 +700,7 @@ jobs: strategy: fail-fast: false matrix: - SUITE: [aes_encryption, base_58, datetime64_extended_range, disk_level_encryption, example, extended_precision_data_types, kafka, lightweight_delete, data_types, part_moves_between_shards, rbac, selects, tiered_storage, window_functions] + SUITE: [aes_encryption, atomic_insert, base_58, datetime64_extended_range, data_types, disk_level_encryption, dns, engines, example, extended_precision_data_types, kafka, key_value, lightweight_delete, part_moves_between_shards, rbac, selects, session_timezone, tiered_storage, window_functions] needs: [RegressionStart] runs-on: [self-hosted, regression-tester, on-demand, type-cax41, image-arm-app-docker-ce] env: @@ -711,6 +712,7 @@ jobs: uses: actions/checkout@v3 with: repository: Altinity/clickhouse-regression + ref: ${{ env.REGRESSION_ARM_COMMIT }} - name: Set envs run: | cat >> "$GITHUB_ENV" << 'EOF' @@ -842,6 +844,7 @@ jobs: uses: actions/checkout@v3 with: repository: Altinity/clickhouse-regression + ref: ${{ env.REGRESSION_ARM_COMMIT }} - name: Set envs run: | cat >> "$GITHUB_ENV" << 'EOF' @@ -973,6 +976,7 @@ jobs: uses: actions/checkout@v3 with: repository: Altinity/clickhouse-regression + ref: ${{ env.REGRESSION_ARM_COMMIT }} - name: Set envs run: | cat >> "$GITHUB_ENV" << 'EOF' @@ -1171,6 +1175,7 @@ jobs: uses: actions/checkout@v3 with: repository: Altinity/clickhouse-regression + ref: ${{ env.REGRESSION_ARM_COMMIT }} - name: Set envs run: | cat >> "$GITHUB_ENV" << 'EOF' @@ -1310,6 +1315,7 @@ jobs: uses: actions/checkout@v3 with: repository: Altinity/clickhouse-regression + ref: ${{ env.REGRESSION_ARM_COMMIT }} - name: Set envs run: | cat >> "$GITHUB_ENV" << 'EOF'