22.8 Backport of #54370 - fix parquet array inconsistent offsets #724

Workflow file for this run

.github/workflows/release_branches.yml at b3f3b3b

	name: ReleaseBranchCI

	env:
	# Force the stdout and stderr streams to be unbuffered
	PYTHONUNBUFFERED: 1
	AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }}
	AWS_DEFAULT_REGION: ${{ secrets.AWS_DEFAULT_REGION }}
	AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
	REGRESSION_RESULTS_URL: altinity-build-artifacts/${{github.event.number}}/$GITHUB_SHA


	on: # yamllint disable-line rule:truthy
	pull_request:
	types:
	- synchronize
	- reopened
	- opened
	branches:
	# Anything/22.8 (e.g customizations/22.8)
	- '*/22.8'
	release:
	types:
	- published
	- prereleased
	push:
	branches:
	- 'releases/22.8**'

	jobs:
	# DockerHubPushAarch64:
	# runs-on: [self-hosted, style-checker-aarch64]
	# steps:
	# - name: Check out repository code
	# uses: ClickHouse/checkout@v1
	# - name: Images check
	# run: \|
	# cd "$GITHUB_WORKSPACE/tests/ci"
	# python3 docker_images_check.py --suffix aarch64
	# - name: Upload images files to artifacts
	# uses: actions/upload-artifact@v2
	# with:
	# name: changed_images_aarch64
	# path: ${{ runner.temp }}/docker_images_check/changed_images_aarch64.json
	# Former DockerHubPushAmd64
	DockerHubPush:
	runs-on: [self-hosted, style-checker]
	steps:
	- name: Check out repository code
	uses: ClickHouse/checkout@v1
	with:
	clear-repository: true
	- name: Images check
	run: \|
	cd "$GITHUB_WORKSPACE/tests/ci"
	python3 docker_images_check.py --suffix amd64
	# TODO(vnemkov): remove this step if you uncomment DockerHubPushAarch64 and DockerHubPush below.
	# The rest of the pipeline expects changed_images.json, which was generated by previous version of DockerHubPush.
	- name: Rename artifact
	run: \|
	mv ${{ runner.temp }}/docker_images_check/changed_images_amd64.json ${{ runner.temp }}/docker_images_check/changed_images.json
	- name: Upload images files to artifacts
	uses: actions/upload-artifact@v3
	with:
	name: changed_images
	path: ${{ runner.temp }}/docker_images_check/changed_images.json
	# DockerHubPush:
	# needs: [DockerHubPushAmd64, DockerHubPushAarch64]
	# runs-on: [self-hosted, style-checker]
	# steps:
	# - name: Check out repository code
	# uses: ClickHouse/checkout@v1
	# - name: Download changed aarch64 images
	# uses: actions/download-artifact@v2
	# with:
	# name: changed_images_aarch64
	# path: ${{ runner.temp }}
	# - name: Download changed amd64 images
	# uses: actions/download-artifact@v2
	# with:
	# name: changed_images_amd64
	# path: ${{ runner.temp }}
	# - name: Images check
	# run: \|
	# cd "$GITHUB_WORKSPACE/tests/ci"
	# python3 docker_manifests_merge.py --suffix amd64 --suffix aarch64
	# - name: Upload images files to artifacts
	# uses: actions/upload-artifact@v2
	# with:
	# name: changed_images
	# path: ${{ runner.temp }}/changed_images.json
	CompatibilityCheck:
	needs: [BuilderDebRelease]
	runs-on: [self-hosted, style-checker]
	steps:
	- name: Set envs
	run: \|
	cat >> "$GITHUB_ENV" << 'EOF'
	TEMP_PATH=${{runner.temp}}/compatibility_check
	REPO_COPY=${{runner.temp}}/compatibility_check/ClickHouse
	REPORTS_PATH=${{runner.temp}}/reports_dir
	EOF
	- name: Check out repository code
	uses: ClickHouse/checkout@v1
	with:
	clear-repository: true
	- name: Download json reports
	uses: actions/download-artifact@v3
	with:
	path: ${{ env.REPORTS_PATH }}
	- name: CompatibilityCheck
	run: \|
	sudo rm -fr "$TEMP_PATH"
	mkdir -p "$TEMP_PATH"
	cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH"
	cd "$REPO_COPY/tests/ci" && python3 compatibility_check.py
	- name: Cleanup
	if: always()
	run: \|
	# shellcheck disable=SC2046
	docker kill $(docker ps -q) \|\|:
	# shellcheck disable=SC2046
	docker rm -f $(docker ps -a -q) \|\|:
	sudo rm -fr "$TEMP_PATH"
	#########################################################################################
	#################################### ORDINARY BUILDS ####################################
	#########################################################################################
	BuilderDebRelease:
	needs: [DockerHubPush]
	runs-on: [self-hosted, builder]
	steps:
	- name: Set envs
	run: \|
	cat >> "$GITHUB_ENV" << 'EOF'
	TEMP_PATH=${{runner.temp}}/build_check
	IMAGES_PATH=${{runner.temp}}/images_path
	REPO_COPY=${{runner.temp}}/build_check/ClickHouse
	CACHES_PATH=${{runner.temp}}/../ccaches
	BUILD_NAME=package_release
	CLICKHOUSE_STABLE_VERSION_SUFFIX=altinitystable
	EOF
	- name: Download changed images
	uses: actions/download-artifact@v3
	with:
	name: changed_images
	path: ${{ env.IMAGES_PATH }}
	- name: Trust My Directory
	run: git config --global --add safe.directory * # https://stackoverflow.com/a/71940133
	- name: Check out repository code
	uses: ClickHouse/checkout@v1
	with:
	clear-repository: true
	submodules: true
	fetch-depth: 0 # otherwise we will have no info about contributors
	- name: Build
	run: \|
	sudo rm -fr "$TEMP_PATH"
	mkdir -p "$TEMP_PATH/build_check/package_release"
	cd .. && tar czf $TEMP_PATH/build_source.src.tar.gz ClickHouse/
	cd $TEMP_PATH && tar xvzf $TEMP_PATH/build_source.src.tar.gz
	cd "$REPO_COPY/tests/ci" && python3 build_check.py "$BUILD_NAME"
	- name: Upload build URLs to artifacts
	if: ${{ success() \|\| failure() }}
	uses: actions/upload-artifact@v3
	with:
	name: ${{ env.BUILD_URLS }}
	path: ${{ env.TEMP_PATH }}/${{ env.BUILD_URLS }}.json
	- name: Cleanup
	if: always()
	run: \|
	# shellcheck disable=SC2046
	docker kill $(docker ps -q) \|\|:
	# shellcheck disable=SC2046
	docker rm -f $(docker ps -a -q) \|\|:
	sudo rm -fr "$TEMP_PATH" "$CACHES_PATH"
	# BuilderDebAarch64:
	# needs: [DockerHubPush]
	# runs-on: [self-hosted, builder]
	# steps:
	# - name: Set envs
	# run: \|
	# cat >> "$GITHUB_ENV" << 'EOF'
	# TEMP_PATH=${{runner.temp}}/build_check
	# IMAGES_PATH=${{runner.temp}}/images_path
	# REPO_COPY=${{runner.temp}}/build_check/ClickHouse
	# CACHES_PATH=${{runner.temp}}/../ccaches
	# BUILD_NAME=package_aarch64
	# EOF
	# - name: Download changed images
	# uses: actions/download-artifact@v2
	# with:
	# name: changed_images
	# path: ${{ runner.temp }}/images_path
	# - name: Check out repository code
	# uses: ClickHouse/checkout@v1
	# with:
	# fetch-depth: 0 # otherwise we will have no info about contributors
	# - name: Build
	# run: \|
	# git -C "$GITHUB_WORKSPACE" submodule sync
	# git -C "$GITHUB_WORKSPACE" submodule update --depth=1 --recursive --init --jobs=10
	# sudo rm -fr "$TEMP_PATH"
	# mkdir -p "$TEMP_PATH"
	# cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH"
	# cd "$REPO_COPY/tests/ci" && python3 build_check.py "$BUILD_NAME"
	# - name: Upload build URLs to artifacts
	# uses: actions/upload-artifact@v2
	# with:
	# name: ${{ env.BUILD_URLS }}
	# path: ${{ runner.temp }}/build_check/${{ env.BUILD_URLS }}.json
	# - name: Cleanup
	# if: always()
	# run: \|
	# # shellcheck disable=SC2046
	# docker kill $(docker ps -q) \|\|:
	# # shellcheck disable=SC2046
	# docker rm -f $(docker ps -a -q) \|\|:
	# sudo rm -fr "$TEMP_PATH" "$CACHES_PATH"
	############################################################################################
	##################################### Docker images #######################################
	############################################################################################
	DockerServerImages:
	needs:
	- BuilderDebRelease
	# - BuilderDebAarch64
	runs-on: [self-hosted, style-checker]
	steps:
	- name: Check out repository code
	uses: ClickHouse/checkout@v1
	with:
	clear-repository: true
	fetch-depth: 0 # It MUST BE THE SAME for all dependencies and the job itself
	- name: Check docker clickhouse/clickhouse-server building
	run: \|
	cd "$GITHUB_WORKSPACE/tests/ci"
	python3 docker_server.py --release-type head --no-push \
	--image-repo clickhouse/clickhouse-server --image-path docker/server
	python3 docker_server.py --release-type head --no-push --no-ubuntu \
	--image-repo clickhouse/clickhouse-keeper --image-path docker/keeper
	- name: Cleanup
	if: always()
	run: \|
	# shellcheck disable=SC2046
	docker kill $(docker ps -q) \|\|:
	# shellcheck disable=SC2046
	docker rm -f $(docker ps -a -q) \|\|:
	sudo rm -fr "$TEMP_PATH"
	############################################################################################
	##################################### BUILD REPORTER #######################################
	############################################################################################
	BuilderReport:
	needs:
	- BuilderDebRelease
	# - BuilderDebAarch64
	runs-on: [self-hosted, style-checker]
	if: ${{ success() \|\| failure() }}
	steps:
	- name: Set envs
	run: \|
	cat >> "$GITHUB_ENV" << 'EOF'
	CHECK_NAME=ClickHouse build check
	REPORTS_PATH=${{runner.temp}}/reports_dir
	REPORTS_PATH=${{runner.temp}}/reports_dir
	TEMP_PATH=${{runner.temp}}/report_check
	NEEDS_DATA_PATH=${{runner.temp}}/needs.json
	EOF
	- name: Download json reports
	uses: actions/download-artifact@v3
	with:
	path: ${{ env.REPORTS_PATH }}
	- name: Check out repository code
	uses: ClickHouse/checkout@v1
	with:
	clear-repository: true
	- name: Report Builder
	run: \|
	sudo rm -fr "$TEMP_PATH"
	mkdir -p "$TEMP_PATH"
	cat > "$NEEDS_DATA_PATH" << 'EOF'
	${{ toJSON(needs) }}
	EOF
	cd "$GITHUB_WORKSPACE/tests/ci"
	python3 build_report_check.py "$CHECK_NAME"
	- name: Cleanup
	if: always()
	run: \|
	# shellcheck disable=SC2046
	docker kill $(docker ps -q) \|\|:
	# shellcheck disable=SC2046
	docker rm -f $(docker ps -a -q) \|\|:
	sudo rm -fr "$TEMP_PATH"
	# BuilderSpecialReport:
	# needs:
	# # - BuilderBinDarwin
	# - BuilderBinDarwinAarch64
	# runs-on: [self-hosted, style-checker]
	# if: ${{ success() \|\| failure() }}
	# steps:
	# - name: Set envs
	# run: \|
	# cat >> "$GITHUB_ENV" << 'EOF'
	# TEMP_PATH=${{runner.temp}}/report_check
	# REPORTS_PATH=${{runner.temp}}/reports_dir
	# CHECK_NAME=ClickHouse special build check
	# NEEDS_DATA_PATH=${{runner.temp}}/needs.json
	# EOF
	# - name: Download json reports
	# uses: actions/download-artifact@v3
	# with:
	# path: ${{ env.REPORTS_PATH }}
	# - name: Check out repository code
	# uses: ClickHouse/checkout@v1
	# with:
	# clear-repository: true
	# - name: Report Builder
	# run: \|
	# sudo rm -fr "$TEMP_PATH"
	# mkdir -p "$TEMP_PATH"
	# cat > "$NEEDS_DATA_PATH" << 'EOF'
	# ${{ toJSON(needs) }}
	# EOF
	# cd "$GITHUB_WORKSPACE/tests/ci"
	# python3 build_report_check.py "$CHECK_NAME"
	# - name: Cleanup
	# if: always()
	# run: \|
	# # shellcheck disable=SC2046
	# docker kill $(docker ps -q) \|\|:
	# # shellcheck disable=SC2046
	# docker rm -f $(docker ps -a -q) \|\|:
	# sudo rm -fr "$TEMP_PATH"
	MarkReleaseReady:
	needs:
	# - BuilderBinDarwin
	# - BuilderBinDarwinAarch64
	- BuilderDebRelease
	# - BuilderDebAarch64
	runs-on: [self-hosted, style-checker]
	steps:
	- name: Check out repository code
	uses: ClickHouse/checkout@v1
	with:
	clear-repository: true
	- name: Mark Commit Release Ready
	run: \|
	cd "$GITHUB_WORKSPACE/tests/ci"
	python3 mark_release_ready.py
	##############################################################################################
	########################### FUNCTIONAl STATELESS TESTS #######################################
	##############################################################################################
	FunctionalStatelessTestRelease:
	needs: [BuilderDebRelease]
	runs-on: [self-hosted, func-tester]
	steps:
	- name: Set envs
	run: \|
	cat >> "$GITHUB_ENV" << 'EOF'
	TEMP_PATH=${{runner.temp}}/stateless_debug
	REPORTS_PATH=${{runner.temp}}/reports_dir
	CHECK_NAME=Stateless tests (release)
	REPO_COPY=${{runner.temp}}/stateless_debug/ClickHouse
	KILL_TIMEOUT=10800
	EOF
	- name: Download json reports
	uses: actions/download-artifact@v3
	with:
	path: ${{ env.REPORTS_PATH }}
	- name: Check out repository code
	uses: ClickHouse/checkout@v1
	with:
	clear-repository: true
	- name: Functional test
	run: \|
	sudo rm -fr "$TEMP_PATH"
	mkdir -p "$TEMP_PATH"
	cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH"
	cd "$REPO_COPY/tests/ci"
	python3 functional_test_check.py "$CHECK_NAME" "$KILL_TIMEOUT"
	- name: Cleanup
	if: always()
	run: \|
	# shellcheck disable=SC2046
	docker kill $(docker ps -q) \|\|:
	# shellcheck disable=SC2046
	docker rm -f $(docker ps -a -q) \|\|:
	sudo rm -fr "$TEMP_PATH"
	# FunctionalStatelessTestAarch64:
	# needs: [BuilderDebAarch64]
	# runs-on: [self-hosted, func-tester-aarch64]
	# steps:
	# - name: Set envs
	# run: \|
	# cat >> "$GITHUB_ENV" << 'EOF'
	# TEMP_PATH=${{runner.temp}}/stateless_release
	# REPORTS_PATH=${{runner.temp}}/reports_dir
	# CHECK_NAME=Stateless tests (aarch64)
	# REPO_COPY=${{runner.temp}}/stateless_release/ClickHouse
	# KILL_TIMEOUT=10800
	# EOF
	# - name: Download json reports
	# uses: actions/download-artifact@v3
	# with:
	# path: ${{ env.REPORTS_PATH }}
	# - name: Check out repository code
	# uses: ClickHouse/checkout@v1
	# with:
	# clear-repository: true
	# - name: Functional test
	# run: \|
	# sudo rm -fr "$TEMP_PATH"
	# mkdir -p "$TEMP_PATH"
	# cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH"
	# cd "$REPO_COPY/tests/ci"
	# python3 functional_test_check.py "$CHECK_NAME" "$KILL_TIMEOUT"
	# - name: Cleanup
	# if: always()
	# run: \|
	# # shellcheck disable=SC2046
	# docker kill $(docker ps -q) \|\|:
	# # shellcheck disable=SC2046
	# docker rm -f $(docker ps -a -q) \|\|:
	# sudo rm -fr "$TEMP_PATH"
	##############################################################################################
	############################ FUNCTIONAl STATEFUL TESTS #######################################
	##############################################################################################
	FunctionalStatefulTestRelease:
	needs: [BuilderDebRelease]
	runs-on: [self-hosted, func-tester]
	steps:
	- name: Set envs
	run: \|
	cat >> "$GITHUB_ENV" << 'EOF'
	TEMP_PATH=${{runner.temp}}/stateful_debug
	REPORTS_PATH=${{runner.temp}}/reports_dir
	CHECK_NAME=Stateful tests (release)
	REPO_COPY=${{runner.temp}}/stateful_debug/ClickHouse
	KILL_TIMEOUT=3600
	EOF
	- name: Download json reports
	uses: actions/download-artifact@v3
	with:
	path: ${{ env.REPORTS_PATH }}
	- name: Check out repository code
	uses: ClickHouse/checkout@v1
	with:
	clear-repository: true
	- name: Functional test
	run: \|
	sudo rm -fr "$TEMP_PATH"
	mkdir -p "$TEMP_PATH"
	cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH"
	cd "$REPO_COPY/tests/ci"
	python3 functional_test_check.py "$CHECK_NAME" "$KILL_TIMEOUT"
	- name: Cleanup
	if: always()
	run: \|
	# shellcheck disable=SC2046
	docker kill $(docker ps -q) \|\|:
	# shellcheck disable=SC2046
	docker rm -f $(docker ps -a -q) \|\|:
	sudo rm -fr "$TEMP_PATH"
	# FunctionalStatefulTestAarch64:
	# needs: [BuilderDebAarch64]
	# runs-on: [self-hosted, func-tester-aarch64]
	# steps:
	# - name: Set envs
	# run: \|
	# cat >> "$GITHUB_ENV" << 'EOF'
	# TEMP_PATH=${{runner.temp}}/stateful_release
	# REPORTS_PATH=${{runner.temp}}/reports_dir
	# CHECK_NAME=Stateful tests (aarch64)
	# REPO_COPY=${{runner.temp}}/stateful_release/ClickHouse
	# KILL_TIMEOUT=3600
	# EOF
	# - name: Download json reports
	# uses: actions/download-artifact@v3
	# with:
	# path: ${{ env.REPORTS_PATH }}
	# - name: Check out repository code
	# uses: ClickHouse/checkout@v1
	# with:
	# clear-repository: true
	# - name: Functional test
	# run: \|
	# sudo rm -fr "$TEMP_PATH"
	# mkdir -p "$TEMP_PATH"
	# cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH"
	# cd "$REPO_COPY/tests/ci"
	# python3 functional_test_check.py "$CHECK_NAME" "$KILL_TIMEOUT"
	# - name: Cleanup
	# if: always()
	# run: \|
	# # shellcheck disable=SC2046
	# docker kill $(docker ps -q) \|\|:
	# # shellcheck disable=SC2046
	# docker rm -f $(docker ps -a -q) \|\|:
	# sudo rm -fr "$TEMP_PATH"
	#############################################################################################
	############################# INTEGRATION TESTS #############################################
	#############################################################################################
	IntegrationTestsRelease0:
	needs: [BuilderDebRelease]
	runs-on: [self-hosted, stress-tester]
	steps:
	- name: Set envs
	run: \|
	cat >> "$GITHUB_ENV" << 'EOF'
	TEMP_PATH=${{runner.temp}}/integration_tests_release
	REPORTS_PATH=${{runner.temp}}/reports_dir
	CHECK_NAME=Integration tests (release)
	REPO_COPY=${{runner.temp}}/integration_tests_release/ClickHouse
	RUN_BY_HASH_NUM=0
	RUN_BY_HASH_TOTAL=2
	EOF
	- name: Download json reports
	uses: actions/download-artifact@v3
	with:
	path: ${{ env.REPORTS_PATH }}
	- name: Check out repository code
	uses: ClickHouse/checkout@v1
	with:
	clear-repository: true
	- name: Integration test
	run: \|
	sudo rm -fr "$TEMP_PATH"
	mkdir -p "$TEMP_PATH"
	cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH"
	cd "$REPO_COPY/tests/ci"
	python3 integration_test_check.py "$CHECK_NAME"
	- name: Cleanup
	if: always()
	run: \|
	# shellcheck disable=SC2046
	docker kill $(docker ps -q) \|\|:
	# shellcheck disable=SC2046
	docker rm -f $(docker ps -a -q) \|\|:
	sudo rm -fr "$TEMP_PATH"
	IntegrationTestsRelease1:
	needs: [BuilderDebRelease]
	runs-on: [self-hosted, stress-tester]
	steps:
	- name: Set envs
	run: \|
	cat >> "$GITHUB_ENV" << 'EOF'
	TEMP_PATH=${{runner.temp}}/integration_tests_release
	REPORTS_PATH=${{runner.temp}}/reports_dir
	CHECK_NAME=Integration tests (release)
	REPO_COPY=${{runner.temp}}/integration_tests_release/ClickHouse
	RUN_BY_HASH_NUM=1
	RUN_BY_HASH_TOTAL=2
	EOF
	- name: Download json reports
	uses: actions/download-artifact@v3
	with:
	path: ${{ env.REPORTS_PATH }}
	- name: Check out repository code
	uses: ClickHouse/checkout@v1
	with:
	clear-repository: true
	- name: Integration test
	run: \|
	sudo rm -fr "$TEMP_PATH"
	mkdir -p "$TEMP_PATH"
	cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH"
	cd "$REPO_COPY/tests/ci"
	python3 integration_test_check.py "$CHECK_NAME"
	- name: Cleanup
	if: always()
	run: \|
	# shellcheck disable=SC2046
	docker kill $(docker ps -q) \|\|:
	# shellcheck disable=SC2046
	docker rm -f $(docker ps -a -q) \|\|:
	sudo rm -fr "$TEMP_PATH"
	#############################################################################################
	##################################### REGRESSION TESTS ######################################
	#############################################################################################
	regression_start:
	## Not depending on the tests above since they can fail at any given moment.
	needs: [BuilderDebRelease]
	runs-on: ubuntu-latest
	steps:
	- run: true

	regression_common:
	strategy:
	fail-fast: false
	matrix:
	SUITE: [aes_encryption, aggregate_functions, atomic_insert, base_58, clickhouse_keeper, datetime64_extended_range, disk_level_encryption, dns, example, extended_precision_data_types, kafka, kerberos, lightweight_delete, map_type, part_moves_between_shards, rbac, selects, ssl_server, tiered_storage, window_functions]
	needs: [regression_start]
	runs-on: [self-hosted, stress-tester]
	env:
	AWS_ACCESS_KEY_ID: ${{ secrets.AWS_REPORT_KEY_ID }}
	AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_REPORT_SECRET_ACCESS_KEY }}
	AWS_DEFAULT_REGION: ${{ secrets.AWS_REPORT_REGION }}
	steps:
	- name: Checkout regression repo
	uses: actions/checkout@v3
	with:
	repository: Altinity/clickhouse-regression
	ref: releases
	- name: Set envs
	run: \|
	cat >> "$GITHUB_ENV" << 'EOF'
	REPORTS_PATH=${{runner.temp}}/reports_dir
	SUITE=${{ matrix.SUITE }}
	artifacts=public
	EOF
	- name: Download json reports
	uses: actions/download-artifact@v3
	with:
	path: ${{ env.REPORTS_PATH }}
	- name: Setup
	run: .github/setup.sh
	- name: Get deb url
	run: python3 .github/get-deb-url.py --reports-path ${{ env.REPORTS_PATH }} --github-env $GITHUB_ENV
	- name: Run ${{ env.SUITE }} suite
	run: python3
	-u ${{ env.SUITE }}/regression.py
	--clickhouse-binary-path ${{ env.clickhouse_binary_path }}
	--test-to-end
	--local
	--collect-service-logs
	--output classic
	--parallel 1
	--attr project="$GITHUB_REPOSITORY" project.id="$GITHUB_REPOSITORY_ID" package="${{ env.clickhouse_binary_path }}" version="${{ env.version }}" user.name="$GITHUB_ACTOR" repository="https://github.com/Altinity/clickhouse-regression" commit.hash="$(git rev-parse HEAD)" job.id="$GITHUB_RUN_ID" job.url="$GITHUB_SERVER_URL/$GITHUB_REPOSITORY/actions/runs/$GITHUB_RUN_ID" arch="x86_64"
	--log raw.log
	- name: Create and upload logs
	if: always()
	run: .github/create_and_upload_logs.sh 1
	env:
	artifact_s3_dir: build/v${{ env.version }}/$GITHUB_SHA
	- uses: actions/upload-artifact@v3
	if: always()
	with:
	name: ${{ env.SUITE }}-artifacts
	path: \|
	./report.html
	./*.log.txt
	./*.log
	./*.html
	.//_instances/.log
	.//_instances//logs/*.log
	.///_instances//logs/.log
	.///_instances/*.log

	benchmark:
	strategy:
	fail-fast: false
	matrix:
	STORAGE: [minio, aws_s3, gcs]
	needs: [regression_start]
	runs-on: [self-hosted, stress-tester]
	env:
	AWS_ACCESS_KEY_ID: ${{ secrets.AWS_REPORT_KEY_ID }}
	AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_REPORT_SECRET_ACCESS_KEY }}
	AWS_DEFAULT_REGION: ${{ secrets.AWS_REPORT_REGION }}
	steps:
	- name: Checkout regression repo
	uses: actions/checkout@v3
	with:
	repository: Altinity/clickhouse-regression
	- name: Set envs
	run: \|
	cat >> "$GITHUB_ENV" << 'EOF'
	REPORTS_PATH=${{runner.temp}}/reports_dir
	SUITE=ontime_benchmark
	STORAGE=/${{ matrix.STORAGE }}
	artifacts=public
	EOF
	- name: Download json reports
	uses: actions/download-artifact@v3
	with:
	path: ${{ env.REPORTS_PATH }}
	- name: Setup
	run: .github/setup.sh
	- name: Get deb url
	run: python3 .github/get-deb-url.py --reports-path ${{ env.REPORTS_PATH }} --github-env $GITHUB_ENV
	- name: Run ${{ env.SUITE }} suite
	run: python3
	-u ${{ env.SUITE }}/benchmark.py
	--clickhouse-binary-path ${{ env.clickhouse_binary_path }}
	--storage ${{ matrix.STORAGE }}
	--gcs-uri ${{ secrets.REGRESSION_GCS_URI }}
	--gcs-key-id ${{ secrets.REGRESSION_GCS_KEY_ID }}
	--gcs-key-secret ${{ secrets.REGRESSION_GCS_KEY_SECRET }}
	--aws-s3-bucket ${{ secrets.REGRESSION_AWS_S3_BUCKET }}
	--aws-s3-region ${{ secrets.REGRESSION_AWS_S3_REGION }}
	--aws-s3-key-id ${{ secrets.REGRESSION_AWS_S3_KEY_ID }}
	--aws-s3-access-key ${{ secrets.REGRESSION_AWS_S3_SECRET_ACCESS_KEY }}
	--test-to-end
	--local
	--collect-service-logs
	--output classic
	--parallel 1
	--attr project="$GITHUB_REPOSITORY" project.id="$GITHUB_REPOSITORY_ID" package="${{ env.clickhouse_binary_path }}" version="${{ env.version }}" user.name="$GITHUB_ACTOR" repository="https://github.com/Altinity/clickhouse-regression" commit.hash="$(git rev-parse HEAD)" job.id="$GITHUB_RUN_ID" job.url="$GITHUB_SERVER_URL/$GITHUB_REPOSITORY/actions/runs/$GITHUB_RUN_ID" arch="x86_64"
	--log raw.log
	- name: Create and upload logs
	if: always()
	run: .github/create_and_upload_logs.sh 1
	env:
	artifact_s3_dir: build/v${{ env.version }}/$GITHUB_SHA
	- uses: actions/upload-artifact@v3
	if: always()
	with:
	name: ${{ env.SUITE }}-minio-artifacts
	path: \|
	./report.html
	./*.log.txt
	./*.log
	./*.html
	.//_instances/.log
	.//_instances//logs/*.log
	.///_instances//logs/.log
	.///_instances/*.log

	ldap:
	strategy:
	fail-fast: false
	matrix:
	SUITE: [authentication, external_user_directory, role_mapping]
	needs: [regression_start]
	runs-on: [self-hosted, stress-tester]
	env:
	AWS_ACCESS_KEY_ID: ${{ secrets.AWS_REPORT_KEY_ID }}
	AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_REPORT_SECRET_ACCESS_KEY }}
	AWS_DEFAULT_REGION: ${{ secrets.AWS_REPORT_REGION }}
	steps:
	- name: Checkout regression repo
	uses: actions/checkout@v3
	with:
	repository: Altinity/clickhouse-regression
	ref: releases
	- name: Set envs
	run: \|
	cat >> "$GITHUB_ENV" << 'EOF'
	REPORTS_PATH=${{runner.temp}}/reports_dir
	SUITE=ldap/${{ matrix.SUITE }}
	artifacts=public
	EOF
	- name: Download json reports
	uses: actions/download-artifact@v3
	with:
	path: ${{ env.REPORTS_PATH }}
	- name: Setup
	run: .github/setup.sh
	- name: Get deb url
	run: python3 .github/get-deb-url.py --reports-path ${{ env.REPORTS_PATH }} --github-env $GITHUB_ENV
	- name: Run ${{ env.SUITE }} suite
	run: python3
	-u ${{ env.SUITE }}/regression.py
	--clickhouse-binary-path ${{ env.clickhouse_binary_path }}
	--test-to-end
	--local
	--collect-service-logs
	--output classic
	--parallel 1
	--attr project="$GITHUB_REPOSITORY" project.id="$GITHUB_REPOSITORY_ID" package="${{ env.clickhouse_binary_path }}" version="${{ env.version }}" user.name="$GITHUB_ACTOR" repository="https://github.com/Altinity/clickhouse-regression" commit.hash="$(git rev-parse HEAD)" job.id="$GITHUB_RUN_ID" job.url="$GITHUB_SERVER_URL/$GITHUB_REPOSITORY/actions/runs/$GITHUB_RUN_ID" arch="x86_64"
	--log raw.log
	- name: Create and upload logs
	if: always()
	run: .github/create_and_upload_logs.sh 1
	env:
	artifact_s3_dir: build/v${{ env.version }}/$GITHUB_SHA
	- uses: actions/upload-artifact@v3
	if: always()
	with:
	name: ldap-authentication-artifacts
	path: \|
	./report.html
	./*.log.txt
	./*.log
	./*.html
	.//_instances/.log
	.//_instances//logs/*.log
	.///_instances//logs/.log
	.///_instances/*.log

	parquet:
	needs: [regression_start]
	runs-on: [self-hosted, stress-tester]
	env:
	AWS_ACCESS_KEY_ID: ${{ secrets.AWS_REPORT_KEY_ID }}
	AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_REPORT_SECRET_ACCESS_KEY }}
	AWS_DEFAULT_REGION: ${{ secrets.AWS_REPORT_REGION }}
	steps:
	- name: Checkout regression repo
	uses: actions/checkout@v3
	with:
	repository: Altinity/clickhouse-regression
	ref: releases
	- name: Set envs
	run: \|
	cat >> "$GITHUB_ENV" << 'EOF'
	REPORTS_PATH=${{runner.temp}}/reports_dir
	SUITE=parquet
	artifacts=public
	EOF
	- name: Download json reports
	uses: actions/download-artifact@v3
	with:
	path: ${{ env.REPORTS_PATH }}
	- name: Setup
	run: .github/setup.sh
	- name: Get deb url
	run: python3 .github/get-deb-url.py --reports-path ${{ env.REPORTS_PATH }} --github-env $GITHUB_ENV
	- name: Run ${{ env.SUITE }} suite
	run: python3
	-u ${{ env.SUITE }}/regression.py
	--clickhouse-binary-path ${{ env.clickhouse_binary_path }}
	--test-to-end
	--local
	--collect-service-logs
	--output classic
	--parallel 1
	--attr project="$GITHUB_REPOSITORY" project.id="$GITHUB_REPOSITORY_ID" package="${{ env.clickhouse_binary_path }}" version="${{ env.version }}" user.name="$GITHUB_ACTOR" repository="https://github.com/Altinity/clickhouse-regression" commit.hash="$(git rev-parse HEAD)" job.id="$GITHUB_RUN_ID" job.url="$GITHUB_SERVER_URL/$GITHUB_REPOSITORY/actions/runs/$GITHUB_RUN_ID" arch="x86_64"
	--log raw.log
	--storage minio
	--storage aws_s3
	--aws-s3-bucket ${{ secrets.REGRESSION_AWS_S3_BUCKET }}
	--aws-s3-region ${{ secrets.REGRESSION_AWS_S3_REGION }}
	--aws-s3-key-id ${{ secrets.REGRESSION_AWS_S3_KEY_ID }}
	--aws-s3-access-key ${{ secrets.REGRESSION_AWS_S3_SECRET_ACCESS_KEY }}
	--storage gcs
	--gcs-uri ${{ secrets.REGRESSION_GCS_URI }}
	--gcs-key-id ${{ secrets.REGRESSION_GCS_KEY_ID }}
	--gcs-key-secret ${{ secrets.REGRESSION_GCS_KEY_SECRET }}
	- name: Create and upload logs
	if: always()
	run: .github/create_and_upload_logs.sh 1
	env:
	artifact_s3_dir: build/v${{ env.version }}/$GITHUB_SHA
	- uses: actions/upload-artifact@v3
	if: always()
	with:
	name: ${{ env.SUITE }}-artifacts
	path: \|
	./report.html
	./*.log.txt
	./*.log
	./*.html
	.//_instances/.log
	.//_instances//logs/*.log
	.///_instances//logs/.log
	.///_instances/*.log

	s3:
	strategy:
	fail-fast: false
	matrix:
	STORAGE: [minio, aws_s3, gcs]
	needs: [regression_start]
	runs-on: [self-hosted, stress-tester]
	env:
	AWS_ACCESS_KEY_ID: ${{ secrets.AWS_REPORT_KEY_ID }}
	AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_REPORT_SECRET_ACCESS_KEY }}
	AWS_DEFAULT_REGION: ${{ secrets.AWS_REPORT_REGION }}
	steps:
	- name: Checkout regression repo
	uses: actions/checkout@v3
	with:
	repository: Altinity/clickhouse-regression
	ref: releases
	- name: Set envs
	run: \|
	cat >> "$GITHUB_ENV" << 'EOF'
	REPORTS_PATH=${{runner.temp}}/reports_dir
	SUITE=s3
	STORAGE=/${{ matrix.STORAGE }}
	artifacts=public
	EOF
	- name: Download json reports
	uses: actions/download-artifact@v3
	with:
	path: ${{ env.REPORTS_PATH }}
	- name: Setup
	run: .github/setup.sh
	- name: Get deb url
	run: python3 .github/get-deb-url.py --reports-path ${{ env.REPORTS_PATH }} --github-env $GITHUB_ENV
	- name: Run ${{ env.SUITE }} suite
	run: python3
	-u ${{ env.SUITE }}/regression.py
	--clickhouse-binary-path ${{ env.clickhouse_binary_path }}
	--test-to-end
	--local
	--collect-service-logs
	--output classic
	--parallel 1
	--attr project="$GITHUB_REPOSITORY" project.id="$GITHUB_REPOSITORY_ID" package="${{ env.clickhouse_binary_path }}" version="${{ env.version }}" user.name="$GITHUB_ACTOR" repository="https://github.com/Altinity/clickhouse-regression" commit.hash="$(git rev-parse HEAD)" job.id="$GITHUB_RUN_ID" job.url="$GITHUB_SERVER_URL/$GITHUB_REPOSITORY/actions/runs/$GITHUB_RUN_ID" arch="x86_64"
	--log raw.log
	--storage ${{ matrix.STORAGE }}
	--gcs-uri ${{ secrets.REGRESSION_GCS_URI }}
	--gcs-key-id ${{ secrets.REGRESSION_GCS_KEY_ID }}
	--gcs-key-secret ${{ secrets.REGRESSION_GCS_KEY_SECRET }}
	--aws-s3-bucket ${{ secrets.REGRESSION_AWS_S3_BUCKET }}
	--aws-s3-region ${{ secrets.REGRESSION_AWS_S3_REGION }}
	--aws-s3-key-id ${{ secrets.REGRESSION_AWS_S3_KEY_ID }}
	--aws-s3-access-key ${{ secrets.REGRESSION_AWS_S3_SECRET_ACCESS_KEY }}
	- name: Create and upload logs
	if: always()
	run: .github/create_and_upload_logs.sh 1
	env:
	artifact_s3_dir: build/v${{ env.version }}/$GITHUB_SHA
	- uses: actions/upload-artifact@v3
	if: always()
	with:
	name: ${{ env.SUITE }}-${{ matrix.STORAGE }}-artifacts
	path: \|
	./report.html
	./*.log.txt
	./*.log
	./*.html
	.//_instances/.log
	.//_instances//logs/*.log
	.///_instances//logs/.log
	.///_instances/*.log

	tiered_storage_s3:
	strategy:
	fail-fast: false
	matrix:
	STORAGE: [minio, s3amazon, s3gcs]
	needs: [regression_start]
	runs-on: [self-hosted, stress-tester]
	env:
	AWS_ACCESS_KEY_ID: ${{ secrets.AWS_REPORT_KEY_ID }}
	AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_REPORT_SECRET_ACCESS_KEY }}
	AWS_DEFAULT_REGION: ${{ secrets.AWS_REPORT_REGION }}
	steps:
	- name: Checkout regression repo
	uses: actions/checkout@v3
	with:
	repository: Altinity/clickhouse-regression
	ref: releases
	- name: Set envs
	run: \|
	cat >> "$GITHUB_ENV" << 'EOF'
	REPORTS_PATH=${{runner.temp}}/reports_dir
	SUITE=tiered_storage
	STORAGE=/${{ matrix.STORAGE }}
	artifacts=public
	EOF
	- name: Download json reports
	uses: actions/download-artifact@v3
	with:
	path: ${{ env.REPORTS_PATH }}
	- name: Setup
	run: .github/setup.sh
	- name: Get deb url
	run: python3 .github/get-deb-url.py --reports-path ${{ env.REPORTS_PATH }} --github-env $GITHUB_ENV
	- name: Run ${{ env.SUITE }} suite
	run: python3
	-u ${{ env.SUITE }}/regression.py
	--clickhouse-binary-path ${{ env.clickhouse_binary_path }}
	--test-to-end
	--local
	--collect-service-logs
	--output classic
	--parallel 1
	--attr project="$GITHUB_REPOSITORY" project.id="$GITHUB_REPOSITORY_ID" package="${{ env.clickhouse_binary_path }}" version="${{ env.version }}" user.name="$GITHUB_ACTOR" repository="https://github.com/Altinity/clickhouse-regression" commit.hash="$(git rev-parse HEAD)" job.id="$GITHUB_RUN_ID" job.url="$GITHUB_SERVER_URL/$GITHUB_REPOSITORY/actions/runs/$GITHUB_RUN_ID" arch="x86_64"
	--log raw.log
	--aws-s3-access-key ${{ secrets.REGRESSION_AWS_S3_SECRET_ACCESS_KEY }}
	--aws-s3-key-id ${{ secrets.REGRESSION_AWS_S3_KEY_ID }}
	--aws-s3-uri https://s3.${{ secrets.REGRESSION_AWS_S3_REGION}}.amazonaws.com/${{ secrets.REGRESSION_AWS_S3_BUCKET }}/data/
	--gcs-key-id ${{ secrets.REGRESSION_GCS_KEY_ID }}
	--gcs-key-secret ${{ secrets.REGRESSION_GCS_KEY_SECRET }}
	--gcs-uri ${{ secrets.REGRESSION_GCS_URI }}
	--with-${{ matrix.STORAGE }}
	- name: Create and upload logs
	if: always()
	run: .github/create_and_upload_logs.sh 1
	env:
	artifact_s3_dir: build/v${{ env.version }}/$GITHUB_SHA
	- uses: actions/upload-artifact@v3
	if: always()
	with:
	name: ${{ env.SUITE }}-${{ matrix.STORAGE }}-artifacts
	path: \|
	./report.html
	./*.log.txt
	./*.log
	./*.html
	.//_instances/.log
	.//_instances//logs/*.log
	.///_instances//logs/.log
	.///_instances/*.log

	SignRelease:
	needs: [BuilderDebRelease]
	runs-on: [ self-hosted ]
	steps:
	- name: Set envs
	run: \|
	cat >> "$GITHUB_ENV" << 'EOF'
	TEMP_PATH=${{runner.temp}}/signed
	REPORTS_PATH=${{runner.temp}}/reports_dir
	EOF
	- name: Clear repository
	run: \|
	sudo rm -fr "$GITHUB_WORKSPACE" && mkdir "$GITHUB_WORKSPACE"
	- name: Check out repository code
	uses: actions/checkout@v2
	- name: Download json reports
	uses: actions/download-artifact@v2
	with:
	path: ${{ env.REPORTS_PATH }}
	- name: Sign release
	env:
	GPG_BINARY_SIGNING_KEY: ${{ secrets.GPG_BINARY_SIGNING_KEY }}
	GPG_BINARY_SIGNING_PASSPHRASE: ${{ secrets.GPG_BINARY_SIGNING_PASSPHRASE }}
	REPORTS_PATH: ${{ env.REPORTS_PATH }}
	run: \|
	cd "$GITHUB_WORKSPACE/tests/ci"
	python3 sign_release.py
	- name: Upload signed hashes
	uses: actions/upload-artifact@v2
	with:
	name: signed-hashes
	path: ${{ env.TEMP_PATH }}/*.gpg
	- name: Cleanup
	if: always()
	run: \|
	docker ps --quiet \| xargs --no-run-if-empty docker kill \|\|:
	docker ps --all --quiet \| xargs --no-run-if-empty docker rm -f \|\|:
	sudo rm -fr "$TEMP_PATH"
	###########################################################################################
	################################ FINISH CHECK #############################################
	###########################################################################################
	FinishCheck:
	needs:
	- DockerHubPush
	- DockerServerImages
	- BuilderReport
	# - BuilderSpecialReport
	- MarkReleaseReady
	- FunctionalStatelessTestRelease
	# - FunctionalStatelessTestAarch64
	- FunctionalStatefulTestRelease
	# - FunctionalStatefulTestAarch64
	- IntegrationTestsRelease0
	- IntegrationTestsRelease1
	- CompatibilityCheck
	- SignRelease
	- regression_common
	- benchmark
	- ldap
	- parquet
	- s3
	- tiered_storage_s3
	runs-on: [self-hosted, style-checker]
	steps:
	- name: Check out repository code
	uses: ClickHouse/checkout@v1
	with:
	clear-repository: true
	- name: Finish label
	run: \|
	cd "$GITHUB_WORKSPACE/tests/ci"
	python3 finish_check.py

Provide feedback

Saved searches

Use saved searches to filter your results more quickly

22.8 Backport of #54370 - fix parquet array inconsistent offsets #724

Workflow file

22.8 Backport of #54370 - fix parquet array inconsistent offsets #724

Jobs

Run details

Workflow file for this run