From 23dd03d1f351961f7bc4ac78cd37cf6658a2e69a Mon Sep 17 00:00:00 2001 From: MyroTk <44327070+MyroTk@users.noreply.github.com> Date: Thu, 28 Mar 2024 08:57:48 -0700 Subject: [PATCH 001/123] Update release_branches.yml --- .github/workflows/release_branches.yml | 1341 ++++++++---------------- 1 file changed, 428 insertions(+), 913 deletions(-) diff --git a/.github/workflows/release_branches.yml b/.github/workflows/release_branches.yml index 5bff177d13c5..435f00e0abf9 100644 --- a/.github/workflows/release_branches.yml +++ b/.github/workflows/release_branches.yml @@ -9,9 +9,6 @@ env: AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }} DOCKER_USERNAME: ${{ secrets.DOCKER_USERNAME }} DOCKER_PASSWORD: ${{ secrets.DOCKER_PASSWORD }} - REGRESSION_RESULTS_URL: altinity-build-artifacts/${{github.event.number}}/$GITHUB_SHA - REGRESSION_ARM_COMMIT: 19e8624c5e4ccc65b128d27b19836c0570e53991 - on: # yamllint disable-line rule:truthy pull_request: @@ -32,7 +29,7 @@ on: # yamllint disable-line rule:truthy jobs: DockerHubPushAarch64: - runs-on: [self-hosted, style-checker, on-demand, type-cax41, in-hel1, image-arm-app-docker-ce] + runs-on: [self-hosted, altinity-on-demand, altinity-type-cax41, altinity-in-hel1, altinity-image-arm-app-docker-ce] steps: - name: Check out repository code uses: Altinity/checkout@19599efdf36c4f3f30eb55d5bb388896faea69f6 @@ -43,10 +40,6 @@ jobs: run: | cd "$GITHUB_WORKSPACE/tests/ci" python3 docker_images_check.py --suffix aarch64 - env: - RUNNER_IP: "$(hostname -I | cut -d ' ' -f 1)" - RUNNER_SSH_COMMAND: "ssh -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null root@$(hostname -I | cut -d ' ' -f 1)" - - name: Upload images files to artifacts uses: actions/upload-artifact@v3 with: @@ -54,7 +47,7 @@ jobs: path: ${{ runner.temp }}/docker_images_check/changed_images_aarch64.json DockerHubPushAmd64: - runs-on: [self-hosted, style-checker, on-demand, type-cpx51, image-x86-app-docker-ce] + runs-on: [self-hosted, altinity-on-demand, altinity-type-cpx51, altinity-in-ash, altinity-image-x86-app-docker-ce] steps: - name: Check out repository code uses: Altinity/checkout@19599efdf36c4f3f30eb55d5bb388896faea69f6 @@ -65,10 +58,6 @@ jobs: run: | cd "$GITHUB_WORKSPACE/tests/ci" python3 docker_images_check.py --suffix amd64 - env: - RUNNER_IP: "$(hostname -I | cut -d ' ' -f 1)" - RUNNER_SSH_COMMAND: "ssh -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null root@$(hostname -I | cut -d ' ' -f 1)" - - name: Upload images files to artifacts uses: actions/upload-artifact@v3 with: @@ -77,7 +66,7 @@ jobs: DockerHubPush: needs: [DockerHubPushAmd64, DockerHubPushAarch64] - runs-on: [self-hosted, style-checker, on-demand, type-cpx41, image-x86-app-docker-ce] + runs-on: [self-hosted, altinity-on-demand, altinity-type-cpx41, altinity-image-x86-app-docker-ce] steps: - name: Check out repository code uses: Altinity/checkout@19599efdf36c4f3f30eb55d5bb388896faea69f6 @@ -115,18 +104,29 @@ jobs: name: changed_images path: ${{ runner.temp }}/changed_images.json - CompatibilityCheck: + CompatibilityCheckX86: needs: [BuilderDebRelease] uses: ./.github/workflows/reusable_test.yml secrets: inherit with: test_name: Compatibility check X86 - runner_type: style-checker, on-demand, type-cpx41, image-x86-app-docker-ce + runner_type: altinity-on-demand, altinity-type-cpx41, altinity-image-x86-app-docker-ce timeout_minutes: 180 run_command: | cd "$REPO_COPY/tests/ci" python3 compatibility_check.py --check-name "Compatibility check (amd64)" --check-glibc --check-distributions + CompatibilityCheckAarch64: + needs: [BuilderDebAarch64] + uses: ./.github/workflows/reusable_test.yml + secrets: inherit + with: + test_name: Compatibility check Aarch64 + runner_type: altinity-on-demand, altinity-type-cax41, altinity-image-arm-app-docker-ce + run_command: | + cd "$REPO_COPY/tests/ci" + python3 compatibility_check.py --check-name "Compatibility check (aarch64)" --check-glibc + ######################################################################################### #################################### ORDINARY BUILDS #################################### ######################################################################################### @@ -138,7 +138,7 @@ jobs: build_name: package_release checkout_depth: 0 timeout_minutes: 180 - runner_type: builder, on-demand, type-ccx53, image-x86-app-docker-ce + runner_type: altinity-on-demand, altinity-type-ccx53, altinity-in-ash, altinity-image-x86-app-docker-ce additional_envs: | CLICKHOUSE_STABLE_VERSION_SUFFIX=altinitystable @@ -149,8 +149,70 @@ jobs: with: build_name: package_aarch64 checkout_depth: 0 - runner_type: builder, on-demand, type-ccx53, image-x86-app-docker-ce + timeout_minutes: 180 + runner_type: altinity-on-demand, altinity-type-ccx53, altinity-in-ash, altinity-image-x86-app-docker-ce + additional_envs: | + CLICKHOUSE_STABLE_VERSION_SUFFIX=altinitystable + + BuilderDebAsan: + needs: [DockerHubPush] + uses: ./.github/workflows/reusable_build.yml + secrets: inherit + with: + build_name: package_asan + checkout_depth: 0 + timeout_minutes: 180 + runner_type: altinity-on-demand, altinity-type-ccx53, altinity-in-ash, altinity-image-x86-app-docker-ce + additional_envs: | + CLICKHOUSE_STABLE_VERSION_SUFFIX=altinitystable + BuilderDebUBsan: + needs: [DockerHubPush] + uses: ./.github/workflows/reusable_build.yml + secrets: inherit + with: + build_name: package_ubsan + checkout_depth: 0 + timeout_minutes: 180 + runner_type: altinity-on-demand, altinity-type-ccx53, altinity-in-ash, altinity-image-x86-app-docker-ce + additional_envs: | + CLICKHOUSE_STABLE_VERSION_SUFFIX=altinitystable + + BuilderDebTsan: + needs: [DockerHubPush] + uses: ./.github/workflows/reusable_build.yml + secrets: inherit + with: + build_name: package_tsan + checkout_depth: 0 + timeout_minutes: 180 + runner_type: altinity-on-demand, altinity-type-ccx53, altinity-in-ash, altinity-image-x86-app-docker-ce + additional_envs: | + CLICKHOUSE_STABLE_VERSION_SUFFIX=altinitystable + + BuilderDebMsan: + needs: [DockerHubPush] + uses: ./.github/workflows/reusable_build.yml + secrets: inherit + with: + build_name: package_msan + checkout_depth: 0 + timeout_minutes: 180 + runner_type: altinity-on-demand, altinity-type-ccx53, altinity-in-ash, altinity-image-x86-app-docker-ce + additional_envs: | + CLICKHOUSE_STABLE_VERSION_SUFFIX=altinitystable + + BuilderDebDebug: + needs: [DockerHubPush] + uses: ./.github/workflows/reusable_build.yml + secrets: inherit + with: + build_name: package_debug + checkout_depth: 0 + timeout_minutes: 180 + runner_type: altinity-on-demand, altinity-type-ccx53, altinity-in-ash, altinity-image-x86-app-docker-ce + additional_envs: | + CLICKHOUSE_STABLE_VERSION_SUFFIX=altinitystable ############################################################################################ ##################################### Docker images ####################################### @@ -159,7 +221,7 @@ jobs: needs: - BuilderDebRelease - BuilderDebAarch64 - runs-on: [self-hosted, style-checker, on-demand, type-cpx51, image-x86-app-docker-ce] + runs-on: [self-hosted, altinity-on-demand, altinity-type-cpx51, altinity-image-x86-app-docker-ce] timeout-minutes: 180 steps: - name: Check out repository code @@ -171,9 +233,10 @@ jobs: - name: Check docker altinityinfra/clickhouse-server building run: | cd "$GITHUB_WORKSPACE/tests/ci" - python3 docker_server.py --release-type head --no-push \ + docker buildx create --use + python3 docker_server.py --release-type head \ --image-repo altinityinfra/clickhouse-server --image-path docker/server - python3 docker_server.py --release-type head --no-push \ + python3 docker_server.py --release-type head \ --image-repo altinityinfra/clickhouse-keeper --image-path docker/keeper - name: Cleanup if: always() @@ -194,7 +257,7 @@ jobs: secrets: inherit with: test_name: ClickHouse build check - runner_type: style-checker, on-demand, type-cpx31, image-x86-app-docker-ce + runner_type: altinity-on-demand, altinity-type-cpx41, altinity-image-x86-app-docker-ce timeout_minutes: 180 additional_envs: | NEEDS_DATA<> "$GITHUB_ENV" << 'EOF' - REPORTS_PATH=${{runner.temp}}/reports_dir - SUITE=${{ matrix.SUITE }} - artifacts=public - EOF - - name: Download json reports - uses: actions/download-artifact@v3 - with: - path: ${{ env.REPORTS_PATH }} - - name: Setup - run: .github/setup.sh - - name: Get deb url - run: python3 .github/get-deb-url.py --reports-path ${{ env.REPORTS_PATH }} --github-env $GITHUB_ENV - - name: Run ${{ env.SUITE }} suite - run: python3 - -u ${{ env.SUITE }}/regression.py - --clickhouse-binary-path ${{ env.clickhouse_binary_path }} - --test-to-end - --local - --collect-service-logs - --output classic - --parallel 1 - --attr project="$GITHUB_REPOSITORY" project.id="$GITHUB_REPOSITORY_ID" package="${{ env.clickhouse_binary_path }}" version="${{ env.version }}" user.name="$GITHUB_ACTOR" repository="https://github.com/Altinity/clickhouse-regression" commit.hash="$(git rev-parse HEAD)" job.id="$GITHUB_RUN_ID" job.url="$GITHUB_SERVER_URL/$GITHUB_REPOSITORY/actions/runs/$GITHUB_RUN_ID" arch="$(uname -i)" - --log raw.log - - name: Create and upload logs - if: always() - run: .github/create_and_upload_logs.sh 1 - env: - artifact_s3_dir: build/v${{ env.version }}/$GITHUB_SHA - - uses: actions/upload-artifact@v3 - if: always() - with: - name: ${{ env.SUITE }}-amd64-artifacts - path: | - ./report.html - ./*.log.txt - ./*.log - ./*.html - ./*/_instances/*.log - ./*/_instances/*/logs/*.log - ./*/*/_instances/*/logs/*.log - ./*/*/_instances/*.log - - RegressionCommonAarch64: - strategy: - fail-fast: false - matrix: - SUITE: [aes_encryption, aggregate_functions, atomic_insert, base_58, clickhouse_keeper, data_types, datetime64_extended_range, disk_level_encryption, dns, engines, example, extended_precision_data_types, kafka, kerberos, key_value, lightweight_delete, part_moves_between_shards, rbac, selects, session_timezone, ssl_server, tiered_storage, window_functions] - needs: [RegressionStart] - runs-on: [self-hosted, regression-tester, on-demand, type-cax41, image-arm-app-docker-ce] - timeout-minutes: 180 - env: - AWS_ACCESS_KEY_ID: ${{ secrets.AWS_REPORT_KEY_ID }} - AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_REPORT_SECRET_ACCESS_KEY }} - AWS_DEFAULT_REGION: ${{ secrets.AWS_REPORT_REGION }} - steps: - - name: Checkout regression repo - uses: actions/checkout@v3 - with: - repository: Altinity/clickhouse-regression - ref: ${{ env.REGRESSION_ARM_COMMIT }} - - name: Set envs - run: | - cat >> "$GITHUB_ENV" << 'EOF' - REPORTS_PATH=${{runner.temp}}/reports_dir - SUITE=${{ matrix.SUITE }} - artifacts=public - EOF - - name: Download json reports - uses: actions/download-artifact@v3 - with: - path: ${{ env.REPORTS_PATH }} - - name: Setup - run: .github/setup.sh - - name: Get deb url - run: python3 .github/get-deb-url.py --reports-path ${{ env.REPORTS_PATH }} --github-env $GITHUB_ENV - - name: Run ${{ env.SUITE }} suite - run: python3 - -u ${{ env.SUITE }}/regression.py - --clickhouse-binary-path ${{ env.clickhouse_binary_path }} - --test-to-end - --local - --collect-service-logs - --output classic - --parallel 1 - --attr project="$GITHUB_REPOSITORY" project.id="$GITHUB_REPOSITORY_ID" package="${{ env.clickhouse_binary_path }}" version="${{ env.version }}" user.name="$GITHUB_ACTOR" repository="https://github.com/Altinity/clickhouse-regression" commit.hash="$(git rev-parse HEAD)" job.id="$GITHUB_RUN_ID" job.url="$GITHUB_SERVER_URL/$GITHUB_REPOSITORY/actions/runs/$GITHUB_RUN_ID" arch="$(uname -i)" - --log raw.log - - name: Create and upload logs - if: always() - run: .github/create_and_upload_logs.sh 1 - env: - artifact_s3_dir: build/v${{ env.version }}/$GITHUB_SHA - - uses: actions/upload-artifact@v3 - if: always() - with: - name: ${{ env.SUITE }}-aarch64-artifacts - path: | - ./report.html - ./*.log.txt - ./*.log - ./*.html - ./*/_instances/*.log - ./*/_instances/*/logs/*.log - ./*/*/_instances/*/logs/*.log - ./*/*/_instances/*.log - - RegressionBenchmarkAmd64: - strategy: - fail-fast: false - matrix: - STORAGE: [minio, aws_s3, gcs] - needs: [RegressionStart] - runs-on: [self-hosted, regression-tester, on-demand, type-cpx51, image-x86-app-docker-ce] - timeout-minutes: 180 - env: - AWS_ACCESS_KEY_ID: ${{ secrets.AWS_REPORT_KEY_ID }} - AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_REPORT_SECRET_ACCESS_KEY }} - AWS_DEFAULT_REGION: ${{ secrets.AWS_REPORT_REGION }} - steps: - - name: Checkout regression repo - uses: actions/checkout@v3 - with: - repository: Altinity/clickhouse-regression - - name: Set envs - run: | - cat >> "$GITHUB_ENV" << 'EOF' - REPORTS_PATH=${{runner.temp}}/reports_dir - SUITE=ontime_benchmark - STORAGE=/${{ matrix.STORAGE }} - artifacts=public - EOF - - name: Download json reports - uses: actions/download-artifact@v3 - with: - path: ${{ env.REPORTS_PATH }} - - name: Setup - run: .github/setup.sh - - name: Get deb url - run: python3 .github/get-deb-url.py --reports-path ${{ env.REPORTS_PATH }} --github-env $GITHUB_ENV - - name: Run ${{ env.SUITE }} suite - run: python3 - -u ${{ env.SUITE }}/benchmark.py - --clickhouse-binary-path ${{ env.clickhouse_binary_path }} - --storage ${{ matrix.STORAGE }} - --gcs-uri ${{ secrets.REGRESSION_GCS_URI }} - --gcs-key-id ${{ secrets.REGRESSION_GCS_KEY_ID }} - --gcs-key-secret ${{ secrets.REGRESSION_GCS_KEY_SECRET }} - --aws-s3-bucket ${{ secrets.REGRESSION_AWS_S3_BUCKET }} - --aws-s3-region ${{ secrets.REGRESSION_AWS_S3_REGION }} - --aws-s3-key-id ${{ secrets.REGRESSION_AWS_S3_KEY_ID }} - --aws-s3-access-key ${{ secrets.REGRESSION_AWS_S3_SECRET_ACCESS_KEY }} - --test-to-end - --local - --collect-service-logs - --output classic - --parallel 1 - --attr project="$GITHUB_REPOSITORY" project.id="$GITHUB_REPOSITORY_ID" package="${{ env.clickhouse_binary_path }}" version="${{ env.version }}" user.name="$GITHUB_ACTOR" repository="https://github.com/Altinity/clickhouse-regression" commit.hash="$(git rev-parse HEAD)" job.id="$GITHUB_RUN_ID" job.url="$GITHUB_SERVER_URL/$GITHUB_REPOSITORY/actions/runs/$GITHUB_RUN_ID" arch="$(uname -i)" - --log raw.log - - name: Create and upload logs - if: always() - run: .github/create_and_upload_logs.sh 1 - env: - artifact_s3_dir: build/v${{ env.version }}/$GITHUB_SHA - - uses: actions/upload-artifact@v3 - if: always() - with: - name: benchmark-${{ matrix.STORAGE }}-amd64-artifacts - path: | - ./report.html - ./*.log.txt - ./*.log - ./*.html - ./*/_instances/*.log - ./*/_instances/*/logs/*.log - ./*/*/_instances/*/logs/*.log - ./*/*/_instances/*.log - - RegressionBenchmarkAarch64: - strategy: - fail-fast: false - matrix: - STORAGE: [minio, aws_s3, gcs] - needs: [RegressionStart] - runs-on: [self-hosted, regression-tester, on-demand, type-cax41, image-arm-app-docker-ce] - timeout-minutes: 180 - env: - AWS_ACCESS_KEY_ID: ${{ secrets.AWS_REPORT_KEY_ID }} - AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_REPORT_SECRET_ACCESS_KEY }} - AWS_DEFAULT_REGION: ${{ secrets.AWS_REPORT_REGION }} - steps: - - name: Checkout regression repo - uses: actions/checkout@v3 - with: - repository: Altinity/clickhouse-regression - ref: ${{ env.REGRESSION_ARM_COMMIT }} - - name: Set envs - run: | - cat >> "$GITHUB_ENV" << 'EOF' - REPORTS_PATH=${{runner.temp}}/reports_dir - SUITE=ontime_benchmark - STORAGE=/${{ matrix.STORAGE }} - artifacts=public - EOF - - name: Download json reports - uses: actions/download-artifact@v3 - with: - path: ${{ env.REPORTS_PATH }} - - name: Setup - run: .github/setup.sh - - name: Get deb url - run: python3 .github/get-deb-url.py --reports-path ${{ env.REPORTS_PATH }} --github-env $GITHUB_ENV - - name: Run ${{ env.SUITE }} suite - run: python3 - -u ${{ env.SUITE }}/benchmark.py - --clickhouse-binary-path ${{ env.clickhouse_binary_path }} - --storage ${{ matrix.STORAGE }} - --gcs-uri ${{ secrets.REGRESSION_GCS_URI }} - --gcs-key-id ${{ secrets.REGRESSION_GCS_KEY_ID }} - --gcs-key-secret ${{ secrets.REGRESSION_GCS_KEY_SECRET }} - --aws-s3-bucket ${{ secrets.REGRESSION_AWS_S3_BUCKET }} - --aws-s3-region ${{ secrets.REGRESSION_AWS_S3_REGION }} - --aws-s3-key-id ${{ secrets.REGRESSION_AWS_S3_KEY_ID }} - --aws-s3-access-key ${{ secrets.REGRESSION_AWS_S3_SECRET_ACCESS_KEY }} - --test-to-end - --local - --collect-service-logs - --output classic - --parallel 1 - --attr project="$GITHUB_REPOSITORY" project.id="$GITHUB_REPOSITORY_ID" package="${{ env.clickhouse_binary_path }}" version="${{ env.version }}" user.name="$GITHUB_ACTOR" repository="https://github.com/Altinity/clickhouse-regression" commit.hash="$(git rev-parse HEAD)" job.id="$GITHUB_RUN_ID" job.url="$GITHUB_SERVER_URL/$GITHUB_REPOSITORY/actions/runs/$GITHUB_RUN_ID" arch="$(uname -i)" - --log raw.log - - name: Create and upload logs - if: always() - run: .github/create_and_upload_logs.sh 1 - env: - artifact_s3_dir: build/v${{ env.version }}/$GITHUB_SHA - - uses: actions/upload-artifact@v3 - if: always() - with: - name: benchmark-${{ matrix.STORAGE }}-aarch64-artifacts - path: | - ./report.html - ./*.log.txt - ./*.log - ./*.html - ./*/_instances/*.log - ./*/_instances/*/logs/*.log - ./*/*/_instances/*/logs/*.log - ./*/*/_instances/*.log - - RegressionClickHouseKeeperSSLAmd64: - needs: [RegressionStart] - runs-on: [self-hosted, regression-tester, on-demand, type-cpx51, image-x86-app-docker-ce] - env: - AWS_ACCESS_KEY_ID: ${{ secrets.AWS_REPORT_KEY_ID }} - AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_REPORT_SECRET_ACCESS_KEY }} - AWS_DEFAULT_REGION: ${{ secrets.AWS_REPORT_REGION }} - steps: - - name: Checkout regression repo - uses: actions/checkout@v3 - with: - repository: Altinity/clickhouse-regression - ref: ${{ env.REGRESSION_COMMON_COMMIT }} - - name: Set envs - run: | - cat >> "$GITHUB_ENV" << 'EOF' - REPORTS_PATH=${{runner.temp}}/reports_dir - SUITE=clickhouse_keeper - STORAGE=/ssl - artifacts=builds - EOF - - name: Download json reports - uses: actions/download-artifact@v3 - with: - path: ${{ env.REPORTS_PATH }} - - name: Setup - run: .github/setup.sh - - name: Get deb url - run: python3 .github/get-deb-url.py --reports-path ${{ env.REPORTS_PATH }} --github-env $GITHUB_ENV - - name: Run ${{ env.SUITE }} suite - run: python3 - -u ${{ env.SUITE }}/regression.py - --ssl - --clickhouse-binary-path ${{ env.clickhouse_binary_path }} - --test-to-end - --local - --collect-service-logs - --output classic - --parallel 1 - --attr project="${GITHUB_REPOSITORY}" project.id="${GITHUB_REPOSITORY_ID}" package="${{ env.clickhouse_binary_path }}" version="${{ env.version }}" user.name="${GITHUB_ACTOR}" repository="https://github.com/Altinity/clickhouse-regression" commit.hash="$(git rev-parse HEAD)" job.id="${GITHUB_RUN_ID}" job.url="${GITHUB_SERVER_URL}/${GITHUB_REPOSITORY}/actions/runs/${GITHUB_RUN_ID}" arch="$(uname -i)" - --log raw.log - - name: Create and upload logs - if: always() - run: .github/create_and_upload_logs.sh 1 - - uses: actions/upload-artifact@v3 - if: always() - with: - name: ${{ env.SUITE }}-ssl-amd64-artifacts - path: | - ./report.html - ./*.log.txt - ./*.log - ./*.html - ./*/_instances/*.log - ./*/_instances/*/logs/*.log - ./*/*/_instances/*/logs/*.log - ./*/*/_instances/*.log - - RegressionClickHouseKeeperSSLAarch64: - needs: [RegressionStart] - runs-on: [self-hosted, regression-tester, on-demand, type-cax41, image-arm-app-docker-ce] - env: - AWS_ACCESS_KEY_ID: ${{ secrets.AWS_REPORT_KEY_ID }} - AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_REPORT_SECRET_ACCESS_KEY }} - AWS_DEFAULT_REGION: ${{ secrets.AWS_REPORT_REGION }} - steps: - - name: Checkout regression repo - uses: actions/checkout@v3 - with: - repository: Altinity/clickhouse-regression - ref: ${{ env.REGRESSION_COMMON_COMMIT }} - - name: Set envs - run: | - cat >> "$GITHUB_ENV" << 'EOF' - REPORTS_PATH=${{runner.temp}}/reports_dir - SUITE=clickhouse_keeper - STORAGE=/ssl - artifacts=builds - EOF - - name: Download json reports - uses: actions/download-artifact@v3 - with: - path: ${{ env.REPORTS_PATH }} - - name: Setup - run: .github/setup.sh - - name: Get deb url - run: python3 .github/get-deb-url.py --reports-path ${{ env.REPORTS_PATH }} --github-env $GITHUB_ENV - - name: Run ${{ env.SUITE }} suite - run: python3 - -u ${{ env.SUITE }}/regression.py - --ssl - --clickhouse-binary-path ${{ env.clickhouse_binary_path }} - --test-to-end - --local - --collect-service-logs - --output classic - --parallel 1 - --attr project="${GITHUB_REPOSITORY}" project.id="${GITHUB_REPOSITORY_ID}" package="${{ env.clickhouse_binary_path }}" version="${{ env.version }}" user.name="${GITHUB_ACTOR}" repository="https://github.com/Altinity/clickhouse-regression" commit.hash="$(git rev-parse HEAD)" job.id="${GITHUB_RUN_ID}" job.url="${GITHUB_SERVER_URL}/${GITHUB_REPOSITORY}/actions/runs/${GITHUB_RUN_ID}" arch="$(uname -i)" - --log raw.log - - name: Create and upload logs - if: always() - run: .github/create_and_upload_logs.sh 1 - - uses: actions/upload-artifact@v3 - if: always() - with: - name: ${{ env.SUITE }}-ssl-aarch64-artifacts - path: | - ./report.html - ./*.log.txt - ./*.log - ./*.html - ./*/_instances/*.log - ./*/_instances/*/logs/*.log - ./*/*/_instances/*/logs/*.log - ./*/*/_instances/*.log - - RegressionLDAPAmd64: - strategy: - fail-fast: false - matrix: - SUITE: [authentication, external_user_directory, role_mapping] - needs: [RegressionStart] - runs-on: [self-hosted, regression-tester, on-demand, type-cpx51, image-x86-app-docker-ce] - timeout-minutes: 180 - env: - AWS_ACCESS_KEY_ID: ${{ secrets.AWS_REPORT_KEY_ID }} - AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_REPORT_SECRET_ACCESS_KEY }} - AWS_DEFAULT_REGION: ${{ secrets.AWS_REPORT_REGION }} - steps: - - name: Checkout regression repo - uses: actions/checkout@v3 - with: - repository: Altinity/clickhouse-regression - - name: Set envs - run: | - cat >> "$GITHUB_ENV" << 'EOF' - REPORTS_PATH=${{runner.temp}}/reports_dir - SUITE=ldap/${{ matrix.SUITE }} - artifacts=public - EOF - - name: Download json reports - uses: actions/download-artifact@v3 - with: - path: ${{ env.REPORTS_PATH }} - - name: Setup - run: .github/setup.sh - - name: Get deb url - run: python3 .github/get-deb-url.py --reports-path ${{ env.REPORTS_PATH }} --github-env $GITHUB_ENV - - name: Run ${{ env.SUITE }} suite - run: python3 - -u ${{ env.SUITE }}/regression.py - --clickhouse-binary-path ${{ env.clickhouse_binary_path }} - --test-to-end - --local - --collect-service-logs - --output classic - --parallel 1 - --attr project="$GITHUB_REPOSITORY" project.id="$GITHUB_REPOSITORY_ID" package="${{ env.clickhouse_binary_path }}" version="${{ env.version }}" user.name="$GITHUB_ACTOR" repository="https://github.com/Altinity/clickhouse-regression" commit.hash="$(git rev-parse HEAD)" job.id="$GITHUB_RUN_ID" job.url="$GITHUB_SERVER_URL/$GITHUB_REPOSITORY/actions/runs/$GITHUB_RUN_ID" arch="$(uname -i)" - --log raw.log - - name: Create and upload logs - if: always() - run: .github/create_and_upload_logs.sh 1 - env: - artifact_s3_dir: build/v${{ env.version }}/$GITHUB_SHA - - uses: actions/upload-artifact@v3 - if: always() - with: - name: ldap-${{ matrix.SUITE }}-amd64-artifacts - path: | - ./report.html - ./*.log.txt - ./*.log - ./*.html - ./*/_instances/*.log - ./*/_instances/*/logs/*.log - ./*/*/_instances/*/logs/*.log - ./*/*/_instances/*.log - - RegressionLDAPAarch64: - strategy: - fail-fast: false - matrix: - SUITE: [authentication, external_user_directory, role_mapping] - needs: [RegressionStart] - runs-on: [self-hosted, regression-tester, on-demand, type-cax41, image-arm-app-docker-ce] - timeout-minutes: 180 - env: - AWS_ACCESS_KEY_ID: ${{ secrets.AWS_REPORT_KEY_ID }} - AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_REPORT_SECRET_ACCESS_KEY }} - AWS_DEFAULT_REGION: ${{ secrets.AWS_REPORT_REGION }} - steps: - - name: Checkout regression repo - uses: actions/checkout@v3 - with: - repository: Altinity/clickhouse-regression - ref: ${{ env.REGRESSION_ARM_COMMIT }} - - name: Set envs - run: | - cat >> "$GITHUB_ENV" << 'EOF' - REPORTS_PATH=${{runner.temp}}/reports_dir - SUITE=ldap/${{ matrix.SUITE }} - artifacts=public - EOF - - name: Download json reports - uses: actions/download-artifact@v3 - with: - path: ${{ env.REPORTS_PATH }} - - name: Setup - run: .github/setup.sh - - name: Get deb url - run: python3 .github/get-deb-url.py --reports-path ${{ env.REPORTS_PATH }} --github-env $GITHUB_ENV - - name: Run ${{ env.SUITE }} suite - run: python3 - -u ${{ env.SUITE }}/regression.py - --clickhouse-binary-path ${{ env.clickhouse_binary_path }} - --test-to-end - --local - --collect-service-logs - --output classic - --parallel 1 - --attr project="$GITHUB_REPOSITORY" project.id="$GITHUB_REPOSITORY_ID" package="${{ env.clickhouse_binary_path }}" version="${{ env.version }}" user.name="$GITHUB_ACTOR" repository="https://github.com/Altinity/clickhouse-regression" commit.hash="$(git rev-parse HEAD)" job.id="$GITHUB_RUN_ID" job.url="$GITHUB_SERVER_URL/$GITHUB_REPOSITORY/actions/runs/$GITHUB_RUN_ID" arch="$(uname -i)" - --log raw.log - - name: Create and upload logs - if: always() - run: .github/create_and_upload_logs.sh 1 - env: - artifact_s3_dir: build/v${{ env.version }}/$GITHUB_SHA - - uses: actions/upload-artifact@v3 - if: always() - with: - name: ldap-${{ matrix.SUITE }}-aarch64-artifacts - path: | - ./report.html - ./*.log.txt - ./*.log - ./*.html - ./*/_instances/*.log - ./*/_instances/*/logs/*.log - ./*/*/_instances/*/logs/*.log - ./*/*/_instances/*.log - - RegressionParquetAmd64: - needs: [RegressionStart] - runs-on: [self-hosted, regression-tester, on-demand, type-cpx51, image-x86-app-docker-ce] - timeout-minutes: 180 - env: - AWS_ACCESS_KEY_ID: ${{ secrets.AWS_REPORT_KEY_ID }} - AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_REPORT_SECRET_ACCESS_KEY }} - AWS_DEFAULT_REGION: ${{ secrets.AWS_REPORT_REGION }} - steps: - - name: Checkout regression repo - uses: actions/checkout@v3 - with: - repository: Altinity/clickhouse-regression - - name: Set envs - run: | - cat >> "$GITHUB_ENV" << 'EOF' - REPORTS_PATH=${{runner.temp}}/reports_dir - SUITE=parquet - artifacts=public - EOF - - name: Download json reports - uses: actions/download-artifact@v3 - with: - path: ${{ env.REPORTS_PATH }} - - name: Setup - run: .github/setup.sh - - name: Get deb url - run: python3 .github/get-deb-url.py --reports-path ${{ env.REPORTS_PATH }} --github-env $GITHUB_ENV - - name: Run ${{ env.SUITE }} suite - run: python3 - -u ${{ env.SUITE }}/regression.py - --clickhouse-binary-path ${{ env.clickhouse_binary_path }} - --test-to-end - --local - --collect-service-logs - --output classic - --parallel 1 - --attr project="$GITHUB_REPOSITORY" project.id="$GITHUB_REPOSITORY_ID" package="${{ env.clickhouse_binary_path }}" version="${{ env.version }}" user.name="$GITHUB_ACTOR" repository="https://github.com/Altinity/clickhouse-regression" commit.hash="$(git rev-parse HEAD)" job.id="$GITHUB_RUN_ID" job.url="$GITHUB_SERVER_URL/$GITHUB_REPOSITORY/actions/runs/$GITHUB_RUN_ID" arch="$(uname -i)" - --log raw.log - --storage minio - --storage aws_s3 - --aws-s3-bucket ${{ secrets.REGRESSION_AWS_S3_BUCKET }} - --aws-s3-region ${{ secrets.REGRESSION_AWS_S3_REGION }} - --aws-s3-key-id ${{ secrets.REGRESSION_AWS_S3_KEY_ID }} - --aws-s3-access-key ${{ secrets.REGRESSION_AWS_S3_SECRET_ACCESS_KEY }} - --storage gcs - --gcs-uri ${{ secrets.REGRESSION_GCS_URI }} - --gcs-key-id ${{ secrets.REGRESSION_GCS_KEY_ID }} - --gcs-key-secret ${{ secrets.REGRESSION_GCS_KEY_SECRET }} - - name: Create and upload logs - if: always() - run: .github/create_and_upload_logs.sh 1 - env: - artifact_s3_dir: build/v${{ env.version }}/$GITHUB_SHA - - uses: actions/upload-artifact@v3 - if: always() - with: - name: ${{ env.SUITE }}-amd64-artifacts - path: | - ./report.html - ./*.log.txt - ./*.log - ./*.html - ./*/_instances/*.log - ./*/_instances/*/logs/*.log - ./*/*/_instances/*/logs/*.log - ./*/*/_instances/*.log - - RegressionS3Amd64: - strategy: - fail-fast: false - matrix: - STORAGE: [minio, aws_s3, gcs] - needs: [RegressionStart] - runs-on: [self-hosted, regression-tester, on-demand, type-cpx51, image-x86-app-docker-ce] - timeout-minutes: 180 - env: - AWS_ACCESS_KEY_ID: ${{ secrets.AWS_REPORT_KEY_ID }} - AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_REPORT_SECRET_ACCESS_KEY }} - AWS_DEFAULT_REGION: ${{ secrets.AWS_REPORT_REGION }} - steps: - - name: Checkout regression repo - uses: actions/checkout@v3 - with: - repository: Altinity/clickhouse-regression - - name: Set envs - run: | - cat >> "$GITHUB_ENV" << 'EOF' - REPORTS_PATH=${{runner.temp}}/reports_dir - SUITE=s3 - STORAGE=/${{ matrix.STORAGE }} - artifacts=public - EOF - - name: Download json reports - uses: actions/download-artifact@v3 - with: - path: ${{ env.REPORTS_PATH }} - - name: Setup - run: .github/setup.sh - - name: Get deb url - run: python3 .github/get-deb-url.py --reports-path ${{ env.REPORTS_PATH }} --github-env $GITHUB_ENV - - name: Run ${{ env.SUITE }} suite - run: python3 - -u ${{ env.SUITE }}/regression.py - --clickhouse-binary-path ${{ env.clickhouse_binary_path }} - --test-to-end - --local - --collect-service-logs - --output classic - --parallel 1 - --attr project="$GITHUB_REPOSITORY" project.id="$GITHUB_REPOSITORY_ID" package="${{ env.clickhouse_binary_path }}" version="${{ env.version }}" user.name="$GITHUB_ACTOR" repository="https://github.com/Altinity/clickhouse-regression" commit.hash="$(git rev-parse HEAD)" job.id="$GITHUB_RUN_ID" job.url="$GITHUB_SERVER_URL/$GITHUB_REPOSITORY/actions/runs/$GITHUB_RUN_ID" arch="$(uname -i)" - --log raw.log - --storage ${{ matrix.STORAGE }} - --gcs-uri ${{ secrets.REGRESSION_GCS_URI }} - --gcs-key-id ${{ secrets.REGRESSION_GCS_KEY_ID }} - --gcs-key-secret ${{ secrets.REGRESSION_GCS_KEY_SECRET }} - --aws-s3-bucket ${{ secrets.REGRESSION_AWS_S3_BUCKET }} - --aws-s3-region ${{ secrets.REGRESSION_AWS_S3_REGION }} - --aws-s3-key-id ${{ secrets.REGRESSION_AWS_S3_KEY_ID }} - --aws-s3-access-key ${{ secrets.REGRESSION_AWS_S3_SECRET_ACCESS_KEY }} - - name: Create and upload logs - if: always() - run: .github/create_and_upload_logs.sh 1 - env: - artifact_s3_dir: build/v${{ env.version }}/$GITHUB_SHA - - uses: actions/upload-artifact@v3 - if: always() - with: - name: ${{ env.SUITE }}-${{ matrix.STORAGE }}-amd64-artifacts - path: | - ./report.html - ./*.log.txt - ./*.log - ./*.html - ./*/_instances/*.log - ./*/_instances/*/logs/*.log - ./*/*/_instances/*/logs/*.log - ./*/*/_instances/*.log - - RegressionS3Aarch64: - strategy: - fail-fast: false - matrix: - STORAGE: [minio, aws_s3, gcs] - needs: [RegressionStart] - runs-on: [self-hosted, regression-tester, on-demand, type-cax41, image-arm-app-docker-ce] - timeout-minutes: 180 - env: - AWS_ACCESS_KEY_ID: ${{ secrets.AWS_REPORT_KEY_ID }} - AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_REPORT_SECRET_ACCESS_KEY }} - AWS_DEFAULT_REGION: ${{ secrets.AWS_REPORT_REGION }} - steps: - - name: Checkout regression repo - uses: actions/checkout@v3 - with: - repository: Altinity/clickhouse-regression - ref: ${{ env.REGRESSION_ARM_COMMIT }} - - name: Set envs - run: | - cat >> "$GITHUB_ENV" << 'EOF' - REPORTS_PATH=${{runner.temp}}/reports_dir - SUITE=s3 - STORAGE=/${{ matrix.STORAGE }} - artifacts=public - EOF - - name: Download json reports - uses: actions/download-artifact@v3 - with: - path: ${{ env.REPORTS_PATH }} - - name: Setup - run: .github/setup.sh - - name: Get deb url - run: python3 .github/get-deb-url.py --reports-path ${{ env.REPORTS_PATH }} --github-env $GITHUB_ENV - - name: Run ${{ env.SUITE }} suite - run: python3 - -u ${{ env.SUITE }}/regression.py - --clickhouse-binary-path ${{ env.clickhouse_binary_path }} - --test-to-end - --local - --collect-service-logs - --output classic - --parallel 1 - --attr project="$GITHUB_REPOSITORY" project.id="$GITHUB_REPOSITORY_ID" package="${{ env.clickhouse_binary_path }}" version="${{ env.version }}" user.name="$GITHUB_ACTOR" repository="https://github.com/Altinity/clickhouse-regression" commit.hash="$(git rev-parse HEAD)" job.id="$GITHUB_RUN_ID" job.url="$GITHUB_SERVER_URL/$GITHUB_REPOSITORY/actions/runs/$GITHUB_RUN_ID" arch="$(uname -i)" - --log raw.log - --storage ${{ matrix.STORAGE }} - --gcs-uri ${{ secrets.REGRESSION_GCS_URI }} - --gcs-key-id ${{ secrets.REGRESSION_GCS_KEY_ID }} - --gcs-key-secret ${{ secrets.REGRESSION_GCS_KEY_SECRET }} - --aws-s3-bucket ${{ secrets.REGRESSION_AWS_S3_BUCKET }} - --aws-s3-region ${{ secrets.REGRESSION_AWS_S3_REGION }} - --aws-s3-key-id ${{ secrets.REGRESSION_AWS_S3_KEY_ID }} - --aws-s3-access-key ${{ secrets.REGRESSION_AWS_S3_SECRET_ACCESS_KEY }} - - name: Create and upload logs - if: always() - run: .github/create_and_upload_logs.sh 1 - env: - artifact_s3_dir: build/v${{ env.version }}/$GITHUB_SHA - - uses: actions/upload-artifact@v3 - if: always() - with: - name: ${{ env.SUITE }}-${{ matrix.STORAGE }}-aarch64-artifacts - path: | - ./report.html - ./*.log.txt - ./*.log - ./*.html - ./*/_instances/*.log - ./*/_instances/*/logs/*.log - ./*/*/_instances/*/logs/*.log - ./*/*/_instances/*.log - - RegressionTieredStorageS3Amd64: - strategy: - fail-fast: false - matrix: - STORAGE: [minio, s3amazon, s3gcs] - needs: [RegressionStart] - runs-on: [self-hosted, regression-tester, on-demand, type-cpx51, image-x86-app-docker-ce] - timeout-minutes: 180 - env: - AWS_ACCESS_KEY_ID: ${{ secrets.AWS_REPORT_KEY_ID }} - AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_REPORT_SECRET_ACCESS_KEY }} - AWS_DEFAULT_REGION: ${{ secrets.AWS_REPORT_REGION }} - steps: - - name: Checkout regression repo - uses: actions/checkout@v3 - with: - repository: Altinity/clickhouse-regression - - name: Set envs - run: | - cat >> "$GITHUB_ENV" << 'EOF' - REPORTS_PATH=${{runner.temp}}/reports_dir - SUITE=tiered_storage - STORAGE=/${{ matrix.STORAGE }} - artifacts=public - EOF - - name: Download json reports - uses: actions/download-artifact@v3 - with: - path: ${{ env.REPORTS_PATH }} - - name: Setup - run: .github/setup.sh - - name: Get deb url - run: python3 .github/get-deb-url.py --reports-path ${{ env.REPORTS_PATH }} --github-env $GITHUB_ENV - - name: Run ${{ env.SUITE }} suite - run: python3 - -u ${{ env.SUITE }}/regression.py - --clickhouse-binary-path ${{ env.clickhouse_binary_path }} - --test-to-end - --local - --collect-service-logs - --output classic - --parallel 1 - --attr project="$GITHUB_REPOSITORY" project.id="$GITHUB_REPOSITORY_ID" package="${{ env.clickhouse_binary_path }}" version="${{ env.version }}" user.name="$GITHUB_ACTOR" repository="https://github.com/Altinity/clickhouse-regression" commit.hash="$(git rev-parse HEAD)" job.id="$GITHUB_RUN_ID" job.url="$GITHUB_SERVER_URL/$GITHUB_REPOSITORY/actions/runs/$GITHUB_RUN_ID" arch="$(uname -i)" - --log raw.log - --aws-s3-access-key ${{ secrets.REGRESSION_AWS_S3_SECRET_ACCESS_KEY }} - --aws-s3-key-id ${{ secrets.REGRESSION_AWS_S3_KEY_ID }} - --aws-s3-uri https://s3.${{ secrets.REGRESSION_AWS_S3_REGION}}.amazonaws.com/${{ secrets.REGRESSION_AWS_S3_BUCKET }}/data/ - --gcs-key-id ${{ secrets.REGRESSION_GCS_KEY_ID }} - --gcs-key-secret ${{ secrets.REGRESSION_GCS_KEY_SECRET }} - --gcs-uri ${{ secrets.REGRESSION_GCS_URI }} - --with-${{ matrix.STORAGE }} - - name: Create and upload logs - if: always() - run: .github/create_and_upload_logs.sh 1 - env: - artifact_s3_dir: build/v${{ env.version }}/$GITHUB_SHA - - uses: actions/upload-artifact@v3 - if: always() - with: - name: ${{ env.SUITE }}-${{ matrix.STORAGE }}-amd64-artifacts - path: | - ./report.html - ./*.log.txt - ./*.log - ./*.html - ./*/_instances/*.log - ./*/_instances/*/logs/*.log - ./*/*/_instances/*/logs/*.log - ./*/*/_instances/*.log - - RegressionTieredStorageS3Aarch64: - strategy: - fail-fast: false - matrix: - STORAGE: [minio, s3amazon, s3gcs] - needs: [RegressionStart] - runs-on: [self-hosted, regression-tester, on-demand, type-cax41, image-arm-app-docker-ce] - timeout-minutes: 180 - env: - AWS_ACCESS_KEY_ID: ${{ secrets.AWS_REPORT_KEY_ID }} - AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_REPORT_SECRET_ACCESS_KEY }} - AWS_DEFAULT_REGION: ${{ secrets.AWS_REPORT_REGION }} - steps: - - name: Checkout regression repo - uses: actions/checkout@v3 - with: - repository: Altinity/clickhouse-regression - ref: ${{ env.REGRESSION_ARM_COMMIT }} - - name: Set envs - run: | - cat >> "$GITHUB_ENV" << 'EOF' - REPORTS_PATH=${{runner.temp}}/reports_dir - SUITE=tiered_storage - STORAGE=/${{ matrix.STORAGE }} - artifacts=public - EOF - - name: Download json reports - uses: actions/download-artifact@v3 - with: - path: ${{ env.REPORTS_PATH }} - - name: Setup - run: .github/setup.sh - - name: Get deb url - run: python3 .github/get-deb-url.py --reports-path ${{ env.REPORTS_PATH }} --github-env $GITHUB_ENV - - name: Run ${{ env.SUITE }} suite - run: python3 - -u ${{ env.SUITE }}/regression.py - --clickhouse-binary-path ${{ env.clickhouse_binary_path }} - --test-to-end - --local - --collect-service-logs - --output classic - --parallel 1 - --attr project="$GITHUB_REPOSITORY" project.id="$GITHUB_REPOSITORY_ID" package="${{ env.clickhouse_binary_path }}" version="${{ env.version }}" user.name="$GITHUB_ACTOR" repository="https://github.com/Altinity/clickhouse-regression" commit.hash="$(git rev-parse HEAD)" job.id="$GITHUB_RUN_ID" job.url="$GITHUB_SERVER_URL/$GITHUB_REPOSITORY/actions/runs/$GITHUB_RUN_ID" arch="$(uname -i)" - --log raw.log - --aws-s3-access-key ${{ secrets.REGRESSION_AWS_S3_SECRET_ACCESS_KEY }} - --aws-s3-key-id ${{ secrets.REGRESSION_AWS_S3_KEY_ID }} - --aws-s3-uri https://s3.${{ secrets.REGRESSION_AWS_S3_REGION}}.amazonaws.com/${{ secrets.REGRESSION_AWS_S3_BUCKET }}/data/ - --gcs-key-id ${{ secrets.REGRESSION_GCS_KEY_ID }} - --gcs-key-secret ${{ secrets.REGRESSION_GCS_KEY_SECRET }} - --gcs-uri ${{ secrets.REGRESSION_GCS_URI }} - --with-${{ matrix.STORAGE }} - - name: Create and upload logs - if: always() - run: .github/create_and_upload_logs.sh 1 - env: - artifact_s3_dir: build/v${{ env.version }}/$GITHUB_SHA - - uses: actions/upload-artifact@v3 - if: always() - with: - name: ${{ env.SUITE }}-${{ matrix.STORAGE }}-aarch64-artifacts - path: | - ./report.html - ./*.log.txt - ./*.log - ./*.html - ./*/_instances/*.log - ./*/_instances/*/logs/*.log - ./*/*/_instances/*/logs/*.log - ./*/*/_instances/*.log + RegressionTestsRelease: + needs: [BuilderReport] + uses: ./.github/workflows/regression.yml + secrets: inherit + with: + runner_type: altinity-on-demand, altinity-type-cpx51, altinity-image-x86-app-docker-ce, altinity-setup-regression + commit: 6da94b78dc53cb8965ab56c04a89ebf54ed04cbc + arch: release + build_sha: ${{ github.event_name == 'pull_request' && github.event.pull_request.head.sha || github.event_name == 'release' && github.sha }} + + RegressionTestsAarch64: + needs: [BuilderReport] + uses: ./.github/workflows/regression.yml + secrets: inherit + with: + runner_type: altinity-on-demand, altinity-type-cax41, altinity-image-arm-app-docker-ce, altinity-setup-regression + commit: 6da94b78dc53cb8965ab56c04a89ebf54ed04cbc + arch: aarch64 + build_sha: ${{ github.event_name == 'pull_request' && github.event.pull_request.head.sha || github.event_name == 'release' && github.sha }} SignRelease: needs: [BuilderDebRelease] - runs-on: [self-hosted, on-demand, type-cpx41, image-x86-app-docker-ce] + runs-on: [self-hosted, altinity-on-demand, altinity-type-cpx41, altinity-in-ash, altinity-image-x86-app-docker-ce] timeout-minutes: 180 steps: - name: Set envs @@ -1232,29 +741,35 @@ jobs: - DockerHubPush - DockerServerImages - BuilderReport - # - BuilderSpecialReport - MarkReleaseReady + - FunctionalStatelessTestDebug - FunctionalStatelessTestRelease - FunctionalStatelessTestAarch64 + - FunctionalStatelessTestAsan + - FunctionalStatelessTestTsan + - FunctionalStatelessTestMsan + - FunctionalStatelessTestUBsan + - FunctionalStatefulTestDebug - FunctionalStatefulTestRelease - FunctionalStatefulTestAarch64 + - FunctionalStatefulTestAsan + - FunctionalStatefulTestTsan + - FunctionalStatefulTestMsan + - FunctionalStatefulTestUBsan + - StressTestDebug + - StressTestAsan + - StressTestTsan + - StressTestMsan + - StressTestUBsan + - IntegrationTestsAsan + - IntegrationTestsTsan - IntegrationTestsRelease - - CompatibilityCheck - - RegressionCommonAmd64 - - RegressionCommonAarch64 - - RegressionBenchmarkAmd64 - - RegressionBenchmarkAarch64 - - RegressionClickHouseKeeperSSLAmd64 - - RegressionClickHouseKeeperSSLAarch64 - - RegressionLDAPAmd64 - - RegressionLDAPAarch64 - - RegressionParquetAmd64 - - RegressionS3Amd64 - - RegressionS3Aarch64 - - RegressionTieredStorageS3Amd64 - - RegressionTieredStorageS3Aarch64 + - CompatibilityCheckX86 + - CompatibilityCheckAarch64 + - RegressionTestsRelease + - RegressionTestsAarch64 - SignRelease - runs-on: [self-hosted, style-checker, on-demand, type-cpx31, image-x86-app-docker-ce] + runs-on: [self-hosted, altinity-on-demand, altinity-type-cpx31, altinity-image-x86-app-docker-ce] steps: - name: Check out repository code uses: Altinity/checkout@19599efdf36c4f3f30eb55d5bb388896faea69f6 From 9a8d06dec3a3dd88ac8e566d79f3f705cb496b5e Mon Sep 17 00:00:00 2001 From: MyroTk <44327070+MyroTk@users.noreply.github.com> Date: Thu, 28 Mar 2024 09:01:52 -0700 Subject: [PATCH 002/123] Update release_branches.yml --- .github/workflows/release_branches.yml | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/.github/workflows/release_branches.yml b/.github/workflows/release_branches.yml index 435f00e0abf9..90737d222549 100644 --- a/.github/workflows/release_branches.yml +++ b/.github/workflows/release_branches.yml @@ -40,6 +40,10 @@ jobs: run: | cd "$GITHUB_WORKSPACE/tests/ci" python3 docker_images_check.py --suffix aarch64 + env: + RUNNER_IP: "$(hostname -I | cut -d ' ' -f 1)" + RUNNER_SSH_COMMAND: "ssh -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null root@$(hostname -I | cut -d ' ' -f 1)" + - name: Upload images files to artifacts uses: actions/upload-artifact@v3 with: @@ -58,6 +62,10 @@ jobs: run: | cd "$GITHUB_WORKSPACE/tests/ci" python3 docker_images_check.py --suffix amd64 + env: + RUNNER_IP: "$(hostname -I | cut -d ' ' -f 1)" + RUNNER_SSH_COMMAND: "ssh -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null root@$(hostname -I | cut -d ' ' -f 1)" + - name: Upload images files to artifacts uses: actions/upload-artifact@v3 with: From 0dac80cdd0fef3d44b3515502c86a7dc7e05a033 Mon Sep 17 00:00:00 2001 From: MyroTk <44327070+MyroTk@users.noreply.github.com> Date: Thu, 28 Mar 2024 09:02:43 -0700 Subject: [PATCH 003/123] Create regression.yml --- .github/workflows/regression.yml | 515 +++++++++++++++++++++++++++++++ 1 file changed, 515 insertions(+) create mode 100644 .github/workflows/regression.yml diff --git a/.github/workflows/regression.yml b/.github/workflows/regression.yml new file mode 100644 index 000000000000..cc89340617a6 --- /dev/null +++ b/.github/workflows/regression.yml @@ -0,0 +1,515 @@ +name: Regression test workflow +'on': + workflow_call: + inputs: + runner_type: + description: the label of runner to use, can be a simple string or a comma-separated list + required: true + type: string + commit: + description: commit hash of the regression tests. + required: true + type: string + arch: + description: arch to run the tests on. + required: true + type: string + timeout_minutes: + description: Maximum number of minutes to let workflow run before GitHub cancels it. + default: 210 + type: number + build_sha: + description: commit sha of the workflow run for artifact upload. + required: true + type: string + checkout_depth: + description: the value of the git shallow checkout + required: false + type: number + default: 1 + submodules: + description: if the submodules should be checked out + required: false + type: boolean + default: false + additional_envs: + description: additional ENV variables to setup the job + type: string + secrets: + secret_envs: + description: if given, it's passed to the environments + required: false + AWS_SECRET_ACCESS_KEY: + description: the access key to the aws param store. + required: true + AWS_ACCESS_KEY_ID: + description: the access key id to the aws param store. + required: true + AWS_DEFAULT_REGION: + description: the region of the aws param store. + required: true + AWS_REPORT_KEY_ID: + description: aws s3 key id used for regression test reports. + required: true + AWS_REPORT_SECRET_ACCESS_KEY: + description: aws s3 secret access key used for regression test reports. + required: true + AWS_REPORT_REGION: + description: aws s3 region used for regression test reports. + required: true + DOCKER_USERNAME: + description: username of the docker user. + required: true + DOCKER_PASSWORD: + description: password to the docker user. + required: true + REGRESSION_AWS_S3_BUCKET: + description: aws s3 bucket used for regression tests. + required: true + REGRESSION_AWS_S3_KEY_ID: + description: aws s3 key id used for regression tests. + required: true + REGRESSION_AWS_S3_SECRET_ACCESS_KEY: + description: aws s3 secret access key used for regression tests. + required: true + REGRESSION_AWS_S3_REGION: + description: aws s3 region used for regression tests. + required: true + REGRESSION_GCS_KEY_ID: + description: gcs key id used for regression tests. + required: true + REGRESSION_GCS_KEY_SECRET: + description: gcs key secret used for regression tests. + required: true + REGRESSION_GCS_URI: + description: gcs uri used for regression tests. + required: true + +env: + # Force the stdout and stderr streams to be unbuffered + PYTHONUNBUFFERED: 1 + AWS_ACCESS_KEY_ID: ${{ secrets.AWS_REPORT_KEY_ID }} + AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_REPORT_SECRET_ACCESS_KEY }} + AWS_DEFAULT_REGION: ${{ secrets.AWS_REPORT_REGION }} + DOCKER_USERNAME: ${{ secrets.DOCKER_USERNAME }} + DOCKER_PASSWORD: ${{ secrets.DOCKER_PASSWORD }} + args: --test-to-end + --no-colors + --local + --collect-service-logs + --output classic + --parallel 1 + --log raw.log + artifacts: builds + artifact_paths: | + ./report.html + ./*.log.txt + ./*.log + ./*.html + ./*/_instances/*.log + ./*/_instances/*/logs/*.log + ./*/*/_instances/*/logs/*.log + ./*/*/_instances/*.log + build_sha: ${{ inputs.build_sha }} + pr_number: ${{ github.event.number }} + event_name: ${{ github.event_name }} + +jobs: + runner_labels_setup: + name: Compute proper runner labels for the rest of the jobs + runs-on: ubuntu-latest + outputs: + runner_labels: ${{ steps.setVariables.outputs.runner_labels }} + steps: + - id: setVariables + name: Prepare runner_labels variables for the later steps + run: | + + # Prepend self-hosted + input="self-hosted, ${input}" + + # Remove all whitespace + input="$(echo ${input} | tr -d [:space:])" + # Make something like a JSON array from comma-separated list + input="[ '${input//\,/\'\, \'}' ]" + + echo "runner_labels=$input" >> ${GITHUB_OUTPUT} + env: + input: ${{ inputs.runner_type }} + + Common: + strategy: + fail-fast: false + matrix: + SUITE: [aes_encryption, aggregate_functions, alter, atomic_insert, base_58, clickhouse_keeper, data_types, datetime64_extended_range, disk_level_encryption, dns, engines, example, extended_precision_data_types, kafka, kerberos, key_value, lightweight_delete, memory, part_moves_between_shards, rbac, selects, session_timezone, ssl_server, tiered_storage, window_functions] + needs: [runner_labels_setup] + runs-on: ${{ fromJson(needs.runner_labels_setup.outputs.runner_labels) }} + timeout-minutes: ${{ inputs.timeout_minutes }} + steps: + - name: Checkout regression repo + uses: actions/checkout@v3 + with: + repository: Altinity/clickhouse-regression + ref: ${{ inputs.commit }} + - name: Set envs + run: | + cat >> "$GITHUB_ENV" << 'EOF' + REPORTS_PATH=${{ runner.temp }}/reports_dir + SUITE=${{ matrix.SUITE }} + EOF + - name: Download json reports + uses: actions/download-artifact@v3 + with: + path: ${{ env.REPORTS_PATH }} + name: build_report_package_${{ inputs.arch }} + - name: Setup + run: .github/setup.sh + - name: Get deb url + run: python3 .github/get-deb-url.py --reports-path ${{ env.REPORTS_PATH }} --github-env $GITHUB_ENV + - name: Run ${{ env.SUITE }} suite + run: python3 + -u ${{ env.SUITE }}/regression.py + --clickhouse-binary-path ${{ env.clickhouse_binary_path }} + --attr project="$GITHUB_REPOSITORY" project.id="$GITHUB_REPOSITORY_ID" package="${{ env.clickhouse_binary_path }}" version="${{ env.version }}" user.name="$GITHUB_ACTOR" repository="https://github.com/Altinity/clickhouse-regression" commit.hash="$(git rev-parse HEAD)" job.id="$GITHUB_RUN_ID" job.url="$GITHUB_SERVER_URL/$GITHUB_REPOSITORY/actions/runs/$GITHUB_RUN_ID" arch="$(uname -i)" + ${{ env.args }} + - name: Create and upload logs + if: always() + run: .github/create_and_upload_logs.sh 1 + - uses: actions/upload-artifact@v3 + if: always() + with: + name: ${{ env.SUITE }}-${{ inputs.arch }}-artifacts + path: ${{ env.artifact_paths}} + + Benchmark: + strategy: + fail-fast: false + matrix: + STORAGE: [minio, aws_s3, gcs] + needs: [runner_labels_setup] + runs-on: ${{ fromJson(needs.runner_labels_setup.outputs.runner_labels) }} + timeout-minutes: ${{ inputs.timeout_minutes }} + steps: + - name: Checkout regression repo + uses: actions/checkout@v3 + with: + repository: Altinity/clickhouse-regression + ref: ${{ inputs.commit }} + - name: Set envs + run: | + cat >> "$GITHUB_ENV" << 'EOF' + REPORTS_PATH=${{ runner.temp }}/reports_dir + SUITE=ontime_benchmark + STORAGE=/${{ matrix.STORAGE }} + EOF + - name: Download json reports + uses: actions/download-artifact@v3 + with: + path: ${{ env.REPORTS_PATH }} + name: build_report_package_${{ inputs.arch }} + - name: Setup + run: .github/setup.sh + - name: Get deb url + run: python3 .github/get-deb-url.py --reports-path ${{ env.REPORTS_PATH }} --github-env $GITHUB_ENV + - name: Run ${{ env.SUITE }} suite + run: python3 + -u ${{ env.SUITE }}/benchmark.py + --clickhouse-binary-path ${{ env.clickhouse_binary_path }} + --storage ${{ matrix.STORAGE }} + --gcs-uri ${{ secrets.REGRESSION_GCS_URI }} + --gcs-key-id ${{ secrets.REGRESSION_GCS_KEY_ID }} + --gcs-key-secret ${{ secrets.REGRESSION_GCS_KEY_SECRET }} + --aws-s3-bucket ${{ secrets.REGRESSION_AWS_S3_BUCKET }} + --aws-s3-region ${{ secrets.REGRESSION_AWS_S3_REGION }} + --aws-s3-key-id ${{ secrets.REGRESSION_AWS_S3_KEY_ID }} + --aws-s3-access-key ${{ secrets.REGRESSION_AWS_S3_SECRET_ACCESS_KEY }} + --attr project="$GITHUB_REPOSITORY" project.id="$GITHUB_REPOSITORY_ID" package="${{ env.clickhouse_binary_path }}" version="${{ env.version }}" user.name="$GITHUB_ACTOR" repository="https://github.com/Altinity/clickhouse-regression" commit.hash="$(git rev-parse HEAD)" job.id="$GITHUB_RUN_ID" job.url="$GITHUB_SERVER_URL/$GITHUB_REPOSITORY/actions/runs/$GITHUB_RUN_ID" arch="$(uname -i)" + ${{ env.args }} + - name: Create and upload logs + if: always() + run: .github/create_and_upload_logs.sh 1 + - uses: actions/upload-artifact@v3 + if: always() + with: + name: benchmark-${{ matrix.STORAGE }}-${{ inputs.arch }}-artifacts + path: ${{ env.artifact_paths }} + + ClickHouseKeeperSSL: + needs: [runner_labels_setup] + runs-on: ${{ fromJson(needs.runner_labels_setup.outputs.runner_labels) }} + timeout-minutes: ${{ inputs.timeout_minutes }} + steps: + - name: Checkout regression repo + uses: actions/checkout@v3 + with: + repository: Altinity/clickhouse-regression + ref: ${{ inputs.commit }} + - name: Set envs + run: | + cat >> "$GITHUB_ENV" << 'EOF' + REPORTS_PATH=${{runner.temp}}/reports_dir + SUITE=clickhouse_keeper + STORAGE=/ssl + EOF + - name: Download json reports + uses: actions/download-artifact@v3 + with: + path: ${{ env.REPORTS_PATH }} + name: build_report_package_${{ inputs.arch }} + - name: Setup + run: .github/setup.sh + - name: Get deb url + run: python3 .github/get-deb-url.py --reports-path ${{ env.REPORTS_PATH }} --github-env $GITHUB_ENV + - name: Run ${{ env.SUITE }} suite + run: python3 + -u ${{ env.SUITE }}/regression.py + --ssl + --clickhouse-binary-path ${{ env.clickhouse_binary_path }} + --attr project="$GITHUB_REPOSITORY" project.id="$GITHUB_REPOSITORY_ID" package="${{ env.clickhouse_binary_path }}" version="${{ env.version }}" user.name="$GITHUB_ACTOR" repository="https://github.com/Altinity/clickhouse-regression" commit.hash="$(git rev-parse HEAD)" job.id="$GITHUB_RUN_ID" job.url="$GITHUB_SERVER_URL/$GITHUB_REPOSITORY/actions/runs/$GITHUB_RUN_ID" arch="$(uname -i)" + ${{ env.args }} + - name: Create and upload logs + if: always() + run: .github/create_and_upload_logs.sh 1 + - uses: actions/upload-artifact@v3 + if: always() + with: + name: ${{ env.SUITE }}-${{ inputs.arch }}-ssl-artifacts + path: ${{ env.artifact_paths }} + + LDAP: + strategy: + fail-fast: false + matrix: + SUITE: [authentication, external_user_directory, role_mapping] + needs: [runner_labels_setup] + runs-on: ${{ fromJson(needs.runner_labels_setup.outputs.runner_labels) }} + timeout-minutes: ${{ inputs.timeout_minutes }} + steps: + - name: Checkout regression repo + uses: actions/checkout@v3 + with: + repository: Altinity/clickhouse-regression + ref: ${{ inputs.commit }} + - name: Set envs + run: | + cat >> "$GITHUB_ENV" << 'EOF' + REPORTS_PATH=${{ runner.temp }}/reports_dir + SUITE=ldap/${{ matrix.SUITE }} + EOF + - name: Download json reports + uses: actions/download-artifact@v3 + with: + path: ${{ env.REPORTS_PATH }} + name: build_report_package_${{ inputs.arch }} + - name: Setup + run: .github/setup.sh + - name: Get deb url + run: python3 .github/get-deb-url.py --reports-path ${{ env.REPORTS_PATH }} --github-env $GITHUB_ENV + - name: Run ${{ env.SUITE }} suite + run: python3 + -u ${{ env.SUITE }}/regression.py + --clickhouse-binary-path ${{ env.clickhouse_binary_path }} + --attr project="$GITHUB_REPOSITORY" project.id="$GITHUB_REPOSITORY_ID" package="${{ env.clickhouse_binary_path }}" version="${{ env.version }}" user.name="$GITHUB_ACTOR" repository="https://github.com/Altinity/clickhouse-regression" commit.hash="$(git rev-parse HEAD)" job.id="$GITHUB_RUN_ID" job.url="$GITHUB_SERVER_URL/$GITHUB_REPOSITORY/actions/runs/$GITHUB_RUN_ID" arch="$(uname -i)" + ${{ env.args }} + - name: Create and upload logs + if: always() + run: .github/create_and_upload_logs.sh 1 + - uses: actions/upload-artifact@v3 + if: always() + with: + name: ldap-${{ matrix.SUITE }}-${{ inputs.arch }}-artifacts + path: ${{ env.artifact_paths }} + + Parquet: + needs: [runner_labels_setup] + runs-on: ${{ fromJson(needs.runner_labels_setup.outputs.runner_labels) }} + timeout-minutes: ${{ inputs.timeout_minutes }} + steps: + - name: Checkout regression repo + uses: actions/checkout@v3 + with: + repository: Altinity/clickhouse-regression + ref: ${{ inputs.commit }} + - name: Set envs + run: | + cat >> "$GITHUB_ENV" << 'EOF' + REPORTS_PATH=${{ runner.temp }}/reports_dir + SUITE=parquet + EOF + - name: Download json reports + uses: actions/download-artifact@v3 + with: + path: ${{ env.REPORTS_PATH }} + name: build_report_package_${{ inputs.arch }} + - name: Setup + run: .github/setup.sh + - name: Get deb url + run: python3 .github/get-deb-url.py --reports-path ${{ env.REPORTS_PATH }} --github-env $GITHUB_ENV + - name: Run ${{ env.SUITE }} suite + run: python3 + -u ${{ env.SUITE }}/regression.py + --clickhouse-binary-path ${{ env.clickhouse_binary_path }} + --attr project="$GITHUB_REPOSITORY" project.id="$GITHUB_REPOSITORY_ID" package="${{ env.clickhouse_binary_path }}" version="${{ env.version }}" user.name="$GITHUB_ACTOR" repository="https://github.com/Altinity/clickhouse-regression" commit.hash="$(git rev-parse HEAD)" job.id="$GITHUB_RUN_ID" job.url="$GITHUB_SERVER_URL/$GITHUB_REPOSITORY/actions/runs/$GITHUB_RUN_ID" arch="$(uname -i)" + ${{ env.args }} + - name: Create and upload logs + if: always() + run: .github/create_and_upload_logs.sh 1 + - uses: actions/upload-artifact@v3 + if: always() + with: + name: ${{ env.SUITE }}-${{ inputs.arch }}-artifacts + path: ${{ env.artifact_paths }} + + ParquetS3: + strategy: + fail-fast: false + matrix: + STORAGE: [minio, aws_s3] + needs: [runner_labels_setup] + runs-on: ${{ fromJson(needs.runner_labels_setup.outputs.runner_labels) }} + timeout-minutes: ${{ inputs.timeout_minutes }} + steps: + - name: Checkout regression repo + uses: actions/checkout@v3 + with: + repository: Altinity/clickhouse-regression + ref: ${{ inputs.commit }} + - name: Set envs + run: | + cat >> "$GITHUB_ENV" << 'EOF' + REPORTS_PATH=${{ runner.temp }}/reports_dir + SUITE=parquet + STORAGE=${{ matrix.STORAGE}} + EOF + - name: Download json reports + uses: actions/download-artifact@v3 + with: + path: ${{ env.REPORTS_PATH }} + name: build_report_package_${{ inputs.arch }} + - name: Setup + run: .github/setup.sh + - name: Get deb url + run: python3 .github/get-deb-url.py --reports-path ${{ env.REPORTS_PATH }} --github-env $GITHUB_ENV + - name: Run ${{ env.SUITE }} suite + run: python3 + -u ${{ env.SUITE }}/regression.py + --clickhouse-binary-path ${{ env.clickhouse_binary_path }} + --storage ${{ matrix.STORAGE }} + --aws-s3-bucket ${{ secrets.REGRESSION_AWS_S3_BUCKET }} + --aws-s3-region ${{ secrets.REGRESSION_AWS_S3_REGION }} + --aws-s3-key-id ${{ secrets.REGRESSION_AWS_S3_KEY_ID }} + --aws-s3-access-key ${{ secrets.REGRESSION_AWS_S3_SECRET_ACCESS_KEY }} + --attr project="$GITHUB_REPOSITORY" project.id="$GITHUB_REPOSITORY_ID" package="${{ env.clickhouse_binary_path }}" version="${{ env.version }}" user.name="$GITHUB_ACTOR" repository="https://github.com/Altinity/clickhouse-regression" commit.hash="$(git rev-parse HEAD)" job.id="$GITHUB_RUN_ID" job.url="$GITHUB_SERVER_URL/$GITHUB_REPOSITORY/actions/runs/$GITHUB_RUN_ID" arch="$(uname -i)" + ${{ env.args }} + - name: Create and upload logs + if: always() + run: .github/create_and_upload_logs.sh 1 + - uses: actions/upload-artifact@v3 + if: always() + with: + name: ${{ env.SUITE }}-${{ env.STORAGE }}-${{ inputs.arch }}-artifacts + path: ${{ env.artifact_paths }} + + S3: + strategy: + fail-fast: false + matrix: + STORAGE: [minio, aws_s3, gcs] + needs: [runner_labels_setup] + runs-on: ${{ fromJson(needs.runner_labels_setup.outputs.runner_labels) }} + timeout-minutes: ${{ inputs.timeout_minutes }} + steps: + - name: Checkout regression repo + uses: actions/checkout@v3 + with: + repository: Altinity/clickhouse-regression + ref: ${{ inputs.commit }} + - name: Set envs + run: | + cat >> "$GITHUB_ENV" << 'EOF' + REPORTS_PATH=${{ runner.temp }}/reports_dir + SUITE=s3 + STORAGE=/${{ matrix.STORAGE }} + EOF + - name: Download json reports + uses: actions/download-artifact@v3 + with: + path: ${{ env.REPORTS_PATH }} + name: build_report_package_${{ inputs.arch }} + - name: Setup + run: .github/setup.sh + - name: Get deb url + run: python3 .github/get-deb-url.py --reports-path ${{ env.REPORTS_PATH }} --github-env $GITHUB_ENV + - name: Run ${{ env.SUITE }} suite + run: python3 + -u ${{ env.SUITE }}/regression.py + --clickhouse-binary-path ${{ env.clickhouse_binary_path }} + --storage ${{ matrix.STORAGE }} + --gcs-uri ${{ secrets.REGRESSION_GCS_URI }} + --gcs-key-id ${{ secrets.REGRESSION_GCS_KEY_ID }} + --gcs-key-secret ${{ secrets.REGRESSION_GCS_KEY_SECRET }} + --aws-s3-bucket ${{ secrets.REGRESSION_AWS_S3_BUCKET }} + --aws-s3-region ${{ secrets.REGRESSION_AWS_S3_REGION }} + --aws-s3-key-id ${{ secrets.REGRESSION_AWS_S3_KEY_ID }} + --aws-s3-access-key ${{ secrets.REGRESSION_AWS_S3_SECRET_ACCESS_KEY }} + --attr project="$GITHUB_REPOSITORY" project.id="$GITHUB_REPOSITORY_ID" package="${{ env.clickhouse_binary_path }}" version="${{ env.version }}" user.name="$GITHUB_ACTOR" repository="https://github.com/Altinity/clickhouse-regression" commit.hash="$(git rev-parse HEAD)" job.id="$GITHUB_RUN_ID" job.url="$GITHUB_SERVER_URL/$GITHUB_REPOSITORY/actions/runs/$GITHUB_RUN_ID" arch="$(uname -i)" + ${{ env.args }} + - name: Create and upload logs + if: always() + run: .github/create_and_upload_logs.sh 1 + - uses: actions/upload-artifact@v3 + if: always() + with: + name: ${{ env.SUITE }}-${{ matrix.STORAGE }}-${{ inputs.arch }}-artifacts + path: ${{ env.artifact_paths}} + + TieredStorage: + strategy: + fail-fast: false + matrix: + STORAGE: [minio, s3amazon, s3gcs] + needs: [runner_labels_setup] + runs-on: ${{ fromJson(needs.runner_labels_setup.outputs.runner_labels) }} + timeout-minutes: ${{ inputs.timeout_minutes }} + steps: + - name: Checkout regression repo + uses: actions/checkout@v3 + with: + repository: Altinity/clickhouse-regression + - name: Set envs + run: | + cat >> "$GITHUB_ENV" << 'EOF' + REPORTS_PATH=${{ runner.temp }}/reports_dir + SUITE=tiered_storage + STORAGE=/${{ matrix.STORAGE }} + EOF + - name: Download json reports + uses: actions/download-artifact@v3 + with: + path: ${{ env.REPORTS_PATH }} + name: build_report_package_${{ inputs.arch }} + - name: Setup + run: .github/setup.sh + - name: Get deb url + run: python3 .github/get-deb-url.py --reports-path ${{ env.REPORTS_PATH }} --github-env $GITHUB_ENV + - name: Run ${{ env.SUITE }} suite + run: python3 + -u ${{ env.SUITE }}/regression.py + --clickhouse-binary-path ${{ env.clickhouse_binary_path }} + --aws-s3-access-key ${{ secrets.REGRESSION_AWS_S3_SECRET_ACCESS_KEY }} + --aws-s3-key-id ${{ secrets.REGRESSION_AWS_S3_KEY_ID }} + --aws-s3-uri https://s3.${{ secrets.REGRESSION_AWS_S3_REGION}}.amazonaws.com/${{ secrets.REGRESSION_AWS_S3_BUCKET }}/data/ + --gcs-key-id ${{ secrets.REGRESSION_GCS_KEY_ID }} + --gcs-key-secret ${{ secrets.REGRESSION_GCS_KEY_SECRET }} + --gcs-uri ${{ secrets.REGRESSION_GCS_URI }} + --with-${{ matrix.STORAGE }} + --attr project="$GITHUB_REPOSITORY" project.id="$GITHUB_REPOSITORY_ID" package="${{ env.clickhouse_binary_path }}" version="${{ env.version }}" user.name="$GITHUB_ACTOR" repository="https://github.com/Altinity/clickhouse-regression" commit.hash="$(git rev-parse HEAD)" job.id="$GITHUB_RUN_ID" job.url="$GITHUB_SERVER_URL/$GITHUB_REPOSITORY/actions/runs/$GITHUB_RUN_ID" arch="$(uname -i)" + ${{ env.args }} + - name: Create and upload logs + if: always() + run: .github/create_and_upload_logs.sh 1 + - uses: actions/upload-artifact@v3 + if: always() + with: + name: ${{ env.SUITE }}-${{ matrix.STORAGE }}-${{ inputs.arch }}-artifacts + path: ${{ env.artifact_paths}} From c0660008bb999dce089b46b183a5e52217111e9f Mon Sep 17 00:00:00 2001 From: MyroTk <44327070+MyroTk@users.noreply.github.com> Date: Thu, 28 Mar 2024 09:53:12 -0700 Subject: [PATCH 004/123] Update release_branches.yml --- .github/workflows/release_branches.yml | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/.github/workflows/release_branches.yml b/.github/workflows/release_branches.yml index 90737d222549..432d5fc7880d 100644 --- a/.github/workflows/release_branches.yml +++ b/.github/workflows/release_branches.yml @@ -576,6 +576,7 @@ jobs: ASTFuzzerTestAsan: needs: [BuilderDebAsan] uses: ./.github/workflows/reusable_test.yml + secrets: inherit with: test_name: AST fuzzer (asan) runner_type: altinity-on-demand, altinity-type-cpx51, altinity-in-ash, altinity-image-x86-app-docker-ce @@ -586,6 +587,7 @@ jobs: ASTFuzzerTestTsan: needs: [BuilderDebTsan] uses: ./.github/workflows/reusable_test.yml + secrets: inherit with: test_name: AST fuzzer (tsan) runner_type: altinity-on-demand, altinity-type-cpx51, altinity-in-ash, altinity-image-x86-app-docker-ce @@ -596,6 +598,7 @@ jobs: ASTFuzzerTestUBSan: needs: [BuilderDebUBsan] uses: ./.github/workflows/reusable_test.yml + secrets: inherit with: test_name: AST fuzzer (ubsan) runner_type: altinity-on-demand, altinity-type-cpx51, altinity-in-ash, altinity-image-x86-app-docker-ce @@ -606,6 +609,7 @@ jobs: ASTFuzzerTestMSan: needs: [BuilderDebMsan] uses: ./.github/workflows/reusable_test.yml + secrets: inherit with: test_name: AST fuzzer (msan) runner_type: altinity-on-demand, altinity-type-cpx51, altinity-in-ash, altinity-image-x86-app-docker-ce @@ -616,6 +620,7 @@ jobs: ASTFuzzerTestDebug: needs: [BuilderDebDebug] uses: ./.github/workflows/reusable_test.yml + secrets: inherit with: test_name: AST fuzzer (debug) runner_type: altinity-on-demand, altinity-type-cpx51, altinity-in-ash, altinity-image-x86-app-docker-ce From 164fb9d37b196b109cc81b1d0dec491c4b2c3fc0 Mon Sep 17 00:00:00 2001 From: MyroTk <44327070+MyroTk@users.noreply.github.com> Date: Wed, 3 Apr 2024 10:38:19 -0700 Subject: [PATCH 005/123] Update release_branches.yml --- .github/workflows/release_branches.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/release_branches.yml b/.github/workflows/release_branches.yml index 432d5fc7880d..ffc0ea5f9ded 100644 --- a/.github/workflows/release_branches.yml +++ b/.github/workflows/release_branches.yml @@ -692,7 +692,7 @@ jobs: secrets: inherit with: runner_type: altinity-on-demand, altinity-type-cpx51, altinity-image-x86-app-docker-ce, altinity-setup-regression - commit: 6da94b78dc53cb8965ab56c04a89ebf54ed04cbc + commit: b029b6a1b48cc99b8621bc32ee0f5bc26e65f002 arch: release build_sha: ${{ github.event_name == 'pull_request' && github.event.pull_request.head.sha || github.event_name == 'release' && github.sha }} @@ -702,7 +702,7 @@ jobs: secrets: inherit with: runner_type: altinity-on-demand, altinity-type-cax41, altinity-image-arm-app-docker-ce, altinity-setup-regression - commit: 6da94b78dc53cb8965ab56c04a89ebf54ed04cbc + commit: b029b6a1b48cc99b8621bc32ee0f5bc26e65f002 arch: aarch64 build_sha: ${{ github.event_name == 'pull_request' && github.event.pull_request.head.sha || github.event_name == 'release' && github.sha }} From 3e58c0bbddc9e3c210b1fb8ef5fb9d591bae2437 Mon Sep 17 00:00:00 2001 From: MyroTk <44327070+MyroTk@users.noreply.github.com> Date: Wed, 3 Apr 2024 12:40:17 -0700 Subject: [PATCH 006/123] Update images.json --- docker/images.json | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/docker/images.json b/docker/images.json index 5242ab82fc44..17fcda59c017 100644 --- a/docker/images.json +++ b/docker/images.json @@ -71,38 +71,47 @@ "dependent": [] }, "docker/test/integration/s3_proxy": { + "only_amd64": true, "name": "altinityinfra/s3-proxy", "dependent": [] }, "docker/test/integration/resolver": { + "only_amd64": true, "name": "altinityinfra/python-bottle", "dependent": [] }, "docker/test/integration/helper_container": { + "only_amd64": true, "name": "altinityinfra/integration-helper", "dependent": [] }, "docker/test/integration/mysql_golang_client": { + "only_amd64": true, "name": "altinityinfra/mysql-golang-client", "dependent": [] }, "docker/test/integration/dotnet_client": { + "only_amd64": true, "name": "altinityinfra/dotnet-client", "dependent": [] }, "docker/test/integration/mysql_java_client": { + "only_amd64": true, "name": "altinityinfra/mysql-java-client", "dependent": [] }, "docker/test/integration/mysql_js_client": { + "only_amd64": true, "name": "altinityinfra/mysql-js-client", "dependent": [] }, "docker/test/integration/mysql_php_client": { + "only_amd64": true, "name": "altinityinfra/mysql-php-client", "dependent": [] }, "docker/test/integration/postgresql_java_client": { + "only_amd64": true, "name": "altinityinfra/postgresql-java-client", "dependent": [] }, @@ -144,6 +153,7 @@ "dependent": [] }, "docker/test/integration/nginx_dav": { + "only_amd64": true, "name": "altinityinfra/nginx-dav", "dependent": [] } From fe847923709bc09583f4a332e3d8cc14da69f955 Mon Sep 17 00:00:00 2001 From: MyroTk <44327070+MyroTk@users.noreply.github.com> Date: Wed, 3 Apr 2024 14:58:29 -0700 Subject: [PATCH 007/123] Update images.json --- docker/images.json | 10 ---------- 1 file changed, 10 deletions(-) diff --git a/docker/images.json b/docker/images.json index 17fcda59c017..5242ab82fc44 100644 --- a/docker/images.json +++ b/docker/images.json @@ -71,47 +71,38 @@ "dependent": [] }, "docker/test/integration/s3_proxy": { - "only_amd64": true, "name": "altinityinfra/s3-proxy", "dependent": [] }, "docker/test/integration/resolver": { - "only_amd64": true, "name": "altinityinfra/python-bottle", "dependent": [] }, "docker/test/integration/helper_container": { - "only_amd64": true, "name": "altinityinfra/integration-helper", "dependent": [] }, "docker/test/integration/mysql_golang_client": { - "only_amd64": true, "name": "altinityinfra/mysql-golang-client", "dependent": [] }, "docker/test/integration/dotnet_client": { - "only_amd64": true, "name": "altinityinfra/dotnet-client", "dependent": [] }, "docker/test/integration/mysql_java_client": { - "only_amd64": true, "name": "altinityinfra/mysql-java-client", "dependent": [] }, "docker/test/integration/mysql_js_client": { - "only_amd64": true, "name": "altinityinfra/mysql-js-client", "dependent": [] }, "docker/test/integration/mysql_php_client": { - "only_amd64": true, "name": "altinityinfra/mysql-php-client", "dependent": [] }, "docker/test/integration/postgresql_java_client": { - "only_amd64": true, "name": "altinityinfra/postgresql-java-client", "dependent": [] }, @@ -153,7 +144,6 @@ "dependent": [] }, "docker/test/integration/nginx_dav": { - "only_amd64": true, "name": "altinityinfra/nginx-dav", "dependent": [] } From 2670a444df9025629d98078197cc31f98f3ef469 Mon Sep 17 00:00:00 2001 From: MyroTk <44327070+MyroTk@users.noreply.github.com> Date: Wed, 3 Apr 2024 19:04:10 -0700 Subject: [PATCH 008/123] Update release_branches.yml --- .github/workflows/release_branches.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.github/workflows/release_branches.yml b/.github/workflows/release_branches.yml index ffc0ea5f9ded..8b36e6b15b3b 100644 --- a/.github/workflows/release_branches.yml +++ b/.github/workflows/release_branches.yml @@ -65,6 +65,8 @@ jobs: env: RUNNER_IP: "$(hostname -I | cut -d ' ' -f 1)" RUNNER_SSH_COMMAND: "ssh -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null root@$(hostname -I | cut -d ' ' -f 1)" + + - run: sleep 3600 - name: Upload images files to artifacts uses: actions/upload-artifact@v3 From f534e558016dbdc640b5962b0b89729b88df9a7d Mon Sep 17 00:00:00 2001 From: MyroTk <44327070+MyroTk@users.noreply.github.com> Date: Wed, 3 Apr 2024 20:39:33 -0700 Subject: [PATCH 009/123] Update release_branches.yml --- .github/workflows/release_branches.yml | 2 -- 1 file changed, 2 deletions(-) diff --git a/.github/workflows/release_branches.yml b/.github/workflows/release_branches.yml index 8b36e6b15b3b..91f2b9c72993 100644 --- a/.github/workflows/release_branches.yml +++ b/.github/workflows/release_branches.yml @@ -66,8 +66,6 @@ jobs: RUNNER_IP: "$(hostname -I | cut -d ' ' -f 1)" RUNNER_SSH_COMMAND: "ssh -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null root@$(hostname -I | cut -d ' ' -f 1)" - - run: sleep 3600 - - name: Upload images files to artifacts uses: actions/upload-artifact@v3 with: From 6357cba09107ae9e39eaaaec881ef0e12130ce33 Mon Sep 17 00:00:00 2001 From: MyroTk <44327070+MyroTk@users.noreply.github.com> Date: Thu, 4 Apr 2024 08:24:01 -0700 Subject: [PATCH 010/123] Remove Fuzzers --- .github/workflows/release_branches.yml | 58 -------------------------- 1 file changed, 58 deletions(-) diff --git a/.github/workflows/release_branches.yml b/.github/workflows/release_branches.yml index 91f2b9c72993..ba13064b03a2 100644 --- a/.github/workflows/release_branches.yml +++ b/.github/workflows/release_branches.yml @@ -570,64 +570,6 @@ jobs: cd "$REPO_COPY/tests/ci" python3 stress_check.py "$CHECK_NAME" -############################################################################################## -##################################### AST FUZZERS ############################################ -############################################################################################## - ASTFuzzerTestAsan: - needs: [BuilderDebAsan] - uses: ./.github/workflows/reusable_test.yml - secrets: inherit - with: - test_name: AST fuzzer (asan) - runner_type: altinity-on-demand, altinity-type-cpx51, altinity-in-ash, altinity-image-x86-app-docker-ce - run_command: | - cd "$REPO_COPY/tests/ci" - python3 ast_fuzzer_check.py "$CHECK_NAME" - - ASTFuzzerTestTsan: - needs: [BuilderDebTsan] - uses: ./.github/workflows/reusable_test.yml - secrets: inherit - with: - test_name: AST fuzzer (tsan) - runner_type: altinity-on-demand, altinity-type-cpx51, altinity-in-ash, altinity-image-x86-app-docker-ce - run_command: | - cd "$REPO_COPY/tests/ci" - python3 ast_fuzzer_check.py "$CHECK_NAME" - - ASTFuzzerTestUBSan: - needs: [BuilderDebUBsan] - uses: ./.github/workflows/reusable_test.yml - secrets: inherit - with: - test_name: AST fuzzer (ubsan) - runner_type: altinity-on-demand, altinity-type-cpx51, altinity-in-ash, altinity-image-x86-app-docker-ce - run_command: | - cd "$REPO_COPY/tests/ci" - python3 ast_fuzzer_check.py "$CHECK_NAME" - - ASTFuzzerTestMSan: - needs: [BuilderDebMsan] - uses: ./.github/workflows/reusable_test.yml - secrets: inherit - with: - test_name: AST fuzzer (msan) - runner_type: altinity-on-demand, altinity-type-cpx51, altinity-in-ash, altinity-image-x86-app-docker-ce - run_command: | - cd "$REPO_COPY/tests/ci" - python3 ast_fuzzer_check.py "$CHECK_NAME" - - ASTFuzzerTestDebug: - needs: [BuilderDebDebug] - uses: ./.github/workflows/reusable_test.yml - secrets: inherit - with: - test_name: AST fuzzer (debug) - runner_type: altinity-on-demand, altinity-type-cpx51, altinity-in-ash, altinity-image-x86-app-docker-ce - run_command: | - cd "$REPO_COPY/tests/ci" - python3 ast_fuzzer_check.py "$CHECK_NAME" - ############################################################################################# ############################# INTEGRATION TESTS ############################################# ############################################################################################# From a6e0c6e363e23f7a0ca9396380fd51ae7847c883 Mon Sep 17 00:00:00 2001 From: MyroTk <44327070+MyroTk@users.noreply.github.com> Date: Thu, 4 Apr 2024 10:21:23 -0700 Subject: [PATCH 011/123] Update release_branches.yml --- .github/workflows/release_branches.yml | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/.github/workflows/release_branches.yml b/.github/workflows/release_branches.yml index ba13064b03a2..a4621a1307ee 100644 --- a/.github/workflows/release_branches.yml +++ b/.github/workflows/release_branches.yml @@ -146,7 +146,7 @@ jobs: build_name: package_release checkout_depth: 0 timeout_minutes: 180 - runner_type: altinity-on-demand, altinity-type-ccx53, altinity-in-ash, altinity-image-x86-app-docker-ce + runner_type: altinity-on-demand, altinity-builder, altinity-type-ccx53, altinity-in-ash, altinity-image-x86-app-docker-ce additional_envs: | CLICKHOUSE_STABLE_VERSION_SUFFIX=altinitystable @@ -158,7 +158,7 @@ jobs: build_name: package_aarch64 checkout_depth: 0 timeout_minutes: 180 - runner_type: altinity-on-demand, altinity-type-ccx53, altinity-in-ash, altinity-image-x86-app-docker-ce + runner_type: altinity-on-demand, altinity-builder, altinity-type-ccx53, altinity-in-ash, altinity-image-x86-app-docker-ce additional_envs: | CLICKHOUSE_STABLE_VERSION_SUFFIX=altinitystable @@ -170,7 +170,7 @@ jobs: build_name: package_asan checkout_depth: 0 timeout_minutes: 180 - runner_type: altinity-on-demand, altinity-type-ccx53, altinity-in-ash, altinity-image-x86-app-docker-ce + runner_type: altinity-on-demand, altinity-builder, altinity-type-ccx53, altinity-in-ash, altinity-image-x86-app-docker-ce additional_envs: | CLICKHOUSE_STABLE_VERSION_SUFFIX=altinitystable @@ -182,7 +182,7 @@ jobs: build_name: package_ubsan checkout_depth: 0 timeout_minutes: 180 - runner_type: altinity-on-demand, altinity-type-ccx53, altinity-in-ash, altinity-image-x86-app-docker-ce + runner_type: altinity-on-demand, altinity-builder, altinity-type-ccx53, altinity-in-ash, altinity-image-x86-app-docker-ce additional_envs: | CLICKHOUSE_STABLE_VERSION_SUFFIX=altinitystable @@ -194,7 +194,7 @@ jobs: build_name: package_tsan checkout_depth: 0 timeout_minutes: 180 - runner_type: altinity-on-demand, altinity-type-ccx53, altinity-in-ash, altinity-image-x86-app-docker-ce + runner_type: altinity-on-demand, altinity-builder, altinity-type-ccx53, altinity-in-ash, altinity-image-x86-app-docker-ce additional_envs: | CLICKHOUSE_STABLE_VERSION_SUFFIX=altinitystable @@ -206,7 +206,7 @@ jobs: build_name: package_msan checkout_depth: 0 timeout_minutes: 180 - runner_type: altinity-on-demand, altinity-type-ccx53, altinity-in-ash, altinity-image-x86-app-docker-ce + runner_type: altinity-on-demand, altinity-builder, altinity-type-ccx53, altinity-in-ash, altinity-image-x86-app-docker-ce additional_envs: | CLICKHOUSE_STABLE_VERSION_SUFFIX=altinitystable @@ -218,7 +218,7 @@ jobs: build_name: package_debug checkout_depth: 0 timeout_minutes: 180 - runner_type: altinity-on-demand, altinity-type-ccx53, altinity-in-ash, altinity-image-x86-app-docker-ce + runner_type: altinity-on-demand, altinity-builder, altinity-type-ccx53, altinity-in-ash, altinity-image-x86-app-docker-ce additional_envs: | CLICKHOUSE_STABLE_VERSION_SUFFIX=altinitystable From 0f433f4361571de6b2d24ef04e4b4cfff66ce283 Mon Sep 17 00:00:00 2001 From: MyroTk <44327070+MyroTk@users.noreply.github.com> Date: Thu, 4 Apr 2024 18:43:22 -0700 Subject: [PATCH 012/123] Update sign_release.py --- tests/ci/sign_release.py | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/tests/ci/sign_release.py b/tests/ci/sign_release.py index 701ccc29b65b..872966a578a5 100644 --- a/tests/ci/sign_release.py +++ b/tests/ci/sign_release.py @@ -7,6 +7,7 @@ from pr_info import PRInfo from build_download_helper import download_builds_filter import hashlib +from pathlib import Path GPG_BINARY_SIGNING_KEY = os.getenv("GPG_BINARY_SIGNING_KEY") GPG_BINARY_SIGNING_PASSPHRASE = os.getenv("GPG_BINARY_SIGNING_PASSPHRASE") @@ -57,9 +58,9 @@ def main(): s3_helper = S3Helper() - s3_path_prefix = f"{pr_info.number}/{pr_info.sha}/" + CHECK_NAME.lower().replace( + s3_path_prefix = Path(f"{pr_info.number}/{pr_info.sha}/" + CHECK_NAME.lower().replace( " ", "_" - ).replace("(", "_").replace(")", "_").replace(",", "_") + ).replace("(", "_").replace(")", "_").replace(",", "_")) # downloads `package_release` artifacts generated download_builds_filter(CHECK_NAME, reports_path, TEMP_PATH) @@ -68,8 +69,8 @@ def main(): full_path = os.path.join(TEMP_PATH, f) hashed_file_path = hash_file(full_path) signed_file_path = sign_file(hashed_file_path) - s3_path = f'{s3_path_prefix}/{os.path.basename(signed_file_path)}' - s3_helper.upload_build_file_to_s3(signed_file_path, s3_path) + s3_path = s3_path_prefix / os.path.basename(signed_file_path) + s3_helper.upload_build_file_to_s3(Path(signed_file_path), str(s3_path)) print(f'Uploaded file {signed_file_path} to {s3_path}') # Signed hashes are: From 5335c35e7c7ac21c68819088a0fa87b0649021ce Mon Sep 17 00:00:00 2001 From: MyroTk <44327070+MyroTk@users.noreply.github.com> Date: Fri, 5 Apr 2024 08:33:04 -0700 Subject: [PATCH 013/123] Update regression.yml --- .github/workflows/regression.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.github/workflows/regression.yml b/.github/workflows/regression.yml index cc89340617a6..170e89b27b00 100644 --- a/.github/workflows/regression.yml +++ b/.github/workflows/regression.yml @@ -16,7 +16,7 @@ name: Regression test workflow type: string timeout_minutes: description: Maximum number of minutes to let workflow run before GitHub cancels it. - default: 210 + default: 240 type: number build_sha: description: commit sha of the workflow run for artifact upload. @@ -394,6 +394,7 @@ jobs: run: python3 -u ${{ env.SUITE }}/regression.py --clickhouse-binary-path ${{ env.clickhouse_binary_path }} + --only "/parquet/${{ matrix.STORAGE }}/*" --storage ${{ matrix.STORAGE }} --aws-s3-bucket ${{ secrets.REGRESSION_AWS_S3_BUCKET }} --aws-s3-region ${{ secrets.REGRESSION_AWS_S3_REGION }} From d0401db5ed418d83dd9ec190d70113d4bbb38d50 Mon Sep 17 00:00:00 2001 From: MyroTk <44327070+MyroTk@users.noreply.github.com> Date: Fri, 5 Apr 2024 08:33:36 -0700 Subject: [PATCH 014/123] Update release_branches.yml --- .github/workflows/release_branches.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/release_branches.yml b/.github/workflows/release_branches.yml index a4621a1307ee..5e84e4a12fc3 100644 --- a/.github/workflows/release_branches.yml +++ b/.github/workflows/release_branches.yml @@ -634,7 +634,7 @@ jobs: secrets: inherit with: runner_type: altinity-on-demand, altinity-type-cpx51, altinity-image-x86-app-docker-ce, altinity-setup-regression - commit: b029b6a1b48cc99b8621bc32ee0f5bc26e65f002 + commit: 55cf702a8e77ef361084fb017d2ef072952d6367 arch: release build_sha: ${{ github.event_name == 'pull_request' && github.event.pull_request.head.sha || github.event_name == 'release' && github.sha }} @@ -644,7 +644,7 @@ jobs: secrets: inherit with: runner_type: altinity-on-demand, altinity-type-cax41, altinity-image-arm-app-docker-ce, altinity-setup-regression - commit: b029b6a1b48cc99b8621bc32ee0f5bc26e65f002 + commit: 55cf702a8e77ef361084fb017d2ef072952d6367 arch: aarch64 build_sha: ${{ github.event_name == 'pull_request' && github.event.pull_request.head.sha || github.event_name == 'release' && github.sha }} From 709f6de6b6bd4f884d1453f7f2f009053143f5d9 Mon Sep 17 00:00:00 2001 From: MyroTk <44327070+MyroTk@users.noreply.github.com> Date: Fri, 5 Apr 2024 12:43:05 -0700 Subject: [PATCH 015/123] Update release_branches.yml --- .github/workflows/release_branches.yml | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/.github/workflows/release_branches.yml b/.github/workflows/release_branches.yml index 5e84e4a12fc3..9169a23ebc3f 100644 --- a/.github/workflows/release_branches.yml +++ b/.github/workflows/release_branches.yml @@ -146,7 +146,7 @@ jobs: build_name: package_release checkout_depth: 0 timeout_minutes: 180 - runner_type: altinity-on-demand, altinity-builder, altinity-type-ccx53, altinity-in-ash, altinity-image-x86-app-docker-ce + runner_type: altinity-on-demand, altinity-setup-builder, altinity-type-ccx53, altinity-in-ash, altinity-image-x86-app-docker-ce additional_envs: | CLICKHOUSE_STABLE_VERSION_SUFFIX=altinitystable @@ -158,7 +158,7 @@ jobs: build_name: package_aarch64 checkout_depth: 0 timeout_minutes: 180 - runner_type: altinity-on-demand, altinity-builder, altinity-type-ccx53, altinity-in-ash, altinity-image-x86-app-docker-ce + runner_type: altinity-on-demand, altinity-setup-builder, altinity-type-ccx53, altinity-in-ash, altinity-image-x86-app-docker-ce additional_envs: | CLICKHOUSE_STABLE_VERSION_SUFFIX=altinitystable @@ -170,7 +170,7 @@ jobs: build_name: package_asan checkout_depth: 0 timeout_minutes: 180 - runner_type: altinity-on-demand, altinity-builder, altinity-type-ccx53, altinity-in-ash, altinity-image-x86-app-docker-ce + runner_type: altinity-on-demand, altinity-setup-builder, altinity-type-ccx53, altinity-in-ash, altinity-image-x86-app-docker-ce additional_envs: | CLICKHOUSE_STABLE_VERSION_SUFFIX=altinitystable @@ -182,7 +182,7 @@ jobs: build_name: package_ubsan checkout_depth: 0 timeout_minutes: 180 - runner_type: altinity-on-demand, altinity-builder, altinity-type-ccx53, altinity-in-ash, altinity-image-x86-app-docker-ce + runner_type: altinity-on-demand, altinity-setup-builder, altinity-type-ccx53, altinity-in-ash, altinity-image-x86-app-docker-ce additional_envs: | CLICKHOUSE_STABLE_VERSION_SUFFIX=altinitystable @@ -194,7 +194,7 @@ jobs: build_name: package_tsan checkout_depth: 0 timeout_minutes: 180 - runner_type: altinity-on-demand, altinity-builder, altinity-type-ccx53, altinity-in-ash, altinity-image-x86-app-docker-ce + runner_type: altinity-on-demand, altinity-setup-builder, altinity-type-ccx53, altinity-in-ash, altinity-image-x86-app-docker-ce additional_envs: | CLICKHOUSE_STABLE_VERSION_SUFFIX=altinitystable @@ -206,7 +206,7 @@ jobs: build_name: package_msan checkout_depth: 0 timeout_minutes: 180 - runner_type: altinity-on-demand, altinity-builder, altinity-type-ccx53, altinity-in-ash, altinity-image-x86-app-docker-ce + runner_type: altinity-on-demand, altinity-setup-builder, altinity-type-ccx53, altinity-in-ash, altinity-image-x86-app-docker-ce additional_envs: | CLICKHOUSE_STABLE_VERSION_SUFFIX=altinitystable @@ -218,7 +218,7 @@ jobs: build_name: package_debug checkout_depth: 0 timeout_minutes: 180 - runner_type: altinity-on-demand, altinity-builder, altinity-type-ccx53, altinity-in-ash, altinity-image-x86-app-docker-ce + runner_type: altinity-on-demand, altinity-setup-builder, altinity-type-ccx53, altinity-in-ash, altinity-image-x86-app-docker-ce additional_envs: | CLICKHOUSE_STABLE_VERSION_SUFFIX=altinitystable From 1fb484d6a7a0f2716a4a092deba98125c2639265 Mon Sep 17 00:00:00 2001 From: MyroTk <44327070+MyroTk@users.noreply.github.com> Date: Fri, 5 Apr 2024 12:44:02 -0700 Subject: [PATCH 016/123] Update cancel.yml --- .github/workflows/cancel.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/cancel.yml b/.github/workflows/cancel.yml index 3c2be767ad22..69d1d5e9f392 100644 --- a/.github/workflows/cancel.yml +++ b/.github/workflows/cancel.yml @@ -11,7 +11,7 @@ on: # yamllint disable-line rule:truthy - requested jobs: cancel: - runs-on: [self-hosted, style-checker] + runs-on: [self-hosted, altinity-on-demand, altinity-type-cx11, altinity-setup-none] steps: - uses: styfle/cancel-workflow-action@0.9.1 with: From c9f49bc625a4aae5b07c3cb0d572ea1290f5e7d6 Mon Sep 17 00:00:00 2001 From: MyroTk <44327070+MyroTk@users.noreply.github.com> Date: Mon, 8 Apr 2024 11:59:02 -0700 Subject: [PATCH 017/123] Update release_branches.yml --- .github/workflows/release_branches.yml | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/.github/workflows/release_branches.yml b/.github/workflows/release_branches.yml index 9169a23ebc3f..9005a0dac14e 100644 --- a/.github/workflows/release_branches.yml +++ b/.github/workflows/release_branches.yml @@ -170,7 +170,7 @@ jobs: build_name: package_asan checkout_depth: 0 timeout_minutes: 180 - runner_type: altinity-on-demand, altinity-setup-builder, altinity-type-ccx53, altinity-in-ash, altinity-image-x86-app-docker-ce + runner_type: altinity-on-demand, altinity-setup-builder, altinity-type-ccx63, altinity-in-ash, altinity-image-x86-app-docker-ce additional_envs: | CLICKHOUSE_STABLE_VERSION_SUFFIX=altinitystable @@ -182,7 +182,7 @@ jobs: build_name: package_ubsan checkout_depth: 0 timeout_minutes: 180 - runner_type: altinity-on-demand, altinity-setup-builder, altinity-type-ccx53, altinity-in-ash, altinity-image-x86-app-docker-ce + runner_type: altinity-on-demand, altinity-setup-builder, altinity-type-ccx63, altinity-in-ash, altinity-image-x86-app-docker-ce additional_envs: | CLICKHOUSE_STABLE_VERSION_SUFFIX=altinitystable @@ -194,7 +194,7 @@ jobs: build_name: package_tsan checkout_depth: 0 timeout_minutes: 180 - runner_type: altinity-on-demand, altinity-setup-builder, altinity-type-ccx53, altinity-in-ash, altinity-image-x86-app-docker-ce + runner_type: altinity-on-demand, altinity-setup-builder, altinity-type-ccx63, altinity-in-ash, altinity-image-x86-app-docker-ce additional_envs: | CLICKHOUSE_STABLE_VERSION_SUFFIX=altinitystable @@ -206,7 +206,7 @@ jobs: build_name: package_msan checkout_depth: 0 timeout_minutes: 180 - runner_type: altinity-on-demand, altinity-setup-builder, altinity-type-ccx53, altinity-in-ash, altinity-image-x86-app-docker-ce + runner_type: altinity-on-demand, altinity-setup-builder, altinity-type-ccx63, altinity-in-ash, altinity-image-x86-app-docker-ce additional_envs: | CLICKHOUSE_STABLE_VERSION_SUFFIX=altinitystable From a4c901e0a566bf22701db76c3026086f62f71530 Mon Sep 17 00:00:00 2001 From: MyroTk <44327070+MyroTk@users.noreply.github.com> Date: Mon, 8 Apr 2024 20:04:54 -0700 Subject: [PATCH 018/123] Update release_branches.yml --- .github/workflows/release_branches.yml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/.github/workflows/release_branches.yml b/.github/workflows/release_branches.yml index 9005a0dac14e..a3cee1f63783 100644 --- a/.github/workflows/release_branches.yml +++ b/.github/workflows/release_branches.yml @@ -522,6 +522,7 @@ jobs: with: test_name: Stress test (asan) runner_type: altinity-on-demand, altinity-type-cpx51, altinity-in-ash, altinity-image-x86-snapshot-docker_ipv6_x86 + timeout_minutes: 180 run_command: | cd "$REPO_COPY/tests/ci" python3 stress_check.py "$CHECK_NAME" @@ -533,6 +534,7 @@ jobs: with: test_name: Stress test (tsan) runner_type: altinity-on-demand, altinity-type-cpx51, altinity-in-ash, altinity-image-x86-snapshot-docker_ipv6_x86 + timeout_minutes: 180 run_command: | cd "$REPO_COPY/tests/ci" python3 stress_check.py "$CHECK_NAME" @@ -555,6 +557,7 @@ jobs: with: test_name: Stress test (ubsan) runner_type: altinity-on-demand, altinity-type-cpx51, altinity-in-ash, altinity-image-x86-snapshot-docker_ipv6_x86 + timeout_minutes: 180 run_command: | cd "$REPO_COPY/tests/ci" python3 stress_check.py "$CHECK_NAME" @@ -566,6 +569,7 @@ jobs: with: test_name: Stress test (debug) runner_type: altinity-on-demand, altinity-type-cpx51, altinity-in-ash, altinity-image-x86-snapshot-docker_ipv6_x86 + timeout_minutes: 180 run_command: | cd "$REPO_COPY/tests/ci" python3 stress_check.py "$CHECK_NAME" From 04f5e9df9ca6b4ca2e0f7a8a0b86449914eb51da Mon Sep 17 00:00:00 2001 From: MyroTk <44327070+MyroTk@users.noreply.github.com> Date: Tue, 9 Apr 2024 09:40:53 -0700 Subject: [PATCH 019/123] Update build_check.py --- tests/ci/build_check.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/ci/build_check.py b/tests/ci/build_check.py index e0bd37181a34..2ea3ae75a0ee 100644 --- a/tests/ci/build_check.py +++ b/tests/ci/build_check.py @@ -491,7 +491,7 @@ def main(): log_url, f"Build ({build_name})", ) - ch_helper.insert_events_into(db="default", table="checks", events=prepared_events) + ch_helper.insert_events_into(db="gh-data", table="checks", events=prepared_events) # Fail the build job if it didn't succeed if build_status != SUCCESS: From 31519f5775b18b0a2430688f7b59695cadb7fece Mon Sep 17 00:00:00 2001 From: MyroTk <44327070+MyroTk@users.noreply.github.com> Date: Tue, 9 Apr 2024 10:06:33 -0700 Subject: [PATCH 020/123] Update build caching --- tests/ci/build_check.py | 34 +++------------------------------- 1 file changed, 3 insertions(+), 31 deletions(-) diff --git a/tests/ci/build_check.py b/tests/ci/build_check.py index 2ea3ae75a0ee..c4039763a2ca 100644 --- a/tests/ci/build_check.py +++ b/tests/ci/build_check.py @@ -12,12 +12,13 @@ from ccache_utils import CargoCache from docker_pull_helper import get_image_with_version from env_helper import ( - CACHES_PATH, GITHUB_JOB_API_URL, IMAGES_PATH, REPO_COPY, + S3_ACCESS_KEY_ID, S3_BUILDS_BUCKET, S3_DOWNLOAD, + S3_SECRET_ACCESS_KEY, TEMP_PATH, CLICKHOUSE_STABLE_VERSION_SUFFIX, ) @@ -38,8 +39,6 @@ get_instance_type, ) from stopwatch import Stopwatch -from ccache_utils import get_ccache_if_not_exists, upload_ccache - IMAGE_NAME = "altinityinfra/binary-builder" BUILD_LOG_NAME = "build_log.log" @@ -62,7 +61,6 @@ def get_packager_cmd( cargo_cache_dir: Path, build_version: str, image_version: str, - ccache_path: str, official: bool, ) -> str: package_type = build_config.package_type @@ -80,9 +78,7 @@ def get_packager_cmd( if build_config.tidy: cmd += " --clang-tidy" - # NOTE(vnemkov): we are going to continue to use ccache for now - cmd += " --cache=ccache" - cmd += f" --ccache-dir={ccache_path}" + cmd += " --cache=sccache" cmd += " --s3-rw-access" cmd += f" --s3-bucket={S3_BUILDS_BUCKET}" cmd += f" --cargo-cache-dir={cargo_cache_dir}" @@ -278,24 +274,6 @@ def main(): ) cargo_cache.download() - # NOTE(vnemkov): since we still want to use CCACHE over SCCACHE, unlike upstream, - # we need to create local directory for that, just as with 22.8 - ccache_path = Path(CACHES_PATH, build_name + "_ccache") - - logging.info("Will try to fetch cache for our build") - try: - get_ccache_if_not_exists( - ccache_path, s3_helper, pr_info.number, temp_path, pr_info.release_pr - ) - except Exception as e: - # In case there are issues with ccache, remove the path and do not fail a build - logging.info("Failed to get ccache, building without it. Error: %s", e) - rmtree(ccache_path, ignore_errors=True) - - if not ccache_path.exists(): - logging.info("cache was not fetched, will create empty dir") - ccache_path.mkdir(parents=True) - packager_cmd = get_packager_cmd( build_config, repo_path / "docker" / "packager", @@ -303,7 +281,6 @@ def main(): cargo_cache.directory, version.string, image_version, - ccache_path, official_flag, ) @@ -320,7 +297,6 @@ def main(): subprocess.check_call( f"sudo chown -R ubuntu:ubuntu {build_output_path}", shell=True ) - subprocess.check_call(f"sudo chown -R ubuntu:ubuntu {ccache_path}", shell=True) logging.info("Build finished as %s, log path %s", build_status, log_path) if build_status == SUCCESS: cargo_cache.upload() @@ -334,10 +310,6 @@ def main(): ) sys.exit(1) - # Upload the ccache first to have the least build time in case of problems - logging.info("Will upload cache") - upload_ccache(ccache_path, s3_helper, pr_info.number, temp_path) - # FIXME performance performance_urls = [] performance_path = build_output_path / "performance.tar.zst" From 729d9448048dda867f7cc504e4fbe917c69d625d Mon Sep 17 00:00:00 2001 From: MyroTk <44327070+MyroTk@users.noreply.github.com> Date: Tue, 9 Apr 2024 10:59:43 -0700 Subject: [PATCH 021/123] Update cancel.yml --- .github/workflows/cancel.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/cancel.yml b/.github/workflows/cancel.yml index 69d1d5e9f392..46ff5794b5ba 100644 --- a/.github/workflows/cancel.yml +++ b/.github/workflows/cancel.yml @@ -11,7 +11,7 @@ on: # yamllint disable-line rule:truthy - requested jobs: cancel: - runs-on: [self-hosted, altinity-on-demand, altinity-type-cx11, altinity-setup-none] + runs-on: ubuntu-latest steps: - uses: styfle/cancel-workflow-action@0.9.1 with: From f4183233e52ea1e702d4970ae96cf90dc0db4f63 Mon Sep 17 00:00:00 2001 From: MyroTk <44327070+MyroTk@users.noreply.github.com> Date: Tue, 9 Apr 2024 11:00:20 -0700 Subject: [PATCH 022/123] Update release_branches.yml --- .github/workflows/release_branches.yml | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/.github/workflows/release_branches.yml b/.github/workflows/release_branches.yml index a3cee1f63783..e607d9cd9e14 100644 --- a/.github/workflows/release_branches.yml +++ b/.github/workflows/release_branches.yml @@ -170,7 +170,7 @@ jobs: build_name: package_asan checkout_depth: 0 timeout_minutes: 180 - runner_type: altinity-on-demand, altinity-setup-builder, altinity-type-ccx63, altinity-in-ash, altinity-image-x86-app-docker-ce + runner_type: altinity-on-demand, altinity-setup-builder, altinity-type-ccx53, altinity-in-ash, altinity-image-x86-app-docker-ce additional_envs: | CLICKHOUSE_STABLE_VERSION_SUFFIX=altinitystable @@ -182,7 +182,7 @@ jobs: build_name: package_ubsan checkout_depth: 0 timeout_minutes: 180 - runner_type: altinity-on-demand, altinity-setup-builder, altinity-type-ccx63, altinity-in-ash, altinity-image-x86-app-docker-ce + runner_type: altinity-on-demand, altinity-setup-builder, altinity-type-ccx53, altinity-in-ash, altinity-image-x86-app-docker-ce additional_envs: | CLICKHOUSE_STABLE_VERSION_SUFFIX=altinitystable @@ -194,7 +194,7 @@ jobs: build_name: package_tsan checkout_depth: 0 timeout_minutes: 180 - runner_type: altinity-on-demand, altinity-setup-builder, altinity-type-ccx63, altinity-in-ash, altinity-image-x86-app-docker-ce + runner_type: altinity-on-demand, altinity-setup-builder, altinity-type-ccx53, altinity-in-ash, altinity-image-x86-app-docker-ce additional_envs: | CLICKHOUSE_STABLE_VERSION_SUFFIX=altinitystable @@ -206,7 +206,7 @@ jobs: build_name: package_msan checkout_depth: 0 timeout_minutes: 180 - runner_type: altinity-on-demand, altinity-setup-builder, altinity-type-ccx63, altinity-in-ash, altinity-image-x86-app-docker-ce + runner_type: altinity-on-demand, altinity-setup-builder, altinity-type-ccx53, altinity-in-ash, altinity-image-x86-app-docker-ce additional_envs: | CLICKHOUSE_STABLE_VERSION_SUFFIX=altinitystable From a57f5e8138af99e6824324290e12ebaa367109e6 Mon Sep 17 00:00:00 2001 From: MyroTk <44327070+MyroTk@users.noreply.github.com> Date: Tue, 9 Apr 2024 12:13:16 -0700 Subject: [PATCH 023/123] Update env_helper.py --- tests/ci/env_helper.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/tests/ci/env_helper.py b/tests/ci/env_helper.py index 08952ed3179d..38592706c1c6 100644 --- a/tests/ci/env_helper.py +++ b/tests/ci/env_helper.py @@ -26,7 +26,9 @@ REPORTS_PATH = os.getenv("REPORTS_PATH", p.abspath(p.join(module_dir, "./reports"))) REPO_COPY = os.getenv("REPO_COPY", GITHUB_WORKSPACE) RUNNER_TEMP = os.getenv("RUNNER_TEMP", p.abspath(p.join(module_dir, "./tmp"))) +S3_ACCESS_KEY_ID = os.getenv("AWS_ACCESS_KEY_ID") S3_BUILDS_BUCKET = os.getenv("S3_BUILDS_BUCKET", "altinity-build-artifacts") +S3_SECRET_ACCESS_KEY = os.getenv("AWS_SECRET_ACCESS_KEY") S3_TEST_REPORTS_BUCKET = os.getenv("S3_TEST_REPORTS_BUCKET", "altinity-build-artifacts") S3_URL = os.getenv("S3_URL", "https://s3.amazonaws.com") CLICKHOUSE_STABLE_VERSION_SUFFIX = os.getenv("CLICKHOUSE_STABLE_VERSION_SUFFIX", "stable") From 6a2fe92826fb898381691c4598a54e392f7ce9f8 Mon Sep 17 00:00:00 2001 From: MyroTk <44327070+MyroTk@users.noreply.github.com> Date: Tue, 9 Apr 2024 13:01:21 -0700 Subject: [PATCH 024/123] Update build_check.py --- tests/ci/build_check.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/tests/ci/build_check.py b/tests/ci/build_check.py index c4039763a2ca..4a1517a63ede 100644 --- a/tests/ci/build_check.py +++ b/tests/ci/build_check.py @@ -82,6 +82,8 @@ def get_packager_cmd( cmd += " --s3-rw-access" cmd += f" --s3-bucket={S3_BUILDS_BUCKET}" cmd += f" --cargo-cache-dir={cargo_cache_dir}" + cmd += f" --s3-access-key-id={S3_ACCESS_KEY_ID}" + cmd += f" --s3-secret-access-key={S3_SECRET_ACCESS_KEY}" if build_config.additional_pkgs: cmd += " --additional-pkgs" From e9744ecba7074a4464fed4afbc7871fec505a920 Mon Sep 17 00:00:00 2001 From: MyroTk <44327070+MyroTk@users.noreply.github.com> Date: Tue, 9 Apr 2024 13:03:24 -0700 Subject: [PATCH 025/123] Update release_branches.yml --- .github/workflows/release_branches.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/release_branches.yml b/.github/workflows/release_branches.yml index e607d9cd9e14..36473201c37c 100644 --- a/.github/workflows/release_branches.yml +++ b/.github/workflows/release_branches.yml @@ -638,7 +638,7 @@ jobs: secrets: inherit with: runner_type: altinity-on-demand, altinity-type-cpx51, altinity-image-x86-app-docker-ce, altinity-setup-regression - commit: 55cf702a8e77ef361084fb017d2ef072952d6367 + commit: ed5f2f0d9f33f46eb8962c2f2d974701c86f187e arch: release build_sha: ${{ github.event_name == 'pull_request' && github.event.pull_request.head.sha || github.event_name == 'release' && github.sha }} @@ -648,7 +648,7 @@ jobs: secrets: inherit with: runner_type: altinity-on-demand, altinity-type-cax41, altinity-image-arm-app-docker-ce, altinity-setup-regression - commit: 55cf702a8e77ef361084fb017d2ef072952d6367 + commit: ed5f2f0d9f33f46eb8962c2f2d974701c86f187e arch: aarch64 build_sha: ${{ github.event_name == 'pull_request' && github.event.pull_request.head.sha || github.event_name == 'release' && github.sha }} From 3487bb9c20cd50e6d4e1c98a0ccbbae3ea3e163b Mon Sep 17 00:00:00 2001 From: MyroTk <44327070+MyroTk@users.noreply.github.com> Date: Tue, 9 Apr 2024 22:45:33 -0700 Subject: [PATCH 026/123] Update packager --- docker/packager/packager | 17 ++++++++++++++++- 1 file changed, 16 insertions(+), 1 deletion(-) diff --git a/docker/packager/packager b/docker/packager/packager index c8025b64fc54..72bb8b565c24 100755 --- a/docker/packager/packager +++ b/docker/packager/packager @@ -125,9 +125,11 @@ def parse_env_variables( sanitizer: str, package_type: str, cache: str, + s3_access_key_id: str, s3_bucket: str, s3_directory: str, s3_rw_access: bool, + s3_secret_access_key: str, clang_tidy: bool, version: str, official: bool, @@ -294,6 +296,10 @@ def parse_env_variables( result.append(f"SCCACHE_S3_KEY_PREFIX={sccache_dir}") if not s3_rw_access: result.append("SCCACHE_S3_NO_CREDENTIALS=true") + if s3_access_key_id: + result.append(f"AWS_ACCESS_KEY_ID={s3_access_key_id}") + if s3_secret_access_key: + result.append(f"AWS_SECRET_ACCESS_KEY={s3_secret_access_key}") if clang_tidy: # `CTCACHE_DIR` has the same purpose as the `CCACHE_DIR` above. @@ -413,6 +419,14 @@ def parse_args() -> argparse.Namespace: type=dir_name, help="a directory with ccache", ) + parser.add_argument( + "--s3-access-key-id", + help="an S3 access key id used for sscache bucket", + ) + parser.add_argument( + "--s3-secret-access-key", + help="an S3 secret access key used for sscache bucket", + ) parser.add_argument( "--s3-bucket", help="an S3 bucket used for sscache and clang-tidy-cache", @@ -494,10 +508,11 @@ def main() -> None: args.compiler, args.sanitizer, args.package_type, - args.cache, + args.s3_access_key_id, args.s3_bucket, args.s3_directory, args.s3_rw_access, + args.s3_secret_access_key, args.clang_tidy, args.version, args.official, From ed0a2b15e7f1c4082ff21aa585f650b14c02e9e5 Mon Sep 17 00:00:00 2001 From: MyroTk <44327070+MyroTk@users.noreply.github.com> Date: Wed, 10 Apr 2024 08:20:00 -0700 Subject: [PATCH 027/123] Update packager --- docker/packager/packager | 1 + 1 file changed, 1 insertion(+) diff --git a/docker/packager/packager b/docker/packager/packager index 72bb8b565c24..333e8080967e 100755 --- a/docker/packager/packager +++ b/docker/packager/packager @@ -508,6 +508,7 @@ def main() -> None: args.compiler, args.sanitizer, args.package_type, + args.cache, args.s3_access_key_id, args.s3_bucket, args.s3_directory, From cd249bb0c92445b07fb406526883359463129fa8 Mon Sep 17 00:00:00 2001 From: MyroTk <44327070+MyroTk@users.noreply.github.com> Date: Wed, 10 Apr 2024 11:29:16 -0700 Subject: [PATCH 028/123] Update release_branches.yml --- .github/workflows/release_branches.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/release_branches.yml b/.github/workflows/release_branches.yml index 36473201c37c..c383d5bc6035 100644 --- a/.github/workflows/release_branches.yml +++ b/.github/workflows/release_branches.yml @@ -638,7 +638,7 @@ jobs: secrets: inherit with: runner_type: altinity-on-demand, altinity-type-cpx51, altinity-image-x86-app-docker-ce, altinity-setup-regression - commit: ed5f2f0d9f33f46eb8962c2f2d974701c86f187e + commit: 8ca52fb119806bb3ee0d4f6b65fe4b7ad5fb58a1 arch: release build_sha: ${{ github.event_name == 'pull_request' && github.event.pull_request.head.sha || github.event_name == 'release' && github.sha }} @@ -648,7 +648,7 @@ jobs: secrets: inherit with: runner_type: altinity-on-demand, altinity-type-cax41, altinity-image-arm-app-docker-ce, altinity-setup-regression - commit: ed5f2f0d9f33f46eb8962c2f2d974701c86f187e + commit: 8ca52fb119806bb3ee0d4f6b65fe4b7ad5fb58a1 arch: aarch64 build_sha: ${{ github.event_name == 'pull_request' && github.event.pull_request.head.sha || github.event_name == 'release' && github.sha }} From 028048010bb8b3c22530e3b1b385c0c60afb93c7 Mon Sep 17 00:00:00 2001 From: MyroTk <44327070+MyroTk@users.noreply.github.com> Date: Thu, 11 Apr 2024 00:26:10 -0700 Subject: [PATCH 029/123] Update release_branches.yml --- .github/workflows/release_branches.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/release_branches.yml b/.github/workflows/release_branches.yml index c383d5bc6035..ec2b8a5b5c44 100644 --- a/.github/workflows/release_branches.yml +++ b/.github/workflows/release_branches.yml @@ -638,7 +638,7 @@ jobs: secrets: inherit with: runner_type: altinity-on-demand, altinity-type-cpx51, altinity-image-x86-app-docker-ce, altinity-setup-regression - commit: 8ca52fb119806bb3ee0d4f6b65fe4b7ad5fb58a1 + commit: f5ab9d5ef344ae1f4c8d5b8f698aceee84947cda arch: release build_sha: ${{ github.event_name == 'pull_request' && github.event.pull_request.head.sha || github.event_name == 'release' && github.sha }} @@ -648,7 +648,7 @@ jobs: secrets: inherit with: runner_type: altinity-on-demand, altinity-type-cax41, altinity-image-arm-app-docker-ce, altinity-setup-regression - commit: 8ca52fb119806bb3ee0d4f6b65fe4b7ad5fb58a1 + commit: f5ab9d5ef344ae1f4c8d5b8f698aceee84947cda arch: aarch64 build_sha: ${{ github.event_name == 'pull_request' && github.event.pull_request.head.sha || github.event_name == 'release' && github.sha }} From 3e4a5820011b849cdd35f4505881c536a37ad52c Mon Sep 17 00:00:00 2001 From: MyroTk <44327070+MyroTk@users.noreply.github.com> Date: Thu, 11 Apr 2024 07:57:58 -0700 Subject: [PATCH 030/123] Update regression.yml --- .github/workflows/regression.yml | 45 ++++++++++++++++++++++++++++++++ 1 file changed, 45 insertions(+) diff --git a/.github/workflows/regression.yml b/.github/workflows/regression.yml index 170e89b27b00..e788f35ce38c 100644 --- a/.github/workflows/regression.yml +++ b/.github/workflows/regression.yml @@ -181,6 +181,51 @@ jobs: name: ${{ env.SUITE }}-${{ inputs.arch }}-artifacts path: ${{ env.artifact_paths}} + Alter: + strategy: + fail-fast: false + matrix: + SUITE: ["replace partition", "attach partition", "move partition"] + needs: [runner_labels_setup] + runs-on: ${{ fromJson(needs.runner_labels_setup.outputs.runner_labels) }} + timeout-minutes: ${{ inputs.timeout_minutes }} + steps: + - name: Checkout regression repo + uses: actions/checkout@v3 + with: + repository: Altinity/clickhouse-regression + ref: ${{ inputs.commit }} + - name: Set envs + run: | + cat >> "$GITHUB_ENV" << 'EOF' + REPORTS_PATH=${{ runner.temp }}/reports_dir + SUITE=${{ matrix.SUITE }} + EOF + - name: Download json reports + uses: actions/download-artifact@v3 + with: + path: ${{ env.REPORTS_PATH }} + name: build_report_package_${{ inputs.arch }} + - name: Setup + run: .github/setup.sh + - name: Get deb url + run: python3 .github/get-deb-url.py --reports-path ${{ env.REPORTS_PATH }} --github-env $GITHUB_ENV + - name: Run ${{ env.SUITE }} suite + run: python3 + -u ${{ env.SUITE }}/regression.py + --clickhouse-binary-path ${{ env.clickhouse_binary_path }} + --only "/alter/${{ env.SUITE }}/*" + --attr project="$GITHUB_REPOSITORY" project.id="$GITHUB_REPOSITORY_ID" package="${{ env.clickhouse_binary_path }}" version="${{ env.version }}" user.name="$GITHUB_ACTOR" repository="https://github.com/Altinity/clickhouse-regression" commit.hash="$(git rev-parse HEAD)" job.id="$GITHUB_RUN_ID" job.url="$GITHUB_SERVER_URL/$GITHUB_REPOSITORY/actions/runs/$GITHUB_RUN_ID" arch="$(uname -i)" + ${{ env.args }} + - name: Create and upload logs + if: always() + run: .github/create_and_upload_logs.sh 1 + - uses: actions/upload-artifact@v3 + if: always() + with: + name: ${{ env.SUITE }}-${{ inputs.arch }}-artifacts + path: ${{ env.artifact_paths}} + Benchmark: strategy: fail-fast: false From df25d10a7e2388d0768d751961e0eaec882256cd Mon Sep 17 00:00:00 2001 From: MyroTk <44327070+MyroTk@users.noreply.github.com> Date: Thu, 11 Apr 2024 10:59:07 -0700 Subject: [PATCH 031/123] Update release_branches.yml --- .github/workflows/release_branches.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/release_branches.yml b/.github/workflows/release_branches.yml index ec2b8a5b5c44..185493167554 100644 --- a/.github/workflows/release_branches.yml +++ b/.github/workflows/release_branches.yml @@ -638,7 +638,7 @@ jobs: secrets: inherit with: runner_type: altinity-on-demand, altinity-type-cpx51, altinity-image-x86-app-docker-ce, altinity-setup-regression - commit: f5ab9d5ef344ae1f4c8d5b8f698aceee84947cda + commit: 17a81c07fc1f41fbee651e0ef0ca4b44e537e5b1 arch: release build_sha: ${{ github.event_name == 'pull_request' && github.event.pull_request.head.sha || github.event_name == 'release' && github.sha }} @@ -648,7 +648,7 @@ jobs: secrets: inherit with: runner_type: altinity-on-demand, altinity-type-cax41, altinity-image-arm-app-docker-ce, altinity-setup-regression - commit: f5ab9d5ef344ae1f4c8d5b8f698aceee84947cda + commit: 17a81c07fc1f41fbee651e0ef0ca4b44e537e5b1 arch: aarch64 build_sha: ${{ github.event_name == 'pull_request' && github.event.pull_request.head.sha || github.event_name == 'release' && github.sha }} From 38345b01dea7b8e884b73a45b240adab275fa3fd Mon Sep 17 00:00:00 2001 From: MyroTk <44327070+MyroTk@users.noreply.github.com> Date: Thu, 11 Apr 2024 14:45:18 -0700 Subject: [PATCH 032/123] Update regression.yml --- .github/workflows/regression.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/regression.yml b/.github/workflows/regression.yml index e788f35ce38c..dfe31ceb3a5c 100644 --- a/.github/workflows/regression.yml +++ b/.github/workflows/regression.yml @@ -212,7 +212,7 @@ jobs: run: python3 .github/get-deb-url.py --reports-path ${{ env.REPORTS_PATH }} --github-env $GITHUB_ENV - name: Run ${{ env.SUITE }} suite run: python3 - -u ${{ env.SUITE }}/regression.py + -u alter/regression.py --clickhouse-binary-path ${{ env.clickhouse_binary_path }} --only "/alter/${{ env.SUITE }}/*" --attr project="$GITHUB_REPOSITORY" project.id="$GITHUB_REPOSITORY_ID" package="${{ env.clickhouse_binary_path }}" version="${{ env.version }}" user.name="$GITHUB_ACTOR" repository="https://github.com/Altinity/clickhouse-regression" commit.hash="$(git rev-parse HEAD)" job.id="$GITHUB_RUN_ID" job.url="$GITHUB_SERVER_URL/$GITHUB_REPOSITORY/actions/runs/$GITHUB_RUN_ID" arch="$(uname -i)" From c27346f6c4b7b75e505fb0844bb96efac6a86c23 Mon Sep 17 00:00:00 2001 From: MyroTk <44327070+MyroTk@users.noreply.github.com> Date: Thu, 11 Apr 2024 18:57:46 -0700 Subject: [PATCH 033/123] Update regression.yml --- .github/workflows/regression.yml | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/.github/workflows/regression.yml b/.github/workflows/regression.yml index dfe31ceb3a5c..869d6e17011c 100644 --- a/.github/workflows/regression.yml +++ b/.github/workflows/regression.yml @@ -185,7 +185,7 @@ jobs: strategy: fail-fast: false matrix: - SUITE: ["replace partition", "attach partition", "move partition"] + ONLY: ["replace partition", "attach partition", "move partition"] needs: [runner_labels_setup] runs-on: ${{ fromJson(needs.runner_labels_setup.outputs.runner_labels) }} timeout-minutes: ${{ inputs.timeout_minutes }} @@ -199,7 +199,7 @@ jobs: run: | cat >> "$GITHUB_ENV" << 'EOF' REPORTS_PATH=${{ runner.temp }}/reports_dir - SUITE=${{ matrix.SUITE }} + SUITE=alter EOF - name: Download json reports uses: actions/download-artifact@v3 @@ -214,7 +214,7 @@ jobs: run: python3 -u alter/regression.py --clickhouse-binary-path ${{ env.clickhouse_binary_path }} - --only "/alter/${{ env.SUITE }}/*" + --only "/alter/${{ matrix.ONLU }}/*" --attr project="$GITHUB_REPOSITORY" project.id="$GITHUB_REPOSITORY_ID" package="${{ env.clickhouse_binary_path }}" version="${{ env.version }}" user.name="$GITHUB_ACTOR" repository="https://github.com/Altinity/clickhouse-regression" commit.hash="$(git rev-parse HEAD)" job.id="$GITHUB_RUN_ID" job.url="$GITHUB_SERVER_URL/$GITHUB_REPOSITORY/actions/runs/$GITHUB_RUN_ID" arch="$(uname -i)" ${{ env.args }} - name: Create and upload logs @@ -223,7 +223,7 @@ jobs: - uses: actions/upload-artifact@v3 if: always() with: - name: ${{ env.SUITE }}-${{ inputs.arch }}-artifacts + name: ${{ env.SUITE }}-${{ matrix.ONLY }}-${{ inputs.arch }}-artifacts path: ${{ env.artifact_paths}} Benchmark: From 6f014976109de10b263d3c09c4d69d59981b0875 Mon Sep 17 00:00:00 2001 From: MyroTk <44327070+MyroTk@users.noreply.github.com> Date: Thu, 11 Apr 2024 23:43:08 -0700 Subject: [PATCH 034/123] Update regression.yml --- .github/workflows/regression.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/regression.yml b/.github/workflows/regression.yml index 869d6e17011c..c7caf3e37621 100644 --- a/.github/workflows/regression.yml +++ b/.github/workflows/regression.yml @@ -141,7 +141,7 @@ jobs: strategy: fail-fast: false matrix: - SUITE: [aes_encryption, aggregate_functions, alter, atomic_insert, base_58, clickhouse_keeper, data_types, datetime64_extended_range, disk_level_encryption, dns, engines, example, extended_precision_data_types, kafka, kerberos, key_value, lightweight_delete, memory, part_moves_between_shards, rbac, selects, session_timezone, ssl_server, tiered_storage, window_functions] + SUITE: [aes_encryption, aggregate_functions, atomic_insert, base_58, clickhouse_keeper, data_types, datetime64_extended_range, disk_level_encryption, dns, engines, example, extended_precision_data_types, kafka, kerberos, key_value, lightweight_delete, memory, part_moves_between_shards, rbac, selects, session_timezone, ssl_server, tiered_storage, window_functions] needs: [runner_labels_setup] runs-on: ${{ fromJson(needs.runner_labels_setup.outputs.runner_labels) }} timeout-minutes: ${{ inputs.timeout_minutes }} @@ -185,7 +185,7 @@ jobs: strategy: fail-fast: false matrix: - ONLY: ["replace partition", "attach partition", "move partition"] + ONLY: [replace, attach, move] needs: [runner_labels_setup] runs-on: ${{ fromJson(needs.runner_labels_setup.outputs.runner_labels) }} timeout-minutes: ${{ inputs.timeout_minutes }} @@ -214,7 +214,7 @@ jobs: run: python3 -u alter/regression.py --clickhouse-binary-path ${{ env.clickhouse_binary_path }} - --only "/alter/${{ matrix.ONLU }}/*" + --only "/alter/${{ matrix.ONLY }} partition/*" --attr project="$GITHUB_REPOSITORY" project.id="$GITHUB_REPOSITORY_ID" package="${{ env.clickhouse_binary_path }}" version="${{ env.version }}" user.name="$GITHUB_ACTOR" repository="https://github.com/Altinity/clickhouse-regression" commit.hash="$(git rev-parse HEAD)" job.id="$GITHUB_RUN_ID" job.url="$GITHUB_SERVER_URL/$GITHUB_REPOSITORY/actions/runs/$GITHUB_RUN_ID" arch="$(uname -i)" ${{ env.args }} - name: Create and upload logs From 0be705cb24648133e37988c02e7bb4e8490e61ac Mon Sep 17 00:00:00 2001 From: Nikita Taranov Date: Thu, 2 Nov 2023 17:14:15 +0100 Subject: [PATCH 035/123] Fix wrong attribution of untracked memory to a user/query (#56089) * impl * add tests * add tests * fix typo --- src/Common/MemoryTracker.cpp | 23 +++++++--- src/Common/MemoryTracker.h | 5 +- ...02896_memory_accounting_for_user.reference | 0 .../02896_memory_accounting_for_user.sh | 46 +++++++++++++++++++ 4 files changed, 64 insertions(+), 10 deletions(-) create mode 100644 tests/queries/0_stateless/02896_memory_accounting_for_user.reference create mode 100755 tests/queries/0_stateless/02896_memory_accounting_for_user.sh diff --git a/src/Common/MemoryTracker.cpp b/src/Common/MemoryTracker.cpp index 93bd50a0b498..09f25f5dc70e 100644 --- a/src/Common/MemoryTracker.cpp +++ b/src/Common/MemoryTracker.cpp @@ -1,18 +1,19 @@ #include "MemoryTracker.h" #include -#include -#include -#include #include +#include #include #include -#include -#include -#include #include +#include #include +#include +#include +#include +#include #include +#include #include "config.h" @@ -589,6 +590,16 @@ bool MemoryTracker::isSizeOkForSampling(UInt64 size) const return ((max_allocation_size_bytes == 0 || size <= max_allocation_size_bytes) && size >= min_allocation_size_bytes); } +void MemoryTracker::setParent(MemoryTracker * elem) +{ + /// Untracked memory shouldn't be accounted to a query or a user if it was allocated before the thread was attached + /// to a query thread group or a user group, because this memory will be (🤞) freed outside of these scopes. + if (level == VariableContext::Thread && DB::current_thread) + DB::current_thread->flushUntrackedMemory(); + + parent.store(elem, std::memory_order_relaxed); +} + bool canEnqueueBackgroundTask() { auto limit = background_memory_tracker.getSoftLimit(); diff --git a/src/Common/MemoryTracker.h b/src/Common/MemoryTracker.h index 5041dc2af41f..a17ca421d204 100644 --- a/src/Common/MemoryTracker.h +++ b/src/Common/MemoryTracker.h @@ -196,10 +196,7 @@ class MemoryTracker /// next should be changed only once: from nullptr to some value. /// NOTE: It is not true in MergeListElement - void setParent(MemoryTracker * elem) - { - parent.store(elem, std::memory_order_relaxed); - } + void setParent(MemoryTracker * elem); MemoryTracker * getParent() { diff --git a/tests/queries/0_stateless/02896_memory_accounting_for_user.reference b/tests/queries/0_stateless/02896_memory_accounting_for_user.reference new file mode 100644 index 000000000000..e69de29bb2d1 diff --git a/tests/queries/0_stateless/02896_memory_accounting_for_user.sh b/tests/queries/0_stateless/02896_memory_accounting_for_user.sh new file mode 100755 index 000000000000..72f4be1475df --- /dev/null +++ b/tests/queries/0_stateless/02896_memory_accounting_for_user.sh @@ -0,0 +1,46 @@ +#!/usr/bin/env bash +# Tags: no-parallel, long + +CUR_DIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) +# shellcheck source=../shell_config.sh +. "$CUR_DIR"/../shell_config.sh + + +total_iterations=16 +parallelism=32 + +$CLICKHOUSE_CLIENT --query='DROP TABLE IF EXISTS test_inserts' +$CLICKHOUSE_CLIENT --query='CREATE TABLE test_inserts ENGINE=Null AS system.numbers' + +run_query() { + ( $CLICKHOUSE_CLIENT --query='SELECT * FROM numbers_mt(1000000) FORMAT CSV' | $CLICKHOUSE_CLIENT --max_threads 8 --max_memory_usage_for_user 1073741824 -q 'INSERT INTO test_inserts FORMAT CSV' 2>/dev/null ) +} + +for ((i = 1; i <= total_iterations; i++)); do + for ((j = 1; j <= parallelism; j++)); do + run_query & pids+=($!) + done + + EXIT_CODE=0 + new_pids=() + for pid in "${pids[@]:0:parallelism}"; do + CODE=0 + wait "${pid}" || CODE=$? + run_query & new_pids+=($!) + if [[ "${CODE}" != "0" ]]; then + EXIT_CODE=1; + fi + done + for pid in "${pids[@]:parallelism}"; do + CODE=0 + wait "${pid}" || CODE=$? + if [[ "${CODE}" != "0" ]]; then + EXIT_CODE=1; + fi + done + pids=("${new_pids[@]}") + + if [[ $EXIT_CODE -ne 0 ]]; then + exit $EXIT_CODE + fi +done From ec866b562b9869fa2fedb2bf2d68f90b1eab8f23 Mon Sep 17 00:00:00 2001 From: Nikita Taranov Date: Tue, 9 Apr 2024 12:32:38 +0000 Subject: [PATCH 036/123] Merge pull request #62208 from arthurpassos/s3_aws_private_link_style Add support for S3 access through aws private link interface --- src/IO/S3/URI.cpp | 14 +++++++++++--- src/IO/S3/URI.h | 1 + src/IO/tests/gtest_s3_uri.cpp | 34 ++++++++++++++++++++++++++++++++++ 3 files changed, 46 insertions(+), 3 deletions(-) diff --git a/src/IO/S3/URI.cpp b/src/IO/S3/URI.cpp index 34590df53973..2e8d892dd006 100644 --- a/src/IO/S3/URI.cpp +++ b/src/IO/S3/URI.cpp @@ -21,12 +21,17 @@ namespace S3 URI::URI(const std::string & uri_) { /// Case when bucket name represented in domain name of S3 URL. - /// E.g. (https://bucket-name.s3.Region.amazonaws.com/key) + /// E.g. (https://bucket-name.s3.region.amazonaws.com/key) /// https://docs.aws.amazon.com/AmazonS3/latest/dev/VirtualHosting.html#virtual-hosted-style-access static const RE2 virtual_hosted_style_pattern(R"((.+)\.(s3|cos|obs|oss)([.\-][a-z0-9\-.:]+))"); + /// Case when AWS Private Link Interface is being used + /// E.g. (bucket.vpce-07a1cd78f1bd55c5f-j3a3vg6w.s3.us-east-1.vpce.amazonaws.com/bucket-name/key) + /// https://docs.aws.amazon.com/AmazonS3/latest/userguide/privatelink-interface-endpoints.html + static const RE2 aws_private_link_style_pattern(R"(bucket\.vpce\-([a-z0-9\-.]+)\.vpce.amazonaws.com(:\d{1,5})?)"); + /// Case when bucket name and key represented in path of S3 URL. - /// E.g. (https://s3.Region.amazonaws.com/bucket-name/key) + /// E.g. (https://s3.region.amazonaws.com/bucket-name/key) /// https://docs.aws.amazon.com/AmazonS3/latest/dev/VirtualHosting.html#path-style-access static const RE2 path_style_pattern("^/([^/]*)/(.*)"); @@ -66,7 +71,10 @@ URI::URI(const std::string & uri_) String name; String endpoint_authority_from_uri; - if (re2::RE2::FullMatch(uri.getAuthority(), virtual_hosted_style_pattern, &bucket, &name, &endpoint_authority_from_uri)) + bool is_using_aws_private_link_interface = re2::RE2::FullMatch(uri.getAuthority(), aws_private_link_style_pattern); + + if (!is_using_aws_private_link_interface + && re2::RE2::FullMatch(uri.getAuthority(), virtual_hosted_style_pattern, &bucket, &name, &endpoint_authority_from_uri)) { is_virtual_hosted_style = true; endpoint = uri.getScheme() + "://" + name + endpoint_authority_from_uri; diff --git a/src/IO/S3/URI.h b/src/IO/S3/URI.h index f8f40cf91086..1b5c85ff2b7d 100644 --- a/src/IO/S3/URI.h +++ b/src/IO/S3/URI.h @@ -17,6 +17,7 @@ namespace DB::S3 * The following patterns are allowed: * s3://bucket/key * http(s)://endpoint/bucket/key + * http(s)://bucket..s3..vpce.amazonaws.com<:port_number>/bucket_name/key */ struct URI { diff --git a/src/IO/tests/gtest_s3_uri.cpp b/src/IO/tests/gtest_s3_uri.cpp index c088e41f1e8a..b04d2e79432a 100644 --- a/src/IO/tests/gtest_s3_uri.cpp +++ b/src/IO/tests/gtest_s3_uri.cpp @@ -74,6 +74,40 @@ const TestCase TestCases[] = { "data", "", true}, + {S3::URI("https://bucket.vpce-07a1cd78f1bd55c5f-j3a3vg6w.s3.us-east-1.vpce.amazonaws.com/root/nested/file.txt"), + "https://bucket.vpce-07a1cd78f1bd55c5f-j3a3vg6w.s3.us-east-1.vpce.amazonaws.com", + "root", + "nested/file.txt", + "", + false}, + // Test with a file with no extension + {S3::URI("https://bucket.vpce-03b2c987f1bd55c5f-j3b4vg7w.s3.ap-southeast-2.vpce.amazonaws.com/some_bucket/document"), + "https://bucket.vpce-03b2c987f1bd55c5f-j3b4vg7w.s3.ap-southeast-2.vpce.amazonaws.com", + "some_bucket", + "document", + "", + false}, + // Test with a deeply nested file path + {S3::URI("https://bucket.vpce-0242cd56f1bd55c5f-l5b7vg8x.s3.sa-east-1.vpce.amazonaws.com/some_bucket/b/c/d/e/f/g/h/i/j/data.json"), + "https://bucket.vpce-0242cd56f1bd55c5f-l5b7vg8x.s3.sa-east-1.vpce.amazonaws.com", + "some_bucket", + "b/c/d/e/f/g/h/i/j/data.json", + "", + false}, + // Zonal + {S3::URI("https://bucket.vpce-07a1cd78f1bd55c5f-j3a3vg6w-us-east-1a.s3.us-east-1.vpce.amazonaws.com/root/nested/file.txt"), + "https://bucket.vpce-07a1cd78f1bd55c5f-j3a3vg6w-us-east-1a.s3.us-east-1.vpce.amazonaws.com", + "root", + "nested/file.txt", + "", + false}, + // Non standard port + {S3::URI("https://bucket.vpce-07a1cd78f1bd55c5f-j3a3vg6w-us-east-1a.s3.us-east-1.vpce.amazonaws.com:65535/root/nested/file.txt"), + "https://bucket.vpce-07a1cd78f1bd55c5f-j3a3vg6w-us-east-1a.s3.us-east-1.vpce.amazonaws.com:65535", + "root", + "nested/file.txt", + "", + false}, }; class S3UriTest : public testing::TestWithParam From d9dbeda5955a545f5258fcc3d508ac776320a547 Mon Sep 17 00:00:00 2001 From: MyroTk <44327070+MyroTk@users.noreply.github.com> Date: Wed, 17 Apr 2024 10:39:28 -0700 Subject: [PATCH 037/123] Update reusable_build.yml --- .github/workflows/reusable_build.yml | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/.github/workflows/reusable_build.yml b/.github/workflows/reusable_build.yml index 4204798f9861..1a18adf5478e 100644 --- a/.github/workflows/reusable_build.yml +++ b/.github/workflows/reusable_build.yml @@ -135,7 +135,13 @@ jobs: with: name: changed_images path: ${{ env.IMAGES_PATH }} - + + - name: Create source tar + run: | + mkdir -p "$TEMP_PATH/build_check/package_release" + cd .. && tar czf $TEMP_PATH/build_source.src.tar.gz ClickHouse/ + cd $TEMP_PATH && tar xvzf $TEMP_PATH/build_source.src.tar.gz + - name: Build run: | cd "$REPO_COPY/tests/ci" && python3 build_check.py "$BUILD_NAME" From 3a7fb8649ba0fba99971720a6041c1cb9ab5bc3b Mon Sep 17 00:00:00 2001 From: Ilya Golshtein Date: Tue, 23 Apr 2024 13:52:24 +0000 Subject: [PATCH 038/123] kerberized_hadoop dockerfile: Download commons-daemon via https --- docker/test/integration/kerberized_hadoop/Dockerfile | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/docker/test/integration/kerberized_hadoop/Dockerfile b/docker/test/integration/kerberized_hadoop/Dockerfile index 592c3e36ef7f..50fe34c31da5 100644 --- a/docker/test/integration/kerberized_hadoop/Dockerfile +++ b/docker/test/integration/kerberized_hadoop/Dockerfile @@ -15,7 +15,10 @@ RUN curl -o krb5-libs-1.10.3-65.el6.x86_64.rpm ftp://ftp.pbone.net/mirror/vault. rm -fr *.rpm RUN cd /tmp && \ - curl http://archive.apache.org/dist/commons/daemon/source/commons-daemon-1.0.15-src.tar.gz -o commons-daemon-1.0.15-src.tar.gz && \ + curl -o wget.rpm ftp://ftp.pbone.net/mirror/vault.centos.org/6.9/os/x86_64/Packages/wget-1.12-10.el6.x86_64.rpm && \ + rpm -i wget.rpm && \ + rm -fr *.rpm && \ + wget --no-check-certificate https://archive.apache.org/dist/commons/daemon/source/commons-daemon-1.0.15-src.tar.gz && \ tar xzf commons-daemon-1.0.15-src.tar.gz && \ cd commons-daemon-1.0.15-src/src/native/unix && \ ./configure && \ From dc4244e8f32d48cdba89cd3b97f1374dfc803df9 Mon Sep 17 00:00:00 2001 From: Vasily Nemkov Date: Tue, 13 Feb 2024 16:55:53 +0100 Subject: [PATCH 039/123] Updated message for Altinity's build --- src/Daemon/BaseDaemon.cpp | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/src/Daemon/BaseDaemon.cpp b/src/Daemon/BaseDaemon.cpp index be323dc67861..e95f2e5f0d73 100644 --- a/src/Daemon/BaseDaemon.cpp +++ b/src/Daemon/BaseDaemon.cpp @@ -492,6 +492,10 @@ class SignalListener : public Poco::Runnable LOG_FATAL(log, "ClickHouse version {} is old and should be upgraded to the latest version.", VERSION_STRING); } } + else if constexpr (std::string_view(VERSION_OFFICIAL).contains("altinity build")) + { + LOG_FATAL(log, "You are using an Altinity Stable Build. Please log issues at https://github.com/Altinity/ClickHouse/issues. Thank you!"); + } else { LOG_FATAL(log, "This ClickHouse version is not official and should be upgraded to the official build."); From 636a295c100c832eaaaa07d92fceec2ab106fd65 Mon Sep 17 00:00:00 2001 From: Vasily Nemkov Date: Thu, 2 May 2024 17:39:54 +0000 Subject: [PATCH 040/123] Building aarch64 builds with '-no-pie' to allow better introspection --- CMakeLists.txt | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index 55bcf5fbf3c5..c6b619fd1743 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -427,12 +427,14 @@ if (NOT SANITIZE) set (CMAKE_POSITION_INDEPENDENT_CODE OFF) endif() -if (OS_LINUX AND NOT (ARCH_AARCH64 OR ARCH_S390X) AND NOT SANITIZE) +if (NOT OS_ANDROID AND OS_LINUX AND NOT ARCH_S390X AND NOT SANITIZE) # Slightly more efficient code can be generated - # It's disabled for ARM because otherwise ClickHouse cannot run on Android. + # Disabled for Android, because otherwise ClickHouse cannot run on Android. set (CMAKE_CXX_FLAGS_RELWITHDEBINFO "${CMAKE_CXX_FLAGS_RELWITHDEBINFO} -fno-pie") set (CMAKE_C_FLAGS_RELWITHDEBINFO "${CMAKE_C_FLAGS_RELWITHDEBINFO} -fno-pie") set (CMAKE_EXE_LINKER_FLAGS "${CMAKE_EXE_LINKER_FLAGS} -no-pie -Wl,-no-pie") +else () + message (WARNING "ClickHouse is built as PIE, system.trace_log will contain invalid addresses after server restart.") endif () if (ENABLE_TESTS) From f4c46b0411c2892a1938067dd60cfcac55925ca4 Mon Sep 17 00:00:00 2001 From: MyroTk <44327070+MyroTk@users.noreply.github.com> Date: Thu, 16 May 2024 00:04:33 -0700 Subject: [PATCH 041/123] Update release_branches.yml --- .github/workflows/release_branches.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.github/workflows/release_branches.yml b/.github/workflows/release_branches.yml index 185493167554..07d057362bb4 100644 --- a/.github/workflows/release_branches.yml +++ b/.github/workflows/release_branches.yml @@ -26,6 +26,8 @@ on: # yamllint disable-line rule:truthy push: branches: - 'releases/23.8**' + schedule: + - cron: "0 0 * * 6" jobs: DockerHubPushAarch64: From efe39cf52c725075a7d38469e65429a86cdb5417 Mon Sep 17 00:00:00 2001 From: MyroTk Date: Mon, 20 May 2024 09:59:45 -0700 Subject: [PATCH 042/123] update action to v4 --- .github/workflows/regression.yml | 54 +++++++++++++------------- .github/workflows/release_branches.yml | 40 +++++++++---------- .github/workflows/reusable_build.yml | 4 +- .github/workflows/reusable_test.yml | 2 +- 4 files changed, 50 insertions(+), 50 deletions(-) diff --git a/.github/workflows/regression.yml b/.github/workflows/regression.yml index c7caf3e37621..aa4b2f68904e 100644 --- a/.github/workflows/regression.yml +++ b/.github/workflows/regression.yml @@ -147,7 +147,7 @@ jobs: timeout-minutes: ${{ inputs.timeout_minutes }} steps: - name: Checkout regression repo - uses: actions/checkout@v3 + uses: actions/checkout@v4 with: repository: Altinity/clickhouse-regression ref: ${{ inputs.commit }} @@ -158,7 +158,7 @@ jobs: SUITE=${{ matrix.SUITE }} EOF - name: Download json reports - uses: actions/download-artifact@v3 + uses: actions/download-artifact@v4 with: path: ${{ env.REPORTS_PATH }} name: build_report_package_${{ inputs.arch }} @@ -175,7 +175,7 @@ jobs: - name: Create and upload logs if: always() run: .github/create_and_upload_logs.sh 1 - - uses: actions/upload-artifact@v3 + - uses: actions/upload-artifact@v4 if: always() with: name: ${{ env.SUITE }}-${{ inputs.arch }}-artifacts @@ -191,7 +191,7 @@ jobs: timeout-minutes: ${{ inputs.timeout_minutes }} steps: - name: Checkout regression repo - uses: actions/checkout@v3 + uses: actions/checkout@v4 with: repository: Altinity/clickhouse-regression ref: ${{ inputs.commit }} @@ -202,7 +202,7 @@ jobs: SUITE=alter EOF - name: Download json reports - uses: actions/download-artifact@v3 + uses: actions/download-artifact@v4 with: path: ${{ env.REPORTS_PATH }} name: build_report_package_${{ inputs.arch }} @@ -220,7 +220,7 @@ jobs: - name: Create and upload logs if: always() run: .github/create_and_upload_logs.sh 1 - - uses: actions/upload-artifact@v3 + - uses: actions/upload-artifact@v4 if: always() with: name: ${{ env.SUITE }}-${{ matrix.ONLY }}-${{ inputs.arch }}-artifacts @@ -236,7 +236,7 @@ jobs: timeout-minutes: ${{ inputs.timeout_minutes }} steps: - name: Checkout regression repo - uses: actions/checkout@v3 + uses: actions/checkout@v4 with: repository: Altinity/clickhouse-regression ref: ${{ inputs.commit }} @@ -248,7 +248,7 @@ jobs: STORAGE=/${{ matrix.STORAGE }} EOF - name: Download json reports - uses: actions/download-artifact@v3 + uses: actions/download-artifact@v4 with: path: ${{ env.REPORTS_PATH }} name: build_report_package_${{ inputs.arch }} @@ -273,7 +273,7 @@ jobs: - name: Create and upload logs if: always() run: .github/create_and_upload_logs.sh 1 - - uses: actions/upload-artifact@v3 + - uses: actions/upload-artifact@v4 if: always() with: name: benchmark-${{ matrix.STORAGE }}-${{ inputs.arch }}-artifacts @@ -285,7 +285,7 @@ jobs: timeout-minutes: ${{ inputs.timeout_minutes }} steps: - name: Checkout regression repo - uses: actions/checkout@v3 + uses: actions/checkout@v4 with: repository: Altinity/clickhouse-regression ref: ${{ inputs.commit }} @@ -297,7 +297,7 @@ jobs: STORAGE=/ssl EOF - name: Download json reports - uses: actions/download-artifact@v3 + uses: actions/download-artifact@v4 with: path: ${{ env.REPORTS_PATH }} name: build_report_package_${{ inputs.arch }} @@ -315,7 +315,7 @@ jobs: - name: Create and upload logs if: always() run: .github/create_and_upload_logs.sh 1 - - uses: actions/upload-artifact@v3 + - uses: actions/upload-artifact@v4 if: always() with: name: ${{ env.SUITE }}-${{ inputs.arch }}-ssl-artifacts @@ -331,7 +331,7 @@ jobs: timeout-minutes: ${{ inputs.timeout_minutes }} steps: - name: Checkout regression repo - uses: actions/checkout@v3 + uses: actions/checkout@v4 with: repository: Altinity/clickhouse-regression ref: ${{ inputs.commit }} @@ -342,7 +342,7 @@ jobs: SUITE=ldap/${{ matrix.SUITE }} EOF - name: Download json reports - uses: actions/download-artifact@v3 + uses: actions/download-artifact@v4 with: path: ${{ env.REPORTS_PATH }} name: build_report_package_${{ inputs.arch }} @@ -359,7 +359,7 @@ jobs: - name: Create and upload logs if: always() run: .github/create_and_upload_logs.sh 1 - - uses: actions/upload-artifact@v3 + - uses: actions/upload-artifact@v4 if: always() with: name: ldap-${{ matrix.SUITE }}-${{ inputs.arch }}-artifacts @@ -371,7 +371,7 @@ jobs: timeout-minutes: ${{ inputs.timeout_minutes }} steps: - name: Checkout regression repo - uses: actions/checkout@v3 + uses: actions/checkout@v4 with: repository: Altinity/clickhouse-regression ref: ${{ inputs.commit }} @@ -382,7 +382,7 @@ jobs: SUITE=parquet EOF - name: Download json reports - uses: actions/download-artifact@v3 + uses: actions/download-artifact@v4 with: path: ${{ env.REPORTS_PATH }} name: build_report_package_${{ inputs.arch }} @@ -399,7 +399,7 @@ jobs: - name: Create and upload logs if: always() run: .github/create_and_upload_logs.sh 1 - - uses: actions/upload-artifact@v3 + - uses: actions/upload-artifact@v4 if: always() with: name: ${{ env.SUITE }}-${{ inputs.arch }}-artifacts @@ -415,7 +415,7 @@ jobs: timeout-minutes: ${{ inputs.timeout_minutes }} steps: - name: Checkout regression repo - uses: actions/checkout@v3 + uses: actions/checkout@v4 with: repository: Altinity/clickhouse-regression ref: ${{ inputs.commit }} @@ -427,7 +427,7 @@ jobs: STORAGE=${{ matrix.STORAGE}} EOF - name: Download json reports - uses: actions/download-artifact@v3 + uses: actions/download-artifact@v4 with: path: ${{ env.REPORTS_PATH }} name: build_report_package_${{ inputs.arch }} @@ -450,7 +450,7 @@ jobs: - name: Create and upload logs if: always() run: .github/create_and_upload_logs.sh 1 - - uses: actions/upload-artifact@v3 + - uses: actions/upload-artifact@v4 if: always() with: name: ${{ env.SUITE }}-${{ env.STORAGE }}-${{ inputs.arch }}-artifacts @@ -466,7 +466,7 @@ jobs: timeout-minutes: ${{ inputs.timeout_minutes }} steps: - name: Checkout regression repo - uses: actions/checkout@v3 + uses: actions/checkout@v4 with: repository: Altinity/clickhouse-regression ref: ${{ inputs.commit }} @@ -478,7 +478,7 @@ jobs: STORAGE=/${{ matrix.STORAGE }} EOF - name: Download json reports - uses: actions/download-artifact@v3 + uses: actions/download-artifact@v4 with: path: ${{ env.REPORTS_PATH }} name: build_report_package_${{ inputs.arch }} @@ -503,7 +503,7 @@ jobs: - name: Create and upload logs if: always() run: .github/create_and_upload_logs.sh 1 - - uses: actions/upload-artifact@v3 + - uses: actions/upload-artifact@v4 if: always() with: name: ${{ env.SUITE }}-${{ matrix.STORAGE }}-${{ inputs.arch }}-artifacts @@ -519,7 +519,7 @@ jobs: timeout-minutes: ${{ inputs.timeout_minutes }} steps: - name: Checkout regression repo - uses: actions/checkout@v3 + uses: actions/checkout@v4 with: repository: Altinity/clickhouse-regression - name: Set envs @@ -530,7 +530,7 @@ jobs: STORAGE=/${{ matrix.STORAGE }} EOF - name: Download json reports - uses: actions/download-artifact@v3 + uses: actions/download-artifact@v4 with: path: ${{ env.REPORTS_PATH }} name: build_report_package_${{ inputs.arch }} @@ -554,7 +554,7 @@ jobs: - name: Create and upload logs if: always() run: .github/create_and_upload_logs.sh 1 - - uses: actions/upload-artifact@v3 + - uses: actions/upload-artifact@v4 if: always() with: name: ${{ env.SUITE }}-${{ matrix.STORAGE }}-${{ inputs.arch }}-artifacts diff --git a/.github/workflows/release_branches.yml b/.github/workflows/release_branches.yml index 07d057362bb4..9b2d6b2baec7 100644 --- a/.github/workflows/release_branches.yml +++ b/.github/workflows/release_branches.yml @@ -47,7 +47,7 @@ jobs: RUNNER_SSH_COMMAND: "ssh -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null root@$(hostname -I | cut -d ' ' -f 1)" - name: Upload images files to artifacts - uses: actions/upload-artifact@v3 + uses: actions/upload-artifact@v4 with: name: changed_images_aarch64 path: ${{ runner.temp }}/docker_images_check/changed_images_aarch64.json @@ -69,7 +69,7 @@ jobs: RUNNER_SSH_COMMAND: "ssh -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null root@$(hostname -I | cut -d ' ' -f 1)" - name: Upload images files to artifacts - uses: actions/upload-artifact@v3 + uses: actions/upload-artifact@v4 with: name: changed_images_amd64 path: ${{ runner.temp }}/docker_images_check/changed_images_amd64.json @@ -92,13 +92,13 @@ jobs: DOCKER_PASSWORD: ${{ secrets.DOCKER_PASSWORD }} - name: Download changed aarch64 images - uses: actions/download-artifact@v3 + uses: actions/download-artifact@v4 with: name: changed_images_aarch64 path: ${{ runner.temp }} - name: Download changed amd64 images - uses: actions/download-artifact@v3 + uses: actions/download-artifact@v4 with: name: changed_images_amd64 path: ${{ runner.temp }} @@ -109,7 +109,7 @@ jobs: python3 docker_manifests_merge.py --suffix amd64 --suffix aarch64 - name: Upload images files to artifacts - uses: actions/upload-artifact@v3 + uses: actions/upload-artifact@v4 with: name: changed_images path: ${{ runner.temp }}/changed_images.json @@ -187,7 +187,7 @@ jobs: runner_type: altinity-on-demand, altinity-setup-builder, altinity-type-ccx53, altinity-in-ash, altinity-image-x86-app-docker-ce additional_envs: | CLICKHOUSE_STABLE_VERSION_SUFFIX=altinitystable - + BuilderDebTsan: needs: [DockerHubPush] uses: ./.github/workflows/reusable_build.yml @@ -391,7 +391,7 @@ jobs: run_command: | cd "$REPO_COPY/tests/ci" python3 functional_test_check.py "$CHECK_NAME" "$KILL_TIMEOUT" - + FunctionalStatelessTestMsan: needs: [BuilderDebMsan] uses: ./.github/workflows/reusable_test.yml @@ -405,7 +405,7 @@ jobs: run_command: | cd "$REPO_COPY/tests/ci" python3 functional_test_check.py "$CHECK_NAME" "$KILL_TIMEOUT" - + FunctionalStatelessTestDebug: needs: [BuilderDebDebug] uses: ./.github/workflows/reusable_test.yml @@ -461,7 +461,7 @@ jobs: run_command: | cd "$REPO_COPY/tests/ci" python3 functional_test_check.py "$CHECK_NAME" "$KILL_TIMEOUT" - + FunctionalStatefulTestTsan: needs: [BuilderDebTsan] uses: ./.github/workflows/reusable_test.yml @@ -474,7 +474,7 @@ jobs: run_command: | cd "$REPO_COPY/tests/ci" python3 functional_test_check.py "$CHECK_NAME" "$KILL_TIMEOUT" - + FunctionalStatefulTestMsan: needs: [BuilderDebMsan] uses: ./.github/workflows/reusable_test.yml @@ -487,7 +487,7 @@ jobs: run_command: | cd "$REPO_COPY/tests/ci" python3 functional_test_check.py "$CHECK_NAME" "$KILL_TIMEOUT" - + FunctionalStatefulTestUBsan: needs: [BuilderDebUBsan] uses: ./.github/workflows/reusable_test.yml @@ -500,7 +500,7 @@ jobs: run_command: | cd "$REPO_COPY/tests/ci" python3 functional_test_check.py "$CHECK_NAME" "$KILL_TIMEOUT" - + FunctionalStatefulTestDebug: needs: [BuilderDebDebug] uses: ./.github/workflows/reusable_test.yml @@ -513,7 +513,7 @@ jobs: run_command: | cd "$REPO_COPY/tests/ci" python3 functional_test_check.py "$CHECK_NAME" "$KILL_TIMEOUT" - + ############################################################################################## ######################################### STRESS TESTS ####################################### ############################################################################################## @@ -528,7 +528,7 @@ jobs: run_command: | cd "$REPO_COPY/tests/ci" python3 stress_check.py "$CHECK_NAME" - + StressTestTsan: needs: [BuilderDebTsan] uses: ./.github/workflows/reusable_test.yml @@ -540,7 +540,7 @@ jobs: run_command: | cd "$REPO_COPY/tests/ci" python3 stress_check.py "$CHECK_NAME" - + StressTestMsan: needs: [BuilderDebMsan] uses: ./.github/workflows/reusable_test.yml @@ -551,7 +551,7 @@ jobs: run_command: | cd "$REPO_COPY/tests/ci" python3 stress_check.py "$CHECK_NAME" - + StressTestUBsan: needs: [BuilderDebUBsan] uses: ./.github/workflows/reusable_test.yml @@ -563,7 +563,7 @@ jobs: run_command: | cd "$REPO_COPY/tests/ci" python3 stress_check.py "$CHECK_NAME" - + StressTestDebug: needs: [BuilderDebDebug] uses: ./.github/workflows/reusable_test.yml @@ -669,9 +669,9 @@ jobs: run: | sudo rm -fr "$GITHUB_WORKSPACE" && mkdir "$GITHUB_WORKSPACE" - name: Check out repository code - uses: actions/checkout@v2 + uses: actions/checkout@v4 - name: Download json reports - uses: actions/download-artifact@v2 + uses: actions/download-artifact@v4 with: path: ${{ env.REPORTS_PATH }} - name: Sign release @@ -683,7 +683,7 @@ jobs: cd "$GITHUB_WORKSPACE/tests/ci" python3 sign_release.py - name: Upload signed hashes - uses: actions/upload-artifact@v2 + uses: actions/upload-artifact@v4 with: name: signed-hashes path: ${{ env.TEMP_PATH }}/*.gpg diff --git a/.github/workflows/reusable_build.yml b/.github/workflows/reusable_build.yml index 1a18adf5478e..58e56d54f409 100644 --- a/.github/workflows/reusable_build.yml +++ b/.github/workflows/reusable_build.yml @@ -131,7 +131,7 @@ jobs: DOCKER_PASSWORD: ${{ secrets.DOCKER_PASSWORD }} - name: Download changed images - uses: actions/download-artifact@v3 + uses: actions/download-artifact@v4 with: name: changed_images path: ${{ env.IMAGES_PATH }} @@ -148,7 +148,7 @@ jobs: - name: Upload build URLs to artifacts if: ${{ success() || failure() }} - uses: actions/upload-artifact@v3 + uses: actions/upload-artifact@v4 with: name: ${{ env.BUILD_URLS }} path: ${{ env.TEMP_PATH }}/${{ env.BUILD_URLS }}.json diff --git a/.github/workflows/reusable_test.yml b/.github/workflows/reusable_test.yml index c20f32bc9f88..d2d0bcc38248 100644 --- a/.github/workflows/reusable_test.yml +++ b/.github/workflows/reusable_test.yml @@ -155,7 +155,7 @@ jobs: job_type: test - name: Download json reports - uses: actions/download-artifact@v3 + uses: actions/download-artifact@v4 with: path: ${{ env.REPORTS_PATH }} From ed737aa0377b45f02ccd273637e43866420ac3a1 Mon Sep 17 00:00:00 2001 From: MyroTk Date: Wed, 22 May 2024 05:14:45 -0700 Subject: [PATCH 043/123] introduce smaller runners for light jobs, and artifacts cleanup --- .github/workflows/release_branches.yml | 16 +++++++++------- .github/workflows/reusable_build.yml | 12 ++---------- .github/workflows/reusable_test.yml | 9 +-------- 3 files changed, 12 insertions(+), 25 deletions(-) diff --git a/.github/workflows/release_branches.yml b/.github/workflows/release_branches.yml index 9b2d6b2baec7..2305b0b53ad7 100644 --- a/.github/workflows/release_branches.yml +++ b/.github/workflows/release_branches.yml @@ -120,7 +120,7 @@ jobs: secrets: inherit with: test_name: Compatibility check X86 - runner_type: altinity-on-demand, altinity-type-cpx41, altinity-image-x86-app-docker-ce + runner_type: altinity-on-demand, altinity-type-cpx51, altinity-image-x86-app-docker-ce timeout_minutes: 180 run_command: | cd "$REPO_COPY/tests/ci" @@ -133,6 +133,7 @@ jobs: with: test_name: Compatibility check Aarch64 runner_type: altinity-on-demand, altinity-type-cax41, altinity-image-arm-app-docker-ce + timeout_minutes: 180 run_command: | cd "$REPO_COPY/tests/ci" python3 compatibility_check.py --check-name "Compatibility check (aarch64)" --check-glibc @@ -267,7 +268,7 @@ jobs: secrets: inherit with: test_name: ClickHouse build check - runner_type: altinity-on-demand, altinity-type-cpx41, altinity-image-x86-app-docker-ce + runner_type: altinity-on-demand, altinity-setup-reporter, altinity-type-cax11, altinity-image-x86-app-docker-ce timeout_minutes: 180 additional_envs: | NEEDS_DATA< Date: Tue, 18 Jun 2024 11:19:23 +0000 Subject: [PATCH 044/123] Fixed issue with requests pip3 dependency In attemt to fix the following error: 'Error while fetching server API version: Not supported URL scheme http+docker' --- docker/test/integration/runner/Dockerfile | 1 + 1 file changed, 1 insertion(+) diff --git a/docker/test/integration/runner/Dockerfile b/docker/test/integration/runner/Dockerfile index b8139354b5fd..206354ed471b 100644 --- a/docker/test/integration/runner/Dockerfile +++ b/docker/test/integration/runner/Dockerfile @@ -99,6 +99,7 @@ RUN python3 -m pip install --no-cache-dir \ pytz \ pyyaml==5.3.1 \ redis \ + requests==2.31.0 \ requests-kerberos \ tzlocal==2.1 \ retry \ From b7463e5806a865ad32f4f1ae0f645f1d59fa8e1d Mon Sep 17 00:00:00 2001 From: Vasily Nemkov Date: Thu, 20 Jun 2024 13:00:18 +0000 Subject: [PATCH 045/123] Fix 02117_show_create_table_system test in accordance with #58665 --- .../0_stateless/02117_show_create_table_system.reference | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/tests/queries/0_stateless/02117_show_create_table_system.reference b/tests/queries/0_stateless/02117_show_create_table_system.reference index d96a02834b40..70bd970c66f2 100644 --- a/tests/queries/0_stateless/02117_show_create_table_system.reference +++ b/tests/queries/0_stateless/02117_show_create_table_system.reference @@ -299,7 +299,7 @@ CREATE TABLE system.grants ( `user_name` Nullable(String), `role_name` Nullable(String), - `access_type` Enum16('SHOW DATABASES' = 0, 'SHOW TABLES' = 1, 'SHOW COLUMNS' = 2, 'SHOW DICTIONARIES' = 3, 'SHOW' = 4, 'SHOW FILESYSTEM CACHES' = 5, 'SELECT' = 6, 'INSERT' = 7, 'ALTER UPDATE' = 8, 'ALTER DELETE' = 9, 'ALTER ADD COLUMN' = 10, 'ALTER MODIFY COLUMN' = 11, 'ALTER DROP COLUMN' = 12, 'ALTER COMMENT COLUMN' = 13, 'ALTER CLEAR COLUMN' = 14, 'ALTER RENAME COLUMN' = 15, 'ALTER MATERIALIZE COLUMN' = 16, 'ALTER COLUMN' = 17, 'ALTER MODIFY COMMENT' = 18, 'ALTER ORDER BY' = 19, 'ALTER SAMPLE BY' = 20, 'ALTER ADD INDEX' = 21, 'ALTER DROP INDEX' = 22, 'ALTER MATERIALIZE INDEX' = 23, 'ALTER CLEAR INDEX' = 24, 'ALTER INDEX' = 25, 'ALTER ADD PROJECTION' = 26, 'ALTER DROP PROJECTION' = 27, 'ALTER MATERIALIZE PROJECTION' = 28, 'ALTER CLEAR PROJECTION' = 29, 'ALTER PROJECTION' = 30, 'ALTER ADD CONSTRAINT' = 31, 'ALTER DROP CONSTRAINT' = 32, 'ALTER CONSTRAINT' = 33, 'ALTER TTL' = 34, 'ALTER MATERIALIZE TTL' = 35, 'ALTER SETTINGS' = 36, 'ALTER MOVE PARTITION' = 37, 'ALTER FETCH PARTITION' = 38, 'ALTER FREEZE PARTITION' = 39, 'ALTER DATABASE SETTINGS' = 40, 'ALTER NAMED COLLECTION' = 41, 'ALTER TABLE' = 42, 'ALTER DATABASE' = 43, 'ALTER VIEW REFRESH' = 44, 'ALTER VIEW MODIFY QUERY' = 45, 'ALTER VIEW' = 46, 'ALTER' = 47, 'CREATE DATABASE' = 48, 'CREATE TABLE' = 49, 'CREATE VIEW' = 50, 'CREATE DICTIONARY' = 51, 'CREATE TEMPORARY TABLE' = 52, 'CREATE ARBITRARY TEMPORARY TABLE' = 53, 'CREATE FUNCTION' = 54, 'CREATE NAMED COLLECTION' = 55, 'CREATE' = 56, 'DROP DATABASE' = 57, 'DROP TABLE' = 58, 'DROP VIEW' = 59, 'DROP DICTIONARY' = 60, 'DROP FUNCTION' = 61, 'DROP NAMED COLLECTION' = 62, 'DROP' = 63, 'UNDROP TABLE' = 64, 'TRUNCATE' = 65, 'OPTIMIZE' = 66, 'BACKUP' = 67, 'KILL QUERY' = 68, 'KILL TRANSACTION' = 69, 'MOVE PARTITION BETWEEN SHARDS' = 70, 'CREATE USER' = 71, 'ALTER USER' = 72, 'DROP USER' = 73, 'CREATE ROLE' = 74, 'ALTER ROLE' = 75, 'DROP ROLE' = 76, 'ROLE ADMIN' = 77, 'CREATE ROW POLICY' = 78, 'ALTER ROW POLICY' = 79, 'DROP ROW POLICY' = 80, 'CREATE QUOTA' = 81, 'ALTER QUOTA' = 82, 'DROP QUOTA' = 83, 'CREATE SETTINGS PROFILE' = 84, 'ALTER SETTINGS PROFILE' = 85, 'DROP SETTINGS PROFILE' = 86, 'SHOW USERS' = 87, 'SHOW ROLES' = 88, 'SHOW ROW POLICIES' = 89, 'SHOW QUOTAS' = 90, 'SHOW SETTINGS PROFILES' = 91, 'SHOW ACCESS' = 92, 'ACCESS MANAGEMENT' = 93, 'SHOW NAMED COLLECTIONS' = 94, 'SHOW NAMED COLLECTIONS SECRETS' = 95, 'NAMED COLLECTION' = 96, 'NAMED COLLECTION ADMIN' = 97, 'SYSTEM SHUTDOWN' = 98, 'SYSTEM DROP DNS CACHE' = 99, 'SYSTEM DROP MARK CACHE' = 100, 'SYSTEM DROP UNCOMPRESSED CACHE' = 101, 'SYSTEM DROP MMAP CACHE' = 102, 'SYSTEM DROP QUERY CACHE' = 103, 'SYSTEM DROP COMPILED EXPRESSION CACHE' = 104, 'SYSTEM DROP FILESYSTEM CACHE' = 105, 'SYSTEM SYNC FILESYSTEM CACHE' = 106, 'SYSTEM DROP SCHEMA CACHE' = 107, 'SYSTEM DROP S3 CLIENT CACHE' = 108, 'SYSTEM DROP CACHE' = 109, 'SYSTEM RELOAD CONFIG' = 110, 'SYSTEM RELOAD USERS' = 111, 'SYSTEM RELOAD DICTIONARY' = 112, 'SYSTEM RELOAD MODEL' = 113, 'SYSTEM RELOAD FUNCTION' = 114, 'SYSTEM RELOAD EMBEDDED DICTIONARIES' = 115, 'SYSTEM RELOAD' = 116, 'SYSTEM RESTART DISK' = 117, 'SYSTEM MERGES' = 118, 'SYSTEM TTL MERGES' = 119, 'SYSTEM FETCHES' = 120, 'SYSTEM MOVES' = 121, 'SYSTEM PULLING REPLICATION LOG' = 122, 'SYSTEM DISTRIBUTED SENDS' = 123, 'SYSTEM REPLICATED SENDS' = 124, 'SYSTEM SENDS' = 125, 'SYSTEM REPLICATION QUEUES' = 126, 'SYSTEM DROP REPLICA' = 127, 'SYSTEM SYNC REPLICA' = 128, 'SYSTEM RESTART REPLICA' = 129, 'SYSTEM RESTORE REPLICA' = 130, 'SYSTEM WAIT LOADING PARTS' = 131, 'SYSTEM SYNC DATABASE REPLICA' = 132, 'SYSTEM SYNC TRANSACTION LOG' = 133, 'SYSTEM SYNC FILE CACHE' = 134, 'SYSTEM FLUSH DISTRIBUTED' = 135, 'SYSTEM FLUSH LOGS' = 136, 'SYSTEM FLUSH ASYNC INSERT QUEUE' = 137, 'SYSTEM FLUSH' = 138, 'SYSTEM THREAD FUZZER' = 139, 'SYSTEM UNFREEZE' = 140, 'SYSTEM FAILPOINT' = 141, 'SYSTEM LISTEN' = 142, 'SYSTEM' = 143, 'dictGet' = 144, 'displaySecretsInShowAndSelect' = 145, 'addressToLine' = 146, 'addressToLineWithInlines' = 147, 'addressToSymbol' = 148, 'demangle' = 149, 'INTROSPECTION' = 150, 'FILE' = 151, 'URL' = 152, 'REMOTE' = 153, 'MONGO' = 154, 'REDIS' = 155, 'MEILISEARCH' = 156, 'MYSQL' = 157, 'POSTGRES' = 158, 'SQLITE' = 159, 'ODBC' = 160, 'JDBC' = 161, 'HDFS' = 162, 'S3' = 163, 'HIVE' = 164, 'AZURE' = 165, 'SOURCES' = 166, 'CLUSTER' = 167, 'ALL' = 168, 'NONE' = 169), + `access_type` Enum16('SHOW DATABASES' = 0, 'SHOW TABLES' = 1, 'SHOW COLUMNS' = 2, 'SHOW DICTIONARIES' = 3, 'SHOW' = 4, 'SHOW FILESYSTEM CACHES' = 5, 'SELECT' = 6, 'INSERT' = 7, 'ALTER UPDATE' = 8, 'ALTER DELETE' = 9, 'ALTER ADD COLUMN' = 10, 'ALTER MODIFY COLUMN' = 11, 'ALTER DROP COLUMN' = 12, 'ALTER COMMENT COLUMN' = 13, 'ALTER CLEAR COLUMN' = 14, 'ALTER RENAME COLUMN' = 15, 'ALTER MATERIALIZE COLUMN' = 16, 'ALTER COLUMN' = 17, 'ALTER MODIFY COMMENT' = 18, 'ALTER ORDER BY' = 19, 'ALTER SAMPLE BY' = 20, 'ALTER ADD INDEX' = 21, 'ALTER DROP INDEX' = 22, 'ALTER MATERIALIZE INDEX' = 23, 'ALTER CLEAR INDEX' = 24, 'ALTER INDEX' = 25, 'ALTER ADD PROJECTION' = 26, 'ALTER DROP PROJECTION' = 27, 'ALTER MATERIALIZE PROJECTION' = 28, 'ALTER CLEAR PROJECTION' = 29, 'ALTER PROJECTION' = 30, 'ALTER ADD CONSTRAINT' = 31, 'ALTER DROP CONSTRAINT' = 32, 'ALTER CONSTRAINT' = 33, 'ALTER TTL' = 34, 'ALTER MATERIALIZE TTL' = 35, 'ALTER SETTINGS' = 36, 'ALTER MOVE PARTITION' = 37, 'ALTER FETCH PARTITION' = 38, 'ALTER FREEZE PARTITION' = 39, 'ALTER DATABASE SETTINGS' = 40, 'ALTER NAMED COLLECTION' = 41, 'ALTER TABLE' = 42, 'ALTER DATABASE' = 43, 'ALTER VIEW REFRESH' = 44, 'ALTER VIEW MODIFY QUERY' = 45, 'ALTER VIEW' = 46, 'ALTER' = 47, 'CREATE DATABASE' = 48, 'CREATE TABLE' = 49, 'CREATE VIEW' = 50, 'CREATE DICTIONARY' = 51, 'CREATE TEMPORARY TABLE' = 52, 'CREATE ARBITRARY TEMPORARY TABLE' = 53, 'CREATE FUNCTION' = 54, 'CREATE NAMED COLLECTION' = 55, 'CREATE' = 56, 'DROP DATABASE' = 57, 'DROP TABLE' = 58, 'DROP VIEW' = 59, 'DROP DICTIONARY' = 60, 'DROP FUNCTION' = 61, 'DROP NAMED COLLECTION' = 62, 'DROP' = 63, 'UNDROP TABLE' = 64, 'TRUNCATE' = 65, 'OPTIMIZE' = 66, 'BACKUP' = 67, 'KILL QUERY' = 68, 'KILL TRANSACTION' = 69, 'MOVE PARTITION BETWEEN SHARDS' = 70, 'CREATE USER' = 71, 'ALTER USER' = 72, 'DROP USER' = 73, 'CREATE ROLE' = 74, 'ALTER ROLE' = 75, 'DROP ROLE' = 76, 'ROLE ADMIN' = 77, 'CREATE ROW POLICY' = 78, 'ALTER ROW POLICY' = 79, 'DROP ROW POLICY' = 80, 'CREATE QUOTA' = 81, 'ALTER QUOTA' = 82, 'DROP QUOTA' = 83, 'CREATE SETTINGS PROFILE' = 84, 'ALTER SETTINGS PROFILE' = 85, 'DROP SETTINGS PROFILE' = 86, 'SHOW USERS' = 87, 'SHOW ROLES' = 88, 'SHOW ROW POLICIES' = 89, 'SHOW QUOTAS' = 90, 'SHOW SETTINGS PROFILES' = 91, 'SHOW ACCESS' = 92, 'ACCESS MANAGEMENT' = 93, 'SHOW NAMED COLLECTIONS' = 94, 'SHOW NAMED COLLECTIONS SECRETS' = 95, 'NAMED COLLECTION' = 96, 'NAMED COLLECTION ADMIN' = 97, 'SYSTEM SHUTDOWN' = 98, 'SYSTEM DROP DNS CACHE' = 99, 'SYSTEM DROP MARK CACHE' = 100, 'SYSTEM DROP UNCOMPRESSED CACHE' = 101, 'SYSTEM DROP MMAP CACHE' = 102, 'SYSTEM DROP QUERY CACHE' = 103, 'SYSTEM DROP COMPILED EXPRESSION CACHE' = 104, 'SYSTEM DROP FILESYSTEM CACHE' = 105, 'SYSTEM SYNC FILESYSTEM CACHE' = 106, 'SYSTEM DROP SCHEMA CACHE' = 107, 'SYSTEM DROP S3 CLIENT CACHE' = 108, 'SYSTEM DROP CACHE' = 109, 'SYSTEM RELOAD CONFIG' = 110, 'SYSTEM RELOAD USERS' = 111, 'SYSTEM RELOAD DICTIONARY' = 112, 'SYSTEM RELOAD MODEL' = 113, 'SYSTEM RELOAD FUNCTION' = 114, 'SYSTEM RELOAD EMBEDDED DICTIONARIES' = 115, 'SYSTEM RELOAD' = 116, 'SYSTEM RESTART DISK' = 117, 'SYSTEM MERGES' = 118, 'SYSTEM TTL MERGES' = 119, 'SYSTEM FETCHES' = 120, 'SYSTEM MOVES' = 121, 'SYSTEM PULLING REPLICATION LOG' = 122, 'SYSTEM DISTRIBUTED SENDS' = 123, 'SYSTEM REPLICATED SENDS' = 124, 'SYSTEM SENDS' = 125, 'SYSTEM REPLICATION QUEUES' = 126, 'SYSTEM DROP REPLICA' = 127, 'SYSTEM SYNC REPLICA' = 128, 'SYSTEM RESTART REPLICA' = 129, 'SYSTEM RESTORE REPLICA' = 130, 'SYSTEM WAIT LOADING PARTS' = 131, 'SYSTEM SYNC DATABASE REPLICA' = 132, 'SYSTEM SYNC TRANSACTION LOG' = 133, 'SYSTEM SYNC FILE CACHE' = 134, 'SYSTEM FLUSH DISTRIBUTED' = 135, 'SYSTEM FLUSH LOGS' = 136, 'SYSTEM FLUSH ASYNC INSERT QUEUE' = 137, 'SYSTEM FLUSH' = 138, 'SYSTEM THREAD FUZZER' = 139, 'SYSTEM UNFREEZE' = 140, 'SYSTEM FAILPOINT' = 141, 'SYSTEM LISTEN' = 142, 'SYSTEM JEMALLOC' = 143, 'SYSTEM' = 144, 'dictGet' = 145, 'displaySecretsInShowAndSelect' = 146, 'addressToLine' = 147, 'addressToLineWithInlines' = 148, 'addressToSymbol' = 149, 'demangle' = 150, 'INTROSPECTION' = 151, 'FILE' = 152, 'URL' = 153, 'REMOTE' = 154, 'MONGO' = 155, 'REDIS' = 156, 'MEILISEARCH' = 157, 'MYSQL' = 158, 'POSTGRES' = 159, 'SQLITE' = 160, 'ODBC' = 161, 'JDBC' = 162, 'HDFS' = 163, 'S3' = 164, 'HIVE' = 165, 'AZURE' = 166, 'SOURCES' = 167, 'CLUSTER' = 168, 'ALL' = 169, 'NONE' = 170), `database` Nullable(String), `table` Nullable(String), `column` Nullable(String), @@ -588,10 +588,10 @@ ENGINE = SystemPartsColumns COMMENT 'SYSTEM TABLE is built on the fly.' CREATE TABLE system.privileges ( - `privilege` Enum16('SHOW DATABASES' = 0, 'SHOW TABLES' = 1, 'SHOW COLUMNS' = 2, 'SHOW DICTIONARIES' = 3, 'SHOW' = 4, 'SHOW FILESYSTEM CACHES' = 5, 'SELECT' = 6, 'INSERT' = 7, 'ALTER UPDATE' = 8, 'ALTER DELETE' = 9, 'ALTER ADD COLUMN' = 10, 'ALTER MODIFY COLUMN' = 11, 'ALTER DROP COLUMN' = 12, 'ALTER COMMENT COLUMN' = 13, 'ALTER CLEAR COLUMN' = 14, 'ALTER RENAME COLUMN' = 15, 'ALTER MATERIALIZE COLUMN' = 16, 'ALTER COLUMN' = 17, 'ALTER MODIFY COMMENT' = 18, 'ALTER ORDER BY' = 19, 'ALTER SAMPLE BY' = 20, 'ALTER ADD INDEX' = 21, 'ALTER DROP INDEX' = 22, 'ALTER MATERIALIZE INDEX' = 23, 'ALTER CLEAR INDEX' = 24, 'ALTER INDEX' = 25, 'ALTER ADD PROJECTION' = 26, 'ALTER DROP PROJECTION' = 27, 'ALTER MATERIALIZE PROJECTION' = 28, 'ALTER CLEAR PROJECTION' = 29, 'ALTER PROJECTION' = 30, 'ALTER ADD CONSTRAINT' = 31, 'ALTER DROP CONSTRAINT' = 32, 'ALTER CONSTRAINT' = 33, 'ALTER TTL' = 34, 'ALTER MATERIALIZE TTL' = 35, 'ALTER SETTINGS' = 36, 'ALTER MOVE PARTITION' = 37, 'ALTER FETCH PARTITION' = 38, 'ALTER FREEZE PARTITION' = 39, 'ALTER DATABASE SETTINGS' = 40, 'ALTER NAMED COLLECTION' = 41, 'ALTER TABLE' = 42, 'ALTER DATABASE' = 43, 'ALTER VIEW REFRESH' = 44, 'ALTER VIEW MODIFY QUERY' = 45, 'ALTER VIEW' = 46, 'ALTER' = 47, 'CREATE DATABASE' = 48, 'CREATE TABLE' = 49, 'CREATE VIEW' = 50, 'CREATE DICTIONARY' = 51, 'CREATE TEMPORARY TABLE' = 52, 'CREATE ARBITRARY TEMPORARY TABLE' = 53, 'CREATE FUNCTION' = 54, 'CREATE NAMED COLLECTION' = 55, 'CREATE' = 56, 'DROP DATABASE' = 57, 'DROP TABLE' = 58, 'DROP VIEW' = 59, 'DROP DICTIONARY' = 60, 'DROP FUNCTION' = 61, 'DROP NAMED COLLECTION' = 62, 'DROP' = 63, 'UNDROP TABLE' = 64, 'TRUNCATE' = 65, 'OPTIMIZE' = 66, 'BACKUP' = 67, 'KILL QUERY' = 68, 'KILL TRANSACTION' = 69, 'MOVE PARTITION BETWEEN SHARDS' = 70, 'CREATE USER' = 71, 'ALTER USER' = 72, 'DROP USER' = 73, 'CREATE ROLE' = 74, 'ALTER ROLE' = 75, 'DROP ROLE' = 76, 'ROLE ADMIN' = 77, 'CREATE ROW POLICY' = 78, 'ALTER ROW POLICY' = 79, 'DROP ROW POLICY' = 80, 'CREATE QUOTA' = 81, 'ALTER QUOTA' = 82, 'DROP QUOTA' = 83, 'CREATE SETTINGS PROFILE' = 84, 'ALTER SETTINGS PROFILE' = 85, 'DROP SETTINGS PROFILE' = 86, 'SHOW USERS' = 87, 'SHOW ROLES' = 88, 'SHOW ROW POLICIES' = 89, 'SHOW QUOTAS' = 90, 'SHOW SETTINGS PROFILES' = 91, 'SHOW ACCESS' = 92, 'ACCESS MANAGEMENT' = 93, 'SHOW NAMED COLLECTIONS' = 94, 'SHOW NAMED COLLECTIONS SECRETS' = 95, 'NAMED COLLECTION' = 96, 'NAMED COLLECTION ADMIN' = 97, 'SYSTEM SHUTDOWN' = 98, 'SYSTEM DROP DNS CACHE' = 99, 'SYSTEM DROP MARK CACHE' = 100, 'SYSTEM DROP UNCOMPRESSED CACHE' = 101, 'SYSTEM DROP MMAP CACHE' = 102, 'SYSTEM DROP QUERY CACHE' = 103, 'SYSTEM DROP COMPILED EXPRESSION CACHE' = 104, 'SYSTEM DROP FILESYSTEM CACHE' = 105, 'SYSTEM SYNC FILESYSTEM CACHE' = 106, 'SYSTEM DROP SCHEMA CACHE' = 107, 'SYSTEM DROP S3 CLIENT CACHE' = 108, 'SYSTEM DROP CACHE' = 109, 'SYSTEM RELOAD CONFIG' = 110, 'SYSTEM RELOAD USERS' = 111, 'SYSTEM RELOAD DICTIONARY' = 112, 'SYSTEM RELOAD MODEL' = 113, 'SYSTEM RELOAD FUNCTION' = 114, 'SYSTEM RELOAD EMBEDDED DICTIONARIES' = 115, 'SYSTEM RELOAD' = 116, 'SYSTEM RESTART DISK' = 117, 'SYSTEM MERGES' = 118, 'SYSTEM TTL MERGES' = 119, 'SYSTEM FETCHES' = 120, 'SYSTEM MOVES' = 121, 'SYSTEM PULLING REPLICATION LOG' = 122, 'SYSTEM DISTRIBUTED SENDS' = 123, 'SYSTEM REPLICATED SENDS' = 124, 'SYSTEM SENDS' = 125, 'SYSTEM REPLICATION QUEUES' = 126, 'SYSTEM DROP REPLICA' = 127, 'SYSTEM SYNC REPLICA' = 128, 'SYSTEM RESTART REPLICA' = 129, 'SYSTEM RESTORE REPLICA' = 130, 'SYSTEM WAIT LOADING PARTS' = 131, 'SYSTEM SYNC DATABASE REPLICA' = 132, 'SYSTEM SYNC TRANSACTION LOG' = 133, 'SYSTEM SYNC FILE CACHE' = 134, 'SYSTEM FLUSH DISTRIBUTED' = 135, 'SYSTEM FLUSH LOGS' = 136, 'SYSTEM FLUSH ASYNC INSERT QUEUE' = 137, 'SYSTEM FLUSH' = 138, 'SYSTEM THREAD FUZZER' = 139, 'SYSTEM UNFREEZE' = 140, 'SYSTEM FAILPOINT' = 141, 'SYSTEM LISTEN' = 142, 'SYSTEM' = 143, 'dictGet' = 144, 'displaySecretsInShowAndSelect' = 145, 'addressToLine' = 146, 'addressToLineWithInlines' = 147, 'addressToSymbol' = 148, 'demangle' = 149, 'INTROSPECTION' = 150, 'FILE' = 151, 'URL' = 152, 'REMOTE' = 153, 'MONGO' = 154, 'REDIS' = 155, 'MEILISEARCH' = 156, 'MYSQL' = 157, 'POSTGRES' = 158, 'SQLITE' = 159, 'ODBC' = 160, 'JDBC' = 161, 'HDFS' = 162, 'S3' = 163, 'HIVE' = 164, 'AZURE' = 165, 'SOURCES' = 166, 'CLUSTER' = 167, 'ALL' = 168, 'NONE' = 169), + `privilege` Enum16('SHOW DATABASES' = 0, 'SHOW TABLES' = 1, 'SHOW COLUMNS' = 2, 'SHOW DICTIONARIES' = 3, 'SHOW' = 4, 'SHOW FILESYSTEM CACHES' = 5, 'SELECT' = 6, 'INSERT' = 7, 'ALTER UPDATE' = 8, 'ALTER DELETE' = 9, 'ALTER ADD COLUMN' = 10, 'ALTER MODIFY COLUMN' = 11, 'ALTER DROP COLUMN' = 12, 'ALTER COMMENT COLUMN' = 13, 'ALTER CLEAR COLUMN' = 14, 'ALTER RENAME COLUMN' = 15, 'ALTER MATERIALIZE COLUMN' = 16, 'ALTER COLUMN' = 17, 'ALTER MODIFY COMMENT' = 18, 'ALTER ORDER BY' = 19, 'ALTER SAMPLE BY' = 20, 'ALTER ADD INDEX' = 21, 'ALTER DROP INDEX' = 22, 'ALTER MATERIALIZE INDEX' = 23, 'ALTER CLEAR INDEX' = 24, 'ALTER INDEX' = 25, 'ALTER ADD PROJECTION' = 26, 'ALTER DROP PROJECTION' = 27, 'ALTER MATERIALIZE PROJECTION' = 28, 'ALTER CLEAR PROJECTION' = 29, 'ALTER PROJECTION' = 30, 'ALTER ADD CONSTRAINT' = 31, 'ALTER DROP CONSTRAINT' = 32, 'ALTER CONSTRAINT' = 33, 'ALTER TTL' = 34, 'ALTER MATERIALIZE TTL' = 35, 'ALTER SETTINGS' = 36, 'ALTER MOVE PARTITION' = 37, 'ALTER FETCH PARTITION' = 38, 'ALTER FREEZE PARTITION' = 39, 'ALTER DATABASE SETTINGS' = 40, 'ALTER NAMED COLLECTION' = 41, 'ALTER TABLE' = 42, 'ALTER DATABASE' = 43, 'ALTER VIEW REFRESH' = 44, 'ALTER VIEW MODIFY QUERY' = 45, 'ALTER VIEW' = 46, 'ALTER' = 47, 'CREATE DATABASE' = 48, 'CREATE TABLE' = 49, 'CREATE VIEW' = 50, 'CREATE DICTIONARY' = 51, 'CREATE TEMPORARY TABLE' = 52, 'CREATE ARBITRARY TEMPORARY TABLE' = 53, 'CREATE FUNCTION' = 54, 'CREATE NAMED COLLECTION' = 55, 'CREATE' = 56, 'DROP DATABASE' = 57, 'DROP TABLE' = 58, 'DROP VIEW' = 59, 'DROP DICTIONARY' = 60, 'DROP FUNCTION' = 61, 'DROP NAMED COLLECTION' = 62, 'DROP' = 63, 'UNDROP TABLE' = 64, 'TRUNCATE' = 65, 'OPTIMIZE' = 66, 'BACKUP' = 67, 'KILL QUERY' = 68, 'KILL TRANSACTION' = 69, 'MOVE PARTITION BETWEEN SHARDS' = 70, 'CREATE USER' = 71, 'ALTER USER' = 72, 'DROP USER' = 73, 'CREATE ROLE' = 74, 'ALTER ROLE' = 75, 'DROP ROLE' = 76, 'ROLE ADMIN' = 77, 'CREATE ROW POLICY' = 78, 'ALTER ROW POLICY' = 79, 'DROP ROW POLICY' = 80, 'CREATE QUOTA' = 81, 'ALTER QUOTA' = 82, 'DROP QUOTA' = 83, 'CREATE SETTINGS PROFILE' = 84, 'ALTER SETTINGS PROFILE' = 85, 'DROP SETTINGS PROFILE' = 86, 'SHOW USERS' = 87, 'SHOW ROLES' = 88, 'SHOW ROW POLICIES' = 89, 'SHOW QUOTAS' = 90, 'SHOW SETTINGS PROFILES' = 91, 'SHOW ACCESS' = 92, 'ACCESS MANAGEMENT' = 93, 'SHOW NAMED COLLECTIONS' = 94, 'SHOW NAMED COLLECTIONS SECRETS' = 95, 'NAMED COLLECTION' = 96, 'NAMED COLLECTION ADMIN' = 97, 'SYSTEM SHUTDOWN' = 98, 'SYSTEM DROP DNS CACHE' = 99, 'SYSTEM DROP MARK CACHE' = 100, 'SYSTEM DROP UNCOMPRESSED CACHE' = 101, 'SYSTEM DROP MMAP CACHE' = 102, 'SYSTEM DROP QUERY CACHE' = 103, 'SYSTEM DROP COMPILED EXPRESSION CACHE' = 104, 'SYSTEM DROP FILESYSTEM CACHE' = 105, 'SYSTEM SYNC FILESYSTEM CACHE' = 106, 'SYSTEM DROP SCHEMA CACHE' = 107, 'SYSTEM DROP S3 CLIENT CACHE' = 108, 'SYSTEM DROP CACHE' = 109, 'SYSTEM RELOAD CONFIG' = 110, 'SYSTEM RELOAD USERS' = 111, 'SYSTEM RELOAD DICTIONARY' = 112, 'SYSTEM RELOAD MODEL' = 113, 'SYSTEM RELOAD FUNCTION' = 114, 'SYSTEM RELOAD EMBEDDED DICTIONARIES' = 115, 'SYSTEM RELOAD' = 116, 'SYSTEM RESTART DISK' = 117, 'SYSTEM MERGES' = 118, 'SYSTEM TTL MERGES' = 119, 'SYSTEM FETCHES' = 120, 'SYSTEM MOVES' = 121, 'SYSTEM PULLING REPLICATION LOG' = 122, 'SYSTEM DISTRIBUTED SENDS' = 123, 'SYSTEM REPLICATED SENDS' = 124, 'SYSTEM SENDS' = 125, 'SYSTEM REPLICATION QUEUES' = 126, 'SYSTEM DROP REPLICA' = 127, 'SYSTEM SYNC REPLICA' = 128, 'SYSTEM RESTART REPLICA' = 129, 'SYSTEM RESTORE REPLICA' = 130, 'SYSTEM WAIT LOADING PARTS' = 131, 'SYSTEM SYNC DATABASE REPLICA' = 132, 'SYSTEM SYNC TRANSACTION LOG' = 133, 'SYSTEM SYNC FILE CACHE' = 134, 'SYSTEM FLUSH DISTRIBUTED' = 135, 'SYSTEM FLUSH LOGS' = 136, 'SYSTEM FLUSH ASYNC INSERT QUEUE' = 137, 'SYSTEM FLUSH' = 138, 'SYSTEM THREAD FUZZER' = 139, 'SYSTEM UNFREEZE' = 140, 'SYSTEM FAILPOINT' = 141, 'SYSTEM LISTEN' = 142, 'SYSTEM JEMALLOC' = 143, 'SYSTEM' = 144, 'dictGet' = 145, 'displaySecretsInShowAndSelect' = 146, 'addressToLine' = 147, 'addressToLineWithInlines' = 148, 'addressToSymbol' = 149, 'demangle' = 150, 'INTROSPECTION' = 151, 'FILE' = 152, 'URL' = 153, 'REMOTE' = 154, 'MONGO' = 155, 'REDIS' = 156, 'MEILISEARCH' = 157, 'MYSQL' = 158, 'POSTGRES' = 159, 'SQLITE' = 160, 'ODBC' = 161, 'JDBC' = 162, 'HDFS' = 163, 'S3' = 164, 'HIVE' = 165, 'AZURE' = 166, 'SOURCES' = 167, 'CLUSTER' = 168, 'ALL' = 169, 'NONE' = 170), `aliases` Array(String), `level` Nullable(Enum8('GLOBAL' = 0, 'DATABASE' = 1, 'TABLE' = 2, 'DICTIONARY' = 3, 'VIEW' = 4, 'COLUMN' = 5, 'NAMED_COLLECTION' = 6)), - `parent_group` Nullable(Enum16('SHOW DATABASES' = 0, 'SHOW TABLES' = 1, 'SHOW COLUMNS' = 2, 'SHOW DICTIONARIES' = 3, 'SHOW' = 4, 'SHOW FILESYSTEM CACHES' = 5, 'SELECT' = 6, 'INSERT' = 7, 'ALTER UPDATE' = 8, 'ALTER DELETE' = 9, 'ALTER ADD COLUMN' = 10, 'ALTER MODIFY COLUMN' = 11, 'ALTER DROP COLUMN' = 12, 'ALTER COMMENT COLUMN' = 13, 'ALTER CLEAR COLUMN' = 14, 'ALTER RENAME COLUMN' = 15, 'ALTER MATERIALIZE COLUMN' = 16, 'ALTER COLUMN' = 17, 'ALTER MODIFY COMMENT' = 18, 'ALTER ORDER BY' = 19, 'ALTER SAMPLE BY' = 20, 'ALTER ADD INDEX' = 21, 'ALTER DROP INDEX' = 22, 'ALTER MATERIALIZE INDEX' = 23, 'ALTER CLEAR INDEX' = 24, 'ALTER INDEX' = 25, 'ALTER ADD PROJECTION' = 26, 'ALTER DROP PROJECTION' = 27, 'ALTER MATERIALIZE PROJECTION' = 28, 'ALTER CLEAR PROJECTION' = 29, 'ALTER PROJECTION' = 30, 'ALTER ADD CONSTRAINT' = 31, 'ALTER DROP CONSTRAINT' = 32, 'ALTER CONSTRAINT' = 33, 'ALTER TTL' = 34, 'ALTER MATERIALIZE TTL' = 35, 'ALTER SETTINGS' = 36, 'ALTER MOVE PARTITION' = 37, 'ALTER FETCH PARTITION' = 38, 'ALTER FREEZE PARTITION' = 39, 'ALTER DATABASE SETTINGS' = 40, 'ALTER NAMED COLLECTION' = 41, 'ALTER TABLE' = 42, 'ALTER DATABASE' = 43, 'ALTER VIEW REFRESH' = 44, 'ALTER VIEW MODIFY QUERY' = 45, 'ALTER VIEW' = 46, 'ALTER' = 47, 'CREATE DATABASE' = 48, 'CREATE TABLE' = 49, 'CREATE VIEW' = 50, 'CREATE DICTIONARY' = 51, 'CREATE TEMPORARY TABLE' = 52, 'CREATE ARBITRARY TEMPORARY TABLE' = 53, 'CREATE FUNCTION' = 54, 'CREATE NAMED COLLECTION' = 55, 'CREATE' = 56, 'DROP DATABASE' = 57, 'DROP TABLE' = 58, 'DROP VIEW' = 59, 'DROP DICTIONARY' = 60, 'DROP FUNCTION' = 61, 'DROP NAMED COLLECTION' = 62, 'DROP' = 63, 'UNDROP TABLE' = 64, 'TRUNCATE' = 65, 'OPTIMIZE' = 66, 'BACKUP' = 67, 'KILL QUERY' = 68, 'KILL TRANSACTION' = 69, 'MOVE PARTITION BETWEEN SHARDS' = 70, 'CREATE USER' = 71, 'ALTER USER' = 72, 'DROP USER' = 73, 'CREATE ROLE' = 74, 'ALTER ROLE' = 75, 'DROP ROLE' = 76, 'ROLE ADMIN' = 77, 'CREATE ROW POLICY' = 78, 'ALTER ROW POLICY' = 79, 'DROP ROW POLICY' = 80, 'CREATE QUOTA' = 81, 'ALTER QUOTA' = 82, 'DROP QUOTA' = 83, 'CREATE SETTINGS PROFILE' = 84, 'ALTER SETTINGS PROFILE' = 85, 'DROP SETTINGS PROFILE' = 86, 'SHOW USERS' = 87, 'SHOW ROLES' = 88, 'SHOW ROW POLICIES' = 89, 'SHOW QUOTAS' = 90, 'SHOW SETTINGS PROFILES' = 91, 'SHOW ACCESS' = 92, 'ACCESS MANAGEMENT' = 93, 'SHOW NAMED COLLECTIONS' = 94, 'SHOW NAMED COLLECTIONS SECRETS' = 95, 'NAMED COLLECTION' = 96, 'NAMED COLLECTION ADMIN' = 97, 'SYSTEM SHUTDOWN' = 98, 'SYSTEM DROP DNS CACHE' = 99, 'SYSTEM DROP MARK CACHE' = 100, 'SYSTEM DROP UNCOMPRESSED CACHE' = 101, 'SYSTEM DROP MMAP CACHE' = 102, 'SYSTEM DROP QUERY CACHE' = 103, 'SYSTEM DROP COMPILED EXPRESSION CACHE' = 104, 'SYSTEM DROP FILESYSTEM CACHE' = 105, 'SYSTEM SYNC FILESYSTEM CACHE' = 106, 'SYSTEM DROP SCHEMA CACHE' = 107, 'SYSTEM DROP S3 CLIENT CACHE' = 108, 'SYSTEM DROP CACHE' = 109, 'SYSTEM RELOAD CONFIG' = 110, 'SYSTEM RELOAD USERS' = 111, 'SYSTEM RELOAD DICTIONARY' = 112, 'SYSTEM RELOAD MODEL' = 113, 'SYSTEM RELOAD FUNCTION' = 114, 'SYSTEM RELOAD EMBEDDED DICTIONARIES' = 115, 'SYSTEM RELOAD' = 116, 'SYSTEM RESTART DISK' = 117, 'SYSTEM MERGES' = 118, 'SYSTEM TTL MERGES' = 119, 'SYSTEM FETCHES' = 120, 'SYSTEM MOVES' = 121, 'SYSTEM PULLING REPLICATION LOG' = 122, 'SYSTEM DISTRIBUTED SENDS' = 123, 'SYSTEM REPLICATED SENDS' = 124, 'SYSTEM SENDS' = 125, 'SYSTEM REPLICATION QUEUES' = 126, 'SYSTEM DROP REPLICA' = 127, 'SYSTEM SYNC REPLICA' = 128, 'SYSTEM RESTART REPLICA' = 129, 'SYSTEM RESTORE REPLICA' = 130, 'SYSTEM WAIT LOADING PARTS' = 131, 'SYSTEM SYNC DATABASE REPLICA' = 132, 'SYSTEM SYNC TRANSACTION LOG' = 133, 'SYSTEM SYNC FILE CACHE' = 134, 'SYSTEM FLUSH DISTRIBUTED' = 135, 'SYSTEM FLUSH LOGS' = 136, 'SYSTEM FLUSH ASYNC INSERT QUEUE' = 137, 'SYSTEM FLUSH' = 138, 'SYSTEM THREAD FUZZER' = 139, 'SYSTEM UNFREEZE' = 140, 'SYSTEM FAILPOINT' = 141, 'SYSTEM LISTEN' = 142, 'SYSTEM' = 143, 'dictGet' = 144, 'displaySecretsInShowAndSelect' = 145, 'addressToLine' = 146, 'addressToLineWithInlines' = 147, 'addressToSymbol' = 148, 'demangle' = 149, 'INTROSPECTION' = 150, 'FILE' = 151, 'URL' = 152, 'REMOTE' = 153, 'MONGO' = 154, 'REDIS' = 155, 'MEILISEARCH' = 156, 'MYSQL' = 157, 'POSTGRES' = 158, 'SQLITE' = 159, 'ODBC' = 160, 'JDBC' = 161, 'HDFS' = 162, 'S3' = 163, 'HIVE' = 164, 'AZURE' = 165, 'SOURCES' = 166, 'CLUSTER' = 167, 'ALL' = 168, 'NONE' = 169)) + `parent_group` Nullable(Enum16('SHOW DATABASES' = 0, 'SHOW TABLES' = 1, 'SHOW COLUMNS' = 2, 'SHOW DICTIONARIES' = 3, 'SHOW' = 4, 'SHOW FILESYSTEM CACHES' = 5, 'SELECT' = 6, 'INSERT' = 7, 'ALTER UPDATE' = 8, 'ALTER DELETE' = 9, 'ALTER ADD COLUMN' = 10, 'ALTER MODIFY COLUMN' = 11, 'ALTER DROP COLUMN' = 12, 'ALTER COMMENT COLUMN' = 13, 'ALTER CLEAR COLUMN' = 14, 'ALTER RENAME COLUMN' = 15, 'ALTER MATERIALIZE COLUMN' = 16, 'ALTER COLUMN' = 17, 'ALTER MODIFY COMMENT' = 18, 'ALTER ORDER BY' = 19, 'ALTER SAMPLE BY' = 20, 'ALTER ADD INDEX' = 21, 'ALTER DROP INDEX' = 22, 'ALTER MATERIALIZE INDEX' = 23, 'ALTER CLEAR INDEX' = 24, 'ALTER INDEX' = 25, 'ALTER ADD PROJECTION' = 26, 'ALTER DROP PROJECTION' = 27, 'ALTER MATERIALIZE PROJECTION' = 28, 'ALTER CLEAR PROJECTION' = 29, 'ALTER PROJECTION' = 30, 'ALTER ADD CONSTRAINT' = 31, 'ALTER DROP CONSTRAINT' = 32, 'ALTER CONSTRAINT' = 33, 'ALTER TTL' = 34, 'ALTER MATERIALIZE TTL' = 35, 'ALTER SETTINGS' = 36, 'ALTER MOVE PARTITION' = 37, 'ALTER FETCH PARTITION' = 38, 'ALTER FREEZE PARTITION' = 39, 'ALTER DATABASE SETTINGS' = 40, 'ALTER NAMED COLLECTION' = 41, 'ALTER TABLE' = 42, 'ALTER DATABASE' = 43, 'ALTER VIEW REFRESH' = 44, 'ALTER VIEW MODIFY QUERY' = 45, 'ALTER VIEW' = 46, 'ALTER' = 47, 'CREATE DATABASE' = 48, 'CREATE TABLE' = 49, 'CREATE VIEW' = 50, 'CREATE DICTIONARY' = 51, 'CREATE TEMPORARY TABLE' = 52, 'CREATE ARBITRARY TEMPORARY TABLE' = 53, 'CREATE FUNCTION' = 54, 'CREATE NAMED COLLECTION' = 55, 'CREATE' = 56, 'DROP DATABASE' = 57, 'DROP TABLE' = 58, 'DROP VIEW' = 59, 'DROP DICTIONARY' = 60, 'DROP FUNCTION' = 61, 'DROP NAMED COLLECTION' = 62, 'DROP' = 63, 'UNDROP TABLE' = 64, 'TRUNCATE' = 65, 'OPTIMIZE' = 66, 'BACKUP' = 67, 'KILL QUERY' = 68, 'KILL TRANSACTION' = 69, 'MOVE PARTITION BETWEEN SHARDS' = 70, 'CREATE USER' = 71, 'ALTER USER' = 72, 'DROP USER' = 73, 'CREATE ROLE' = 74, 'ALTER ROLE' = 75, 'DROP ROLE' = 76, 'ROLE ADMIN' = 77, 'CREATE ROW POLICY' = 78, 'ALTER ROW POLICY' = 79, 'DROP ROW POLICY' = 80, 'CREATE QUOTA' = 81, 'ALTER QUOTA' = 82, 'DROP QUOTA' = 83, 'CREATE SETTINGS PROFILE' = 84, 'ALTER SETTINGS PROFILE' = 85, 'DROP SETTINGS PROFILE' = 86, 'SHOW USERS' = 87, 'SHOW ROLES' = 88, 'SHOW ROW POLICIES' = 89, 'SHOW QUOTAS' = 90, 'SHOW SETTINGS PROFILES' = 91, 'SHOW ACCESS' = 92, 'ACCESS MANAGEMENT' = 93, 'SHOW NAMED COLLECTIONS' = 94, 'SHOW NAMED COLLECTIONS SECRETS' = 95, 'NAMED COLLECTION' = 96, 'NAMED COLLECTION ADMIN' = 97, 'SYSTEM SHUTDOWN' = 98, 'SYSTEM DROP DNS CACHE' = 99, 'SYSTEM DROP MARK CACHE' = 100, 'SYSTEM DROP UNCOMPRESSED CACHE' = 101, 'SYSTEM DROP MMAP CACHE' = 102, 'SYSTEM DROP QUERY CACHE' = 103, 'SYSTEM DROP COMPILED EXPRESSION CACHE' = 104, 'SYSTEM DROP FILESYSTEM CACHE' = 105, 'SYSTEM SYNC FILESYSTEM CACHE' = 106, 'SYSTEM DROP SCHEMA CACHE' = 107, 'SYSTEM DROP S3 CLIENT CACHE' = 108, 'SYSTEM DROP CACHE' = 109, 'SYSTEM RELOAD CONFIG' = 110, 'SYSTEM RELOAD USERS' = 111, 'SYSTEM RELOAD DICTIONARY' = 112, 'SYSTEM RELOAD MODEL' = 113, 'SYSTEM RELOAD FUNCTION' = 114, 'SYSTEM RELOAD EMBEDDED DICTIONARIES' = 115, 'SYSTEM RELOAD' = 116, 'SYSTEM RESTART DISK' = 117, 'SYSTEM MERGES' = 118, 'SYSTEM TTL MERGES' = 119, 'SYSTEM FETCHES' = 120, 'SYSTEM MOVES' = 121, 'SYSTEM PULLING REPLICATION LOG' = 122, 'SYSTEM DISTRIBUTED SENDS' = 123, 'SYSTEM REPLICATED SENDS' = 124, 'SYSTEM SENDS' = 125, 'SYSTEM REPLICATION QUEUES' = 126, 'SYSTEM DROP REPLICA' = 127, 'SYSTEM SYNC REPLICA' = 128, 'SYSTEM RESTART REPLICA' = 129, 'SYSTEM RESTORE REPLICA' = 130, 'SYSTEM WAIT LOADING PARTS' = 131, 'SYSTEM SYNC DATABASE REPLICA' = 132, 'SYSTEM SYNC TRANSACTION LOG' = 133, 'SYSTEM SYNC FILE CACHE' = 134, 'SYSTEM FLUSH DISTRIBUTED' = 135, 'SYSTEM FLUSH LOGS' = 136, 'SYSTEM FLUSH ASYNC INSERT QUEUE' = 137, 'SYSTEM FLUSH' = 138, 'SYSTEM THREAD FUZZER' = 139, 'SYSTEM UNFREEZE' = 140, 'SYSTEM FAILPOINT' = 141, 'SYSTEM LISTEN' = 142, 'SYSTEM JEMALLOC' = 143, 'SYSTEM' = 144, 'dictGet' = 145, 'displaySecretsInShowAndSelect' = 146, 'addressToLine' = 147, 'addressToLineWithInlines' = 148, 'addressToSymbol' = 149, 'demangle' = 150, 'INTROSPECTION' = 151, 'FILE' = 152, 'URL' = 153, 'REMOTE' = 154, 'MONGO' = 155, 'REDIS' = 156, 'MEILISEARCH' = 157, 'MYSQL' = 158, 'POSTGRES' = 159, 'SQLITE' = 160, 'ODBC' = 161, 'JDBC' = 162, 'HDFS' = 163, 'S3' = 164, 'HIVE' = 165, 'AZURE' = 166, 'SOURCES' = 167, 'CLUSTER' = 168, 'ALL' = 169, 'NONE' = 170)) ) ENGINE = SystemPrivileges COMMENT 'SYSTEM TABLE is built on the fly.' From d4068cf8ced2271a0d3c2dcc61f4c4e4e1b33eec Mon Sep 17 00:00:00 2001 From: MyroTk <44327070+MyroTk@users.noreply.github.com> Date: Fri, 21 Jun 2024 08:56:34 -0700 Subject: [PATCH 046/123] Update release_branches.yml --- .github/workflows/release_branches.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/release_branches.yml b/.github/workflows/release_branches.yml index 185493167554..14e36f549088 100644 --- a/.github/workflows/release_branches.yml +++ b/.github/workflows/release_branches.yml @@ -638,7 +638,7 @@ jobs: secrets: inherit with: runner_type: altinity-on-demand, altinity-type-cpx51, altinity-image-x86-app-docker-ce, altinity-setup-regression - commit: 17a81c07fc1f41fbee651e0ef0ca4b44e537e5b1 + commit: f1d3a6169b73f86828da3fbe85c99263054f82fb arch: release build_sha: ${{ github.event_name == 'pull_request' && github.event.pull_request.head.sha || github.event_name == 'release' && github.sha }} @@ -648,7 +648,7 @@ jobs: secrets: inherit with: runner_type: altinity-on-demand, altinity-type-cax41, altinity-image-arm-app-docker-ce, altinity-setup-regression - commit: 17a81c07fc1f41fbee651e0ef0ca4b44e537e5b1 + commit: f1d3a6169b73f86828da3fbe85c99263054f82fb arch: aarch64 build_sha: ${{ github.event_name == 'pull_request' && github.event.pull_request.head.sha || github.event_name == 'release' && github.sha }} From df0a3ff3a950c23f667741d00ea657182c5c639a Mon Sep 17 00:00:00 2001 From: MyroTk <44327070+MyroTk@users.noreply.github.com> Date: Fri, 21 Jun 2024 11:36:37 -0700 Subject: [PATCH 047/123] Update release_branches.yml --- .github/workflows/release_branches.yml | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/.github/workflows/release_branches.yml b/.github/workflows/release_branches.yml index 14e36f549088..68744be50e31 100644 --- a/.github/workflows/release_branches.yml +++ b/.github/workflows/release_branches.yml @@ -637,8 +637,8 @@ jobs: uses: ./.github/workflows/regression.yml secrets: inherit with: - runner_type: altinity-on-demand, altinity-type-cpx51, altinity-image-x86-app-docker-ce, altinity-setup-regression - commit: f1d3a6169b73f86828da3fbe85c99263054f82fb + runner_type: altinity-on-demand, altinity-type-cpx51, altinity-image-x86-system-ubuntu-22.04, altinity-setup-regression + commit: 17a81c07fc1f41fbee651e0ef0ca4b44e537e5b1 arch: release build_sha: ${{ github.event_name == 'pull_request' && github.event.pull_request.head.sha || github.event_name == 'release' && github.sha }} @@ -647,8 +647,8 @@ jobs: uses: ./.github/workflows/regression.yml secrets: inherit with: - runner_type: altinity-on-demand, altinity-type-cax41, altinity-image-arm-app-docker-ce, altinity-setup-regression - commit: f1d3a6169b73f86828da3fbe85c99263054f82fb + runner_type: altinity-on-demand, altinity-type-cax41, altinity-image-arm-system-ubuntu-22.04, altinity-setup-regression + commit: 17a81c07fc1f41fbee651e0ef0ca4b44e537e5b1 arch: aarch64 build_sha: ${{ github.event_name == 'pull_request' && github.event.pull_request.head.sha || github.event_name == 'release' && github.sha }} From 9f92e54b6550990b824ea85570c4cd667698ca7f Mon Sep 17 00:00:00 2001 From: MyroTk Date: Wed, 3 Jul 2024 13:29:34 -0700 Subject: [PATCH 048/123] update docker image tag --- tests/ci/build_check.py | 2 +- tests/ci/docker_images_check.py | 49 +++++++++++++++++++----------- tests/ci/docker_server.py | 3 +- tests/ci/functional_test_check.py | 2 +- tests/ci/install_check.py | 2 +- tests/ci/integration_test_check.py | 2 +- tests/ci/pr_info.py | 11 ++++++- tests/ci/stress_check.py | 2 +- 8 files changed, 49 insertions(+), 24 deletions(-) diff --git a/tests/ci/build_check.py b/tests/ci/build_check.py index 4a1517a63ede..df72d19693ab 100644 --- a/tests/ci/build_check.py +++ b/tests/ci/build_check.py @@ -249,7 +249,7 @@ def main(): # e.g. `binary_darwin_aarch64/clickhouse` for `binary_darwin` check_for_success_run(s3_helper, f"{s3_path_prefix}/", build_name, version) - docker_image = get_image_with_version(IMAGES_PATH, IMAGE_NAME) + docker_image = get_image_with_version(IMAGES_PATH, IMAGE_NAME, version=pr_info.docker_image_tag) image_version = docker_image.version logging.info("Got version from repo %s", version.string) diff --git a/tests/ci/docker_images_check.py b/tests/ci/docker_images_check.py index 54c563d36c57..b7a80f86e917 100644 --- a/tests/ci/docker_images_check.py +++ b/tests/ci/docker_images_check.py @@ -91,23 +91,38 @@ def get_changed_docker_images( str(files_changed), ) - # Rebuild all images - changed_images = [DockerImage(dockerfile_dir, image_description["name"], image_description.get("only_amd64", False)) for dockerfile_dir, image_description in images_dict.items()] - - # for dockerfile_dir, image_description in images_dict.items(): - # for f in files_changed: - # if f.startswith(dockerfile_dir): - # name = image_description["name"] - # only_amd64 = image_description.get("only_amd64", False) - # logging.info( - # "Found changed file '%s' which affects " - # "docker image '%s' with path '%s'", - # f, - # name, - # dockerfile_dir, - # ) - # changed_images.append(DockerImage(dockerfile_dir, name, only_amd64)) - # break + # Find changed images + all_images = [] + changed_images = [] + for dockerfile_dir, image_description in images_dict.items(): + all_images.append(DockerImage(dockerfile_dir, image_description["name"], image_description.get("only_amd64", False))) + for f in files_changed: + if f.startswith(dockerfile_dir): + name = image_description["name"] + only_amd64 = image_description.get("only_amd64", False) + logging.info( + "Found changed file '%s' which affects " + "docker image '%s' with path '%s'", + f, + name, + dockerfile_dir, + ) + changed_images.append(DockerImage(dockerfile_dir, name, only_amd64)) + break + + # Rebuild all on opened PR or during release + if pr_info.event['action'] in ['opened', 'reopened', 'published', 'prereleased']: + changed_images = all_images + + # Check that image for the PR exists + elif pr_info.event['action'] == 'synchronize': + unchanged_images = [ + image for image in all_images if image not in changed_images + ] + logging.info(f"Unchanged images: {unchanged_images}") + for image in unchanged_images: + if subprocess.run(f"docker manifest inspect {image.repo}:{pr_info.number}", shell=True).returncode != 0: + changed_images.append(image) # The order is important: dependents should go later than bases, so that # they are built with updated base versions. diff --git a/tests/ci/docker_server.py b/tests/ci/docker_server.py index c713fbb5f420..f6d7612f8a13 100644 --- a/tests/ci/docker_server.py +++ b/tests/ci/docker_server.py @@ -245,7 +245,7 @@ def build_and_push_image( init_args = ["docker", "buildx", "build"] if push: init_args.append("--push") - init_args.append("--output=type=image,push-by-digest=true") + init_args.append("--output=type=image") init_args.append(f"--tag={image.repo}") else: init_args.append("--output=type=docker") @@ -330,6 +330,7 @@ def main(): image = DockerImage(args.image_path, args.image_repo, False) args.release_type = auto_release_type(args.version, args.release_type) tags = gen_tags(args.version, args.release_type) + tags.append(f"{pr_info.number}-{args.version}") NAME = f"Docker image {image.repo} building check" pr_info = None if CI: diff --git a/tests/ci/functional_test_check.py b/tests/ci/functional_test_check.py index 5046c13f81bd..d5109f7ce5a2 100644 --- a/tests/ci/functional_test_check.py +++ b/tests/ci/functional_test_check.py @@ -303,7 +303,7 @@ def main(): sys.exit(0) image_name = get_image_name(check_name) - docker_image = get_image_with_version(reports_path, image_name) + docker_image = get_image_with_version(reports_path, image_name, version=pr_info.docker_image_tag) packages_path = temp_path / "packages" packages_path.mkdir(parents=True, exist_ok=True) diff --git a/tests/ci/install_check.py b/tests/ci/install_check.py index 44bb4c2ffd09..b2abcf6db10d 100644 --- a/tests/ci/install_check.py +++ b/tests/ci/install_check.py @@ -290,7 +290,7 @@ def main(): sys.exit(0) docker_images = { - name: get_image_with_version(REPORTS_PATH, name, args.download) + name: get_image_with_version(REPORTS_PATH, name, args.download, pr_info.docker_image_tag) for name in (RPM_IMAGE, DEB_IMAGE) } prepare_test_scripts() diff --git a/tests/ci/integration_test_check.py b/tests/ci/integration_test_check.py index 2b77ddaa8ff4..fa0c8a84ae53 100644 --- a/tests/ci/integration_test_check.py +++ b/tests/ci/integration_test_check.py @@ -219,7 +219,7 @@ def main(): # logging.info("Check is already finished according to github status, exiting") # sys.exit(0) - images = get_images_with_versions(reports_path, IMAGES) + images = get_images_with_versions(reports_path, IMAGES, version=pr_info.docker_image_tag) result_path = temp_path / "output_dir" result_path.mkdir(parents=True, exist_ok=True) diff --git a/tests/ci/pr_info.py b/tests/ci/pr_info.py index 8e7da53453f9..2b0110264ccd 100644 --- a/tests/ci/pr_info.py +++ b/tests/ci/pr_info.py @@ -13,6 +13,10 @@ GITHUB_RUN_URL, GITHUB_EVENT_PATH, ) +from version_helper import ( + Git, + get_version_from_repo, +) FORCE_TESTS_LABEL = "force tests" SKIP_MERGEABLE_CHECK_LABEL = "skip mergeable check" @@ -115,6 +119,7 @@ def __init__( if "pull_request" in github_event: # pull request and other similar events self.number = github_event["pull_request"]["number"] # type: int + self.docker_image_tag = str(self.number) # type: str if pr_event_from_api: try: response = get_gh_api( @@ -191,6 +196,7 @@ def __init__( if pull_request is None or pull_request["state"] == "closed": # it's merged PR to master self.number = 0 + self.docker_image_tag = str(self.number) + "-" + str(self.sha) self.labels = set() self.pr_html_url = f"{repo_prefix}/commits/{ref}" self.base_ref = ref @@ -202,6 +208,7 @@ def __init__( ) else: self.number = pull_request["number"] + self.docker_image_tag = str(self.number) self.labels = {label["name"] for label in pull_request["labels"]} self.base_ref = pull_request["base"]["ref"] @@ -247,7 +254,9 @@ def __init__( self.sha = os.getenv( "GITHUB_SHA", "0000000000000000000000000000000000000000" ) - self.number = 0 + version = get_version_from_repo(git=Git(True)) + self.number = f"{version.major}.{version.minor}.{version.patch}" + self.docker_image_tag = str(self.number) + "-" + str(self.sha) self.labels = set() repo_prefix = f"{GITHUB_SERVER_URL}/{GITHUB_REPOSITORY}" self.task_url = GITHUB_RUN_URL diff --git a/tests/ci/stress_check.py b/tests/ci/stress_check.py index 012bd1bcf88c..02d18b568baa 100644 --- a/tests/ci/stress_check.py +++ b/tests/ci/stress_check.py @@ -124,7 +124,7 @@ def run_stress_test(docker_image_name: str) -> None: logging.info("Check is already finished according to github status, exiting") sys.exit(0) - docker_image = get_image_with_version(reports_path, docker_image_name) + docker_image = get_image_with_version(reports_path, docker_image_name, version=pr_info.docker_image_tag) packages_path = temp_path / "packages" packages_path.mkdir(parents=True, exist_ok=True) From b03d72eae130bad10e85531373f4deaeb4eeb754 Mon Sep 17 00:00:00 2001 From: MyroTk Date: Wed, 3 Jul 2024 14:16:01 -0700 Subject: [PATCH 049/123] remove docker app tag --- .github/workflows/release_branches.yml | 50 +++++++++++++------------- 1 file changed, 25 insertions(+), 25 deletions(-) diff --git a/.github/workflows/release_branches.yml b/.github/workflows/release_branches.yml index 2305b0b53ad7..ab151b720cb1 100644 --- a/.github/workflows/release_branches.yml +++ b/.github/workflows/release_branches.yml @@ -31,7 +31,7 @@ on: # yamllint disable-line rule:truthy jobs: DockerHubPushAarch64: - runs-on: [self-hosted, altinity-on-demand, altinity-type-cax41, altinity-in-hel1, altinity-image-arm-app-docker-ce] + runs-on: [self-hosted, altinity-on-demand, altinity-type-cax41, altinity-in-hel1, altinity-image-arm-system-ubuntu-22.04] steps: - name: Check out repository code uses: Altinity/checkout@19599efdf36c4f3f30eb55d5bb388896faea69f6 @@ -53,7 +53,7 @@ jobs: path: ${{ runner.temp }}/docker_images_check/changed_images_aarch64.json DockerHubPushAmd64: - runs-on: [self-hosted, altinity-on-demand, altinity-type-cpx51, altinity-in-ash, altinity-image-x86-app-docker-ce] + runs-on: [self-hosted, altinity-on-demand, altinity-type-cpx51, altinity-in-ash, altinity-image-x86-system-ubuntu-22.04] steps: - name: Check out repository code uses: Altinity/checkout@19599efdf36c4f3f30eb55d5bb388896faea69f6 @@ -76,7 +76,7 @@ jobs: DockerHubPush: needs: [DockerHubPushAmd64, DockerHubPushAarch64] - runs-on: [self-hosted, altinity-on-demand, altinity-type-cpx41, altinity-image-x86-app-docker-ce] + runs-on: [self-hosted, altinity-on-demand, altinity-type-cpx41, altinity-image-x86-system-ubuntu-22.04] steps: - name: Check out repository code uses: Altinity/checkout@19599efdf36c4f3f30eb55d5bb388896faea69f6 @@ -120,7 +120,7 @@ jobs: secrets: inherit with: test_name: Compatibility check X86 - runner_type: altinity-on-demand, altinity-type-cpx51, altinity-image-x86-app-docker-ce + runner_type: altinity-on-demand, altinity-type-cpx51, altinity-image-x86-system-ubuntu-22.04 timeout_minutes: 180 run_command: | cd "$REPO_COPY/tests/ci" @@ -132,7 +132,7 @@ jobs: secrets: inherit with: test_name: Compatibility check Aarch64 - runner_type: altinity-on-demand, altinity-type-cax41, altinity-image-arm-app-docker-ce + runner_type: altinity-on-demand, altinity-type-cax41, altinity-image-arm-system-ubuntu-22.04 timeout_minutes: 180 run_command: | cd "$REPO_COPY/tests/ci" @@ -149,7 +149,7 @@ jobs: build_name: package_release checkout_depth: 0 timeout_minutes: 180 - runner_type: altinity-on-demand, altinity-setup-builder, altinity-type-ccx53, altinity-in-ash, altinity-image-x86-app-docker-ce + runner_type: altinity-on-demand, altinity-setup-builder, altinity-type-ccx53, altinity-in-ash, altinity-image-x86-system-ubuntu-22.04 additional_envs: | CLICKHOUSE_STABLE_VERSION_SUFFIX=altinitystable @@ -161,7 +161,7 @@ jobs: build_name: package_aarch64 checkout_depth: 0 timeout_minutes: 180 - runner_type: altinity-on-demand, altinity-setup-builder, altinity-type-ccx53, altinity-in-ash, altinity-image-x86-app-docker-ce + runner_type: altinity-on-demand, altinity-setup-builder, altinity-type-ccx53, altinity-in-ash, altinity-image-x86-system-ubuntu-22.04 additional_envs: | CLICKHOUSE_STABLE_VERSION_SUFFIX=altinitystable @@ -173,7 +173,7 @@ jobs: build_name: package_asan checkout_depth: 0 timeout_minutes: 180 - runner_type: altinity-on-demand, altinity-setup-builder, altinity-type-ccx53, altinity-in-ash, altinity-image-x86-app-docker-ce + runner_type: altinity-on-demand, altinity-setup-builder, altinity-type-ccx53, altinity-in-ash, altinity-image-x86-system-ubuntu-22.04 additional_envs: | CLICKHOUSE_STABLE_VERSION_SUFFIX=altinitystable @@ -185,7 +185,7 @@ jobs: build_name: package_ubsan checkout_depth: 0 timeout_minutes: 180 - runner_type: altinity-on-demand, altinity-setup-builder, altinity-type-ccx53, altinity-in-ash, altinity-image-x86-app-docker-ce + runner_type: altinity-on-demand, altinity-setup-builder, altinity-type-ccx53, altinity-in-ash, altinity-image-x86-system-ubuntu-22.04 additional_envs: | CLICKHOUSE_STABLE_VERSION_SUFFIX=altinitystable @@ -197,7 +197,7 @@ jobs: build_name: package_tsan checkout_depth: 0 timeout_minutes: 180 - runner_type: altinity-on-demand, altinity-setup-builder, altinity-type-ccx53, altinity-in-ash, altinity-image-x86-app-docker-ce + runner_type: altinity-on-demand, altinity-setup-builder, altinity-type-ccx53, altinity-in-ash, altinity-image-x86-system-ubuntu-22.04 additional_envs: | CLICKHOUSE_STABLE_VERSION_SUFFIX=altinitystable @@ -209,7 +209,7 @@ jobs: build_name: package_msan checkout_depth: 0 timeout_minutes: 180 - runner_type: altinity-on-demand, altinity-setup-builder, altinity-type-ccx53, altinity-in-ash, altinity-image-x86-app-docker-ce + runner_type: altinity-on-demand, altinity-setup-builder, altinity-type-ccx53, altinity-in-ash, altinity-image-x86-system-ubuntu-22.04 additional_envs: | CLICKHOUSE_STABLE_VERSION_SUFFIX=altinitystable @@ -221,7 +221,7 @@ jobs: build_name: package_debug checkout_depth: 0 timeout_minutes: 180 - runner_type: altinity-on-demand, altinity-setup-builder, altinity-type-ccx53, altinity-in-ash, altinity-image-x86-app-docker-ce + runner_type: altinity-on-demand, altinity-setup-builder, altinity-type-ccx53, altinity-in-ash, altinity-image-x86-system-ubuntu-22.04 additional_envs: | CLICKHOUSE_STABLE_VERSION_SUFFIX=altinitystable @@ -232,7 +232,7 @@ jobs: needs: - BuilderDebRelease - BuilderDebAarch64 - runs-on: [self-hosted, altinity-on-demand, altinity-type-cpx51, altinity-image-x86-app-docker-ce] + runs-on: [self-hosted, altinity-on-demand, altinity-type-cpx51, altinity-image-x86-system-ubuntu-22.04] timeout-minutes: 180 steps: - name: Check out repository code @@ -268,7 +268,7 @@ jobs: secrets: inherit with: test_name: ClickHouse build check - runner_type: altinity-on-demand, altinity-setup-reporter, altinity-type-cax11, altinity-image-x86-app-docker-ce + runner_type: altinity-on-demand, altinity-setup-reporter, altinity-type-cax11, altinity-image-x86-system-ubuntu-22.04 timeout_minutes: 180 additional_envs: | NEEDS_DATA< Date: Wed, 3 Jul 2024 16:08:34 -0700 Subject: [PATCH 050/123] docker server tag fix --- tests/ci/docker_server.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/ci/docker_server.py b/tests/ci/docker_server.py index f6d7612f8a13..da15ad2bf4fd 100644 --- a/tests/ci/docker_server.py +++ b/tests/ci/docker_server.py @@ -330,7 +330,6 @@ def main(): image = DockerImage(args.image_path, args.image_repo, False) args.release_type = auto_release_type(args.version, args.release_type) tags = gen_tags(args.version, args.release_type) - tags.append(f"{pr_info.number}-{args.version}") NAME = f"Docker image {image.repo} building check" pr_info = None if CI: @@ -339,6 +338,7 @@ def main(): args.bucket_prefix = ( f"{S3_DOWNLOAD}/{S3_BUILDS_BUCKET}/{release_or_pr}/{pr_info.sha}" ) + tags.append(f"{pr_info.number}-{args.version}") if args.push: subprocess.check_output( # pylint: disable=unexpected-keyword-arg From 36679257c20d2ea93deab0f83e188696b3def3fd Mon Sep 17 00:00:00 2001 From: MyroTk Date: Wed, 24 Jul 2024 08:21:08 -0700 Subject: [PATCH 051/123] match pr_info with 23.3 --- .github/workflows/release_branches.yml | 10 +++++----- tests/ci/pr_info.py | 4 ++-- 2 files changed, 7 insertions(+), 7 deletions(-) diff --git a/.github/workflows/release_branches.yml b/.github/workflows/release_branches.yml index ab151b720cb1..fe26c39c3b61 100644 --- a/.github/workflows/release_branches.yml +++ b/.github/workflows/release_branches.yml @@ -45,7 +45,7 @@ jobs: env: RUNNER_IP: "$(hostname -I | cut -d ' ' -f 1)" RUNNER_SSH_COMMAND: "ssh -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null root@$(hostname -I | cut -d ' ' -f 1)" - + - name: Upload images files to artifacts uses: actions/upload-artifact@v4 with: @@ -364,7 +364,7 @@ jobs: run_command: | cd "$REPO_COPY/tests/ci" python3 functional_test_check.py "$CHECK_NAME" "$KILL_TIMEOUT" - + FunctionalStatelessTestTsan: needs: [BuilderDebTsan] uses: ./.github/workflows/reusable_test.yml @@ -378,7 +378,7 @@ jobs: run_command: | cd "$REPO_COPY/tests/ci" python3 functional_test_check.py "$CHECK_NAME" "$KILL_TIMEOUT" - + FunctionalStatelessTestUBsan: needs: [BuilderDebUBsan] uses: ./.github/workflows/reusable_test.yml @@ -592,7 +592,7 @@ jobs: run_command: | cd "$REPO_COPY/tests/ci" python3 integration_test_check.py "$CHECK_NAME" - + IntegrationTestsAnalyzerAsan: needs: [BuilderDebAsan] uses: ./.github/workflows/reusable_test.yml @@ -605,7 +605,7 @@ jobs: run_command: | cd "$REPO_COPY/tests/ci" python3 integration_test_check.py "$CHECK_NAME" - + IntegrationTestsTsan: needs: [BuilderDebTsan] uses: ./.github/workflows/reusable_test.yml diff --git a/tests/ci/pr_info.py b/tests/ci/pr_info.py index 2b0110264ccd..3fc029958938 100644 --- a/tests/ci/pr_info.py +++ b/tests/ci/pr_info.py @@ -102,6 +102,7 @@ def __init__( # release_pr and merged_pr are used for docker images additional cache self.release_pr = 0 self.merged_pr = 0 + self.version = get_version_from_repo(git=Git(True)) ref = github_event.get("ref", "refs/heads/master") if ref and ref.startswith("refs/heads/"): ref = ref[11:] @@ -254,8 +255,7 @@ def __init__( self.sha = os.getenv( "GITHUB_SHA", "0000000000000000000000000000000000000000" ) - version = get_version_from_repo(git=Git(True)) - self.number = f"{version.major}.{version.minor}.{version.patch}" + self.number = f"{self.version.major}.{self.version.minor}.{self.version.patch}" self.docker_image_tag = str(self.number) + "-" + str(self.sha) self.labels = set() repo_prefix = f"{GITHUB_SERVER_URL}/{GITHUB_REPOSITORY}" From 7efbfa45d518fca03a3b81e715acee4f598ba6a5 Mon Sep 17 00:00:00 2001 From: MyroTk <44327070+MyroTk@users.noreply.github.com> Date: Thu, 25 Jul 2024 06:32:33 -0700 Subject: [PATCH 052/123] fix runner arch --- .github/workflows/release_branches.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/release_branches.yml b/.github/workflows/release_branches.yml index fe26c39c3b61..dbd1e35a9cee 100644 --- a/.github/workflows/release_branches.yml +++ b/.github/workflows/release_branches.yml @@ -268,7 +268,7 @@ jobs: secrets: inherit with: test_name: ClickHouse build check - runner_type: altinity-on-demand, altinity-setup-reporter, altinity-type-cax11, altinity-image-x86-system-ubuntu-22.04 + runner_type: altinity-on-demand, altinity-setup-reporter, altinity-type-cax11, altinity-image-arm-system-ubuntu-22.04 timeout_minutes: 180 additional_envs: | NEEDS_DATA< Date: Fri, 26 Jul 2024 11:30:37 -0700 Subject: [PATCH 053/123] Update release_branches.yml --- .github/workflows/release_branches.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/release_branches.yml b/.github/workflows/release_branches.yml index dbd1e35a9cee..75cdfa8dd91f 100644 --- a/.github/workflows/release_branches.yml +++ b/.github/workflows/release_branches.yml @@ -657,7 +657,7 @@ jobs: SignRelease: needs: [BuilderDebRelease] - runs-on: [self-hosted, altinity-on-demand, altinity-setup-reporter, altinity-type-cax11, altinity-in-ash, altinity-image-arm-system-ubuntu-22.04] + runs-on: [self-hosted, altinity-on-demand, altinity-setup-reporter, altinity-type-cax11, altinity-image-arm-system-ubuntu-22.04] timeout-minutes: 180 steps: - name: Set envs From 39cd464be736b1b93c3d5384b9ffb42d6c7dfede Mon Sep 17 00:00:00 2001 From: MyroTk <44327070+MyroTk@users.noreply.github.com> Date: Fri, 6 Sep 2024 16:26:33 +0000 Subject: [PATCH 054/123] Reuse unchanged images PR #382 rebased onto 23.8.16 --- minor-change | 1 + tests/ci/build_check.py | 2 +- tests/ci/docker_images_check.py | 49 +++++++++++++++++++----------- tests/ci/functional_test_check.py | 2 +- tests/ci/install_check.py | 2 +- tests/ci/integration_test_check.py | 2 +- tests/ci/pr_info.py | 4 +++ tests/ci/stress_check.py | 2 +- 8 files changed, 42 insertions(+), 22 deletions(-) create mode 100644 minor-change diff --git a/minor-change b/minor-change new file mode 100644 index 000000000000..e2fe9a368353 --- /dev/null +++ b/minor-change @@ -0,0 +1 @@ +new commit diff --git a/tests/ci/build_check.py b/tests/ci/build_check.py index e0bd37181a34..63b84fce4cb1 100644 --- a/tests/ci/build_check.py +++ b/tests/ci/build_check.py @@ -251,7 +251,7 @@ def main(): # e.g. `binary_darwin_aarch64/clickhouse` for `binary_darwin` check_for_success_run(s3_helper, f"{s3_path_prefix}/", build_name, version) - docker_image = get_image_with_version(IMAGES_PATH, IMAGE_NAME) + docker_image = get_image_with_version(IMAGES_PATH, IMAGE_NAME, version=pr_info.docker_image_tag) image_version = docker_image.version logging.info("Got version from repo %s", version.string) diff --git a/tests/ci/docker_images_check.py b/tests/ci/docker_images_check.py index 54c563d36c57..b7a80f86e917 100644 --- a/tests/ci/docker_images_check.py +++ b/tests/ci/docker_images_check.py @@ -91,23 +91,38 @@ def get_changed_docker_images( str(files_changed), ) - # Rebuild all images - changed_images = [DockerImage(dockerfile_dir, image_description["name"], image_description.get("only_amd64", False)) for dockerfile_dir, image_description in images_dict.items()] - - # for dockerfile_dir, image_description in images_dict.items(): - # for f in files_changed: - # if f.startswith(dockerfile_dir): - # name = image_description["name"] - # only_amd64 = image_description.get("only_amd64", False) - # logging.info( - # "Found changed file '%s' which affects " - # "docker image '%s' with path '%s'", - # f, - # name, - # dockerfile_dir, - # ) - # changed_images.append(DockerImage(dockerfile_dir, name, only_amd64)) - # break + # Find changed images + all_images = [] + changed_images = [] + for dockerfile_dir, image_description in images_dict.items(): + all_images.append(DockerImage(dockerfile_dir, image_description["name"], image_description.get("only_amd64", False))) + for f in files_changed: + if f.startswith(dockerfile_dir): + name = image_description["name"] + only_amd64 = image_description.get("only_amd64", False) + logging.info( + "Found changed file '%s' which affects " + "docker image '%s' with path '%s'", + f, + name, + dockerfile_dir, + ) + changed_images.append(DockerImage(dockerfile_dir, name, only_amd64)) + break + + # Rebuild all on opened PR or during release + if pr_info.event['action'] in ['opened', 'reopened', 'published', 'prereleased']: + changed_images = all_images + + # Check that image for the PR exists + elif pr_info.event['action'] == 'synchronize': + unchanged_images = [ + image for image in all_images if image not in changed_images + ] + logging.info(f"Unchanged images: {unchanged_images}") + for image in unchanged_images: + if subprocess.run(f"docker manifest inspect {image.repo}:{pr_info.number}", shell=True).returncode != 0: + changed_images.append(image) # The order is important: dependents should go later than bases, so that # they are built with updated base versions. diff --git a/tests/ci/functional_test_check.py b/tests/ci/functional_test_check.py index 5046c13f81bd..d5109f7ce5a2 100644 --- a/tests/ci/functional_test_check.py +++ b/tests/ci/functional_test_check.py @@ -303,7 +303,7 @@ def main(): sys.exit(0) image_name = get_image_name(check_name) - docker_image = get_image_with_version(reports_path, image_name) + docker_image = get_image_with_version(reports_path, image_name, version=pr_info.docker_image_tag) packages_path = temp_path / "packages" packages_path.mkdir(parents=True, exist_ok=True) diff --git a/tests/ci/install_check.py b/tests/ci/install_check.py index 0a7eb4801a21..a4ae124073fa 100644 --- a/tests/ci/install_check.py +++ b/tests/ci/install_check.py @@ -290,7 +290,7 @@ def main(): sys.exit(0) docker_images = { - name: get_image_with_version(REPORTS_PATH, name, args.download) + name: get_image_with_version(REPORTS_PATH, name, args.download, pr_info.docker_image_tag) for name in (RPM_IMAGE, DEB_IMAGE) } prepare_test_scripts() diff --git a/tests/ci/integration_test_check.py b/tests/ci/integration_test_check.py index 2b77ddaa8ff4..fa0c8a84ae53 100644 --- a/tests/ci/integration_test_check.py +++ b/tests/ci/integration_test_check.py @@ -219,7 +219,7 @@ def main(): # logging.info("Check is already finished according to github status, exiting") # sys.exit(0) - images = get_images_with_versions(reports_path, IMAGES) + images = get_images_with_versions(reports_path, IMAGES, version=pr_info.docker_image_tag) result_path = temp_path / "output_dir" result_path.mkdir(parents=True, exist_ok=True) diff --git a/tests/ci/pr_info.py b/tests/ci/pr_info.py index 5aa95ca60af6..63c5869fb581 100644 --- a/tests/ci/pr_info.py +++ b/tests/ci/pr_info.py @@ -115,6 +115,7 @@ def __init__( if "pull_request" in github_event: # pull request and other similar events self.number = github_event["pull_request"]["number"] # type: int + self.docker_image_tag = str(self.number) # type: str if pr_event_from_api: try: response = get_gh_api( @@ -191,6 +192,7 @@ def __init__( if pull_request is None or pull_request["state"] == "closed": # it's merged PR to master self.number = 0 + self.docker_image_tag = str(self.number) + "-" + str(self.sha) self.labels = set() self.pr_html_url = f"{repo_prefix}/commits/{ref}" self.base_ref = ref @@ -202,6 +204,7 @@ def __init__( ) else: self.number = pull_request["number"] + self.docker_image_tag = str(self.number) self.labels = {label["name"] for label in pull_request["labels"]} self.base_ref = pull_request["base"]["ref"] @@ -248,6 +251,7 @@ def __init__( "GITHUB_SHA", "0000000000000000000000000000000000000000" ) self.number = 0 + self.docker_image_tag = str(self.number) + "-" + str(self.sha) self.labels = set() repo_prefix = f"{GITHUB_SERVER_URL}/{GITHUB_REPOSITORY}" self.task_url = GITHUB_RUN_URL diff --git a/tests/ci/stress_check.py b/tests/ci/stress_check.py index 012bd1bcf88c..02d18b568baa 100644 --- a/tests/ci/stress_check.py +++ b/tests/ci/stress_check.py @@ -124,7 +124,7 @@ def run_stress_test(docker_image_name: str) -> None: logging.info("Check is already finished according to github status, exiting") sys.exit(0) - docker_image = get_image_with_version(reports_path, docker_image_name) + docker_image = get_image_with_version(reports_path, docker_image_name, version=pr_info.docker_image_tag) packages_path = temp_path / "packages" packages_path.mkdir(parents=True, exist_ok=True) From 047df0d5dcaf740e078994bfb07452eee2c30b19 Mon Sep 17 00:00:00 2001 From: MyroTk <44327070+MyroTk@users.noreply.github.com> Date: Fri, 6 Sep 2024 10:21:24 -0700 Subject: [PATCH 055/123] Include version in pr_info --- tests/ci/pr_info.py | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/tests/ci/pr_info.py b/tests/ci/pr_info.py index 63c5869fb581..ad57bcecde34 100644 --- a/tests/ci/pr_info.py +++ b/tests/ci/pr_info.py @@ -13,6 +13,10 @@ GITHUB_RUN_URL, GITHUB_EVENT_PATH, ) +from version_helper import ( + Git, + get_version_from_repo, +) FORCE_TESTS_LABEL = "force tests" SKIP_MERGEABLE_CHECK_LABEL = "skip mergeable check" @@ -98,6 +102,7 @@ def __init__( # release_pr and merged_pr are used for docker images additional cache self.release_pr = 0 self.merged_pr = 0 + self.version = get_version_from_repo(git=Git(True)) ref = github_event.get("ref", "refs/heads/master") if ref and ref.startswith("refs/heads/"): ref = ref[11:] @@ -250,7 +255,7 @@ def __init__( self.sha = os.getenv( "GITHUB_SHA", "0000000000000000000000000000000000000000" ) - self.number = 0 + self.number = f"{self.version.major}.{self.version.minor}.{self.version.patch}" self.docker_image_tag = str(self.number) + "-" + str(self.sha) self.labels = set() repo_prefix = f"{GITHUB_SERVER_URL}/{GITHUB_REPOSITORY}" From c1c812775ca3b74f06845c00ba470eac8e390158 Mon Sep 17 00:00:00 2001 From: Ilya Golshtein Date: Tue, 10 Sep 2024 21:22:17 +0000 Subject: [PATCH 056/123] 23.8_kerberized_hadoop_dockerfile_rpmbone: initial --- docker/test/integration/kerberized_hadoop/Dockerfile | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/docker/test/integration/kerberized_hadoop/Dockerfile b/docker/test/integration/kerberized_hadoop/Dockerfile index 592c3e36ef7f..1146a2a2ae50 100644 --- a/docker/test/integration/kerberized_hadoop/Dockerfile +++ b/docker/test/integration/kerberized_hadoop/Dockerfile @@ -6,11 +6,11 @@ FROM sequenceiq/hadoop-docker:2.7.0 RUN sed -i s/xMDkzMDE0MDExNVow/0MDkzMDE4MTQwM1ow/ /etc/pki/tls/certs/ca-bundle.crt -RUN curl -o krb5-libs-1.10.3-65.el6.x86_64.rpm ftp://ftp.pbone.net/mirror/vault.centos.org/6.10/os/x86_64/Packages/krb5-libs-1.10.3-65.el6.x86_64.rpm && \ - curl -o krb5-workstation-1.10.3-65.el6.x86_64.rpm ftp://ftp.pbone.net/mirror/vault.centos.org/6.9/os/x86_64/Packages/krb5-workstation-1.10.3-65.el6.x86_64.rpm && \ - curl -o libkadm5-1.10.3-65.el6.x86_64.rpm ftp://ftp.pbone.net/mirror/vault.centos.org/6.10/os/x86_64/Packages/libkadm5-1.10.3-65.el6.x86_64.rpm && \ - curl -o libss-1.41.12-24.el6.x86_64.rpm ftp://ftp.pbone.net/mirror/vault.centos.org/6.9/cr/x86_64/Packages/libss-1.41.12-24.el6.x86_64.rpm && \ - curl -o libcom_err-1.41.12-24.el6.x86_64.rpm ftp://ftp.pbone.net/mirror/vault.centos.org/6.9/cr/x86_64/Packages/libcom_err-1.41.12-24.el6.x86_64.rpm && \ +RUN curl -o krb5-libs-1.10.3-65.el6.x86_64.rpm ftp://ftp.icm.edu.pl/vol/rzm6/linux-slc/slc610/x86_64/Packages/krb5-libs-1.10.3-65.el6.x86_64.rpm && \ + curl -o krb5-workstation-1.10.3-65.el6.x86_64.rpm ftp://ftp.icm.edu.pl/vol/rzm6/linux-slc/slc610/x86_64/Packages/krb5-workstation-1.10.3-65.el6.x86_64.rpm && \ + curl -o libkadm5-1.10.3-65.el6.x86_64.rpm ftp://ftp.icm.edu.pl/vol/rzm6/linux-slc/slc610/x86_64/Packages/libkadm5-1.10.3-65.el6.x86_64.rpm && \ + curl -o libss-1.41.12-24.el6.x86_64.rpm http://ftp.pasteur.fr/mirrors/centos-vault/6.9/cr/x86_64/Packages/libss-1.41.12-24.el6.x86_64.rpm && \ + curl -o libcom_err-1.41.12-24.el6.x86_64.rpm http://ftp.pasteur.fr/mirrors/centos-vault/6.9/cr/x86_64/Packages/libcom_err-1.41.12-24.el6.x86_64.rpm && \ rpm -Uvh libkadm5-1.10.3-65.el6.x86_64.rpm libss-1.41.12-24.el6.x86_64.rpm krb5-libs-1.10.3-65.el6.x86_64.rpm krb5-workstation-1.10.3-65.el6.x86_64.rpm libcom_err-1.41.12-24.el6.x86_64.rpm && \ rm -fr *.rpm From 4277e250fb5a320ac73c533c22f447407e748c00 Mon Sep 17 00:00:00 2001 From: MyroTk <44327070+MyroTk@users.noreply.github.com> Date: Wed, 11 Sep 2024 11:06:10 -0700 Subject: [PATCH 057/123] Create reusable_sign.yml --- .github/workflows/reusable_sign.yml | 164 ++++++++++++++++++++++++++++ 1 file changed, 164 insertions(+) create mode 100644 .github/workflows/reusable_sign.yml diff --git a/.github/workflows/reusable_sign.yml b/.github/workflows/reusable_sign.yml new file mode 100644 index 000000000000..2bd8ae430fa5 --- /dev/null +++ b/.github/workflows/reusable_sign.yml @@ -0,0 +1,164 @@ +### For the pure soul wishes to move it to another place +# https://github.com/orgs/community/discussions/9050 + +name: Testing workflow +'on': + workflow_call: + inputs: + test_name: + description: the value of test type from tests/ci/ci_config.py, ends up as $CHECK_NAME ENV + required: true + type: string + runner_type: + description: the label of runner to use + required: true + type: string + run_command: + description: the command to launch the check + default: "" + required: false + type: string + checkout_depth: + description: the value of the git shallow checkout + required: false + type: number + default: 1 + submodules: + description: if the submodules should be checked out + required: false + type: boolean + default: false + additional_envs: + description: additional ENV variables to setup the job + type: string + data: + description: ci data + type: string + required: true + working-directory: + description: sets custom working directory + type: string + default: "$GITHUB_WORKSPACE/tests/ci" + secrets: + secret_envs: + description: if given, it's passed to the environments + required: false + AWS_SECRET_ACCESS_KEY: + description: the access key to the aws param store. + required: true + AWS_ACCESS_KEY_ID: + description: the access key id to the aws param store. + required: true + GPG_BINARY_SIGNING_KEY: + description: gpg signing key for packages. + required: true + GPG_BINARY_SIGNING_PASSPHRASE: + description: gpg signing key passphrase. + required: true + +env: + # Force the stdout and stderr streams to be unbuffered + PYTHONUNBUFFERED: 1 + CHECK_NAME: ${{inputs.test_name}} + AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }} + AWS_DEFAULT_REGION: ${{ secrets.AWS_DEFAULT_REGION }} + AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }} + +jobs: + runner_labels_setup: + name: Compute proper runner labels for the rest of the jobs + runs-on: ubuntu-latest + outputs: + runner_labels: ${{ steps.setVariables.outputs.runner_labels }} + steps: + - id: setVariables + name: Prepare runner_labels variables for the later steps + run: | + + # Prepend self-hosted + input="self-hosted, ${input}" + + # Remove all whitespace + input="$(echo ${input} | tr -d [:space:])" + # Make something like a JSON array from comma-separated list + input="[ '${input//\,/\'\, \'}' ]" + + echo "runner_labels=$input" >> ${GITHUB_OUTPUT} + env: + input: ${{ inputs.runner_type }} + + Test: + needs: [runner_labels_setup] + runs-on: ${{ fromJson(needs.runner_labels_setup.outputs.runner_labels) }} + name: ${{inputs.test_name}}${{ fromJson(inputs.data).jobs_data.jobs_params[inputs.test_name].num_batches > 1 && format('-{0}',matrix.batch) || '' }} + env: + GITHUB_JOB_OVERRIDDEN: ${{inputs.test_name}}${{ fromJson(inputs.data).jobs_data.jobs_params[inputs.test_name].num_batches > 1 && format('-{0}',matrix.batch) || '' }} + strategy: + fail-fast: false # we always wait for entire matrix + matrix: + batch: ${{ fromJson(inputs.data).jobs_data.jobs_params[inputs.test_name].batches }} + steps: + - name: Check out repository code + uses: Altinity/checkout@19599efdf36c4f3f30eb55d5bb388896faea69f6 + with: + clear-repository: true + ref: ${{ fromJson(inputs.data).git_ref }} + submodules: ${{inputs.submodules}} + fetch-depth: ${{inputs.checkout_depth}} + filter: tree:0 + - name: Set build envs + run: | + cat >> "$GITHUB_ENV" << 'EOF' + CHECK_NAME=${{ inputs.test_name }} + ${{inputs.additional_envs}} + ${{secrets.secret_envs}} + DOCKER_TAG< 1 }} + run: | + cat >> "$GITHUB_ENV" << 'EOF' + RUN_BY_HASH_NUM=${{matrix.batch}} + RUN_BY_HASH_TOTAL=${{ fromJson(inputs.data).jobs_data.jobs_params[inputs.test_name].num_batches }} + EOF + - name: Pre run + run: | + python3 "$GITHUB_WORKSPACE/tests/ci/ci.py" --infile ${{ toJson(inputs.data) }} --pre --job-name '${{inputs.test_name}}' + - name: Sign release + env: + GPG_BINARY_SIGNING_KEY: ${{ secrets.GPG_BINARY_SIGNING_KEY }} + GPG_BINARY_SIGNING_PASSPHRASE: ${{ secrets.GPG_BINARY_SIGNING_PASSPHRASE }} + run: | + cd "${{ inputs.working-directory }}" + python3 "$GITHUB_WORKSPACE/tests/ci/ci.py" \ + --infile ${{ toJson(inputs.data) }} \ + --job-name '${{inputs.test_name}}' \ + --run \ + --force \ + --run-command '''python3 sign_release.py''' + - name: Post run + if: ${{ !cancelled() }} + run: | + python3 "$GITHUB_WORKSPACE/tests/ci/ci.py" --infile ${{ toJson(inputs.data) }} --post --job-name '${{inputs.test_name}}' + - name: Mark as done + if: ${{ !cancelled() }} + run: | + python3 "$GITHUB_WORKSPACE/tests/ci/ci.py" --infile ${{ toJson(inputs.data) }} --mark-success --job-name '${{inputs.test_name}}' --batch ${{matrix.batch}} + - name: Upload signed hashes + uses: actions/upload-artifact@v4 + with: + name: ${{inputs.test_name}} signed-hashes + path: ${{ env.TEMP_PATH }}/*.gpg + - name: Clean + if: always() + uses: ./.github/actions/clean From 3f0388c9b898c1925637bfffcdd8cae9f43d58ed Mon Sep 17 00:00:00 2001 From: MyroTk <44327070+MyroTk@users.noreply.github.com> Date: Wed, 11 Sep 2024 11:13:08 -0700 Subject: [PATCH 058/123] Update ci_config.py --- tests/ci/ci_config.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/tests/ci/ci_config.py b/tests/ci/ci_config.py index a60932ec9cf1..ac99e0efc85a 100644 --- a/tests/ci/ci_config.py +++ b/tests/ci/ci_config.py @@ -316,7 +316,8 @@ def validate(self) -> None: "SQLancer (debug)": TestConfig("package_debug"), "Sqllogic test (release)": TestConfig("package_release"), "SQLTest": TestConfig("package_release"), - "Sign release (actions)": TestConfig("package_release"), + "Sign release": TestConfig("package_release"), + "Sign release": TestConfig("package_aarch64"), }, ) CI_CONFIG.validate() From 22bd207c287bb359e229e6938c007d64cb896807 Mon Sep 17 00:00:00 2001 From: MyroTk <44327070+MyroTk@users.noreply.github.com> Date: Wed, 11 Sep 2024 11:13:27 -0700 Subject: [PATCH 059/123] Update ci_config.py --- tests/ci/ci_config.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/ci/ci_config.py b/tests/ci/ci_config.py index ac99e0efc85a..e5c7dec90690 100644 --- a/tests/ci/ci_config.py +++ b/tests/ci/ci_config.py @@ -317,7 +317,7 @@ def validate(self) -> None: "Sqllogic test (release)": TestConfig("package_release"), "SQLTest": TestConfig("package_release"), "Sign release": TestConfig("package_release"), - "Sign release": TestConfig("package_aarch64"), + "Sign aarch64": TestConfig("package_aarch64"), }, ) CI_CONFIG.validate() From 936b369e7d881828be77b901957d2a14281a7641 Mon Sep 17 00:00:00 2001 From: MyroTk <44327070+MyroTk@users.noreply.github.com> Date: Wed, 11 Sep 2024 11:18:14 -0700 Subject: [PATCH 060/123] Update release_branches.yml --- .github/workflows/release_branches.yml | 46 ++++++++++++++++++++++++-- 1 file changed, 44 insertions(+), 2 deletions(-) diff --git a/.github/workflows/release_branches.yml b/.github/workflows/release_branches.yml index 553c51d1269c..e35e168a7c35 100644 --- a/.github/workflows/release_branches.yml +++ b/.github/workflows/release_branches.yml @@ -657,7 +657,7 @@ jobs: SignRelease: needs: [BuilderDebRelease] - runs-on: [self-hosted, altinity-on-demand, altinity-setup-reporter, altinity-type-cax11, altinity-image-x86-system-ubuntu-22.04] + runs-on: [self-hosted, altinity-on-demand, altinity-setup-reporter, altinity-type-cax11, altinity-image-arm-system-ubuntu-22.04] timeout-minutes: 180 steps: - name: Set envs @@ -674,7 +674,7 @@ jobs: - name: Download json reports uses: actions/download-artifact@v4 with: - pattern: build_report_package_* + pattern: build_report_package_release path: ${{ env.REPORTS_PATH }} - name: Sign release env: @@ -696,6 +696,47 @@ jobs: docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: sudo rm -fr "$TEMP_PATH" + SignAarch64: + needs: [BuilderDebAarch64] + runs-on: [self-hosted, altinity-on-demand, altinity-setup-reporter, altinity-type-cax11, altinity-image-arm-system-ubuntu-22.04] + timeout-minutes: 180 + steps: + - name: Set envs + run: | + cat >> "$GITHUB_ENV" << 'EOF' + CHECK_NAME=SignAarch64 + TEMP_PATH=${{runner.temp}}/signed + REPORTS_PATH=${{runner.temp}}/reports_dir + EOF + - name: Clear repository + run: | + sudo rm -fr "$GITHUB_WORKSPACE" && mkdir "$GITHUB_WORKSPACE" + - name: Check out repository code + uses: actions/checkout@v4 + - name: Download json reports + uses: actions/download-artifact@v4 + with: + pattern: build_report_package_aarch64 + path: ${{ env.REPORTS_PATH }} + - name: Sign aarch64 + env: + GPG_BINARY_SIGNING_KEY: ${{ secrets.GPG_BINARY_SIGNING_KEY }} + GPG_BINARY_SIGNING_PASSPHRASE: ${{ secrets.GPG_BINARY_SIGNING_PASSPHRASE }} + REPORTS_PATH: ${{ env.REPORTS_PATH }} + run: | + cd "$GITHUB_WORKSPACE/tests/ci" + python3 sign_release.py + - name: Upload signed hashes + uses: actions/upload-artifact@v4 + with: + name: signed-hashes + path: ${{ env.TEMP_PATH }}/*.gpg + - name: Cleanup + if: always() + run: | + docker ps --quiet | xargs --no-run-if-empty docker kill ||: + docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: + sudo rm -fr "$TEMP_PATH" ########################################################################################### ################################ FINISH CHECK ############################################# ########################################################################################### @@ -732,6 +773,7 @@ jobs: - RegressionTestsRelease - RegressionTestsAarch64 - SignRelease + - SignAarch64 runs-on: [self-hosted, altinity-on-demand, altinity-type-cpx41, altinity-in-ash, altinity-image-x86-system-ubuntu-22.04] steps: - name: Check out repository code From cce73160d2d534dd68c99fa1b8d4357bf51bb329 Mon Sep 17 00:00:00 2001 From: MyroTk <44327070+MyroTk@users.noreply.github.com> Date: Wed, 11 Sep 2024 11:19:25 -0700 Subject: [PATCH 061/123] Update sign_release.py --- tests/ci/sign_release.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/ci/sign_release.py b/tests/ci/sign_release.py index 872966a578a5..53bc236b141f 100644 --- a/tests/ci/sign_release.py +++ b/tests/ci/sign_release.py @@ -12,7 +12,7 @@ GPG_BINARY_SIGNING_KEY = os.getenv("GPG_BINARY_SIGNING_KEY") GPG_BINARY_SIGNING_PASSPHRASE = os.getenv("GPG_BINARY_SIGNING_PASSPHRASE") -CHECK_NAME = "Sign release (actions)" +CHECK_NAME = os.getenv("CHECK_NAME", "Sign release") def hash_file(file_path): BLOCK_SIZE = 65536 # The size of each read from the file From 69490baebf6df7da3186087a0bfb57e785822a1f Mon Sep 17 00:00:00 2001 From: MyroTk <44327070+MyroTk@users.noreply.github.com> Date: Wed, 11 Sep 2024 11:58:50 -0700 Subject: [PATCH 062/123] Delete .github/workflows/reusable_sign.yml --- .github/workflows/reusable_sign.yml | 164 ---------------------------- 1 file changed, 164 deletions(-) delete mode 100644 .github/workflows/reusable_sign.yml diff --git a/.github/workflows/reusable_sign.yml b/.github/workflows/reusable_sign.yml deleted file mode 100644 index 2bd8ae430fa5..000000000000 --- a/.github/workflows/reusable_sign.yml +++ /dev/null @@ -1,164 +0,0 @@ -### For the pure soul wishes to move it to another place -# https://github.com/orgs/community/discussions/9050 - -name: Testing workflow -'on': - workflow_call: - inputs: - test_name: - description: the value of test type from tests/ci/ci_config.py, ends up as $CHECK_NAME ENV - required: true - type: string - runner_type: - description: the label of runner to use - required: true - type: string - run_command: - description: the command to launch the check - default: "" - required: false - type: string - checkout_depth: - description: the value of the git shallow checkout - required: false - type: number - default: 1 - submodules: - description: if the submodules should be checked out - required: false - type: boolean - default: false - additional_envs: - description: additional ENV variables to setup the job - type: string - data: - description: ci data - type: string - required: true - working-directory: - description: sets custom working directory - type: string - default: "$GITHUB_WORKSPACE/tests/ci" - secrets: - secret_envs: - description: if given, it's passed to the environments - required: false - AWS_SECRET_ACCESS_KEY: - description: the access key to the aws param store. - required: true - AWS_ACCESS_KEY_ID: - description: the access key id to the aws param store. - required: true - GPG_BINARY_SIGNING_KEY: - description: gpg signing key for packages. - required: true - GPG_BINARY_SIGNING_PASSPHRASE: - description: gpg signing key passphrase. - required: true - -env: - # Force the stdout and stderr streams to be unbuffered - PYTHONUNBUFFERED: 1 - CHECK_NAME: ${{inputs.test_name}} - AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }} - AWS_DEFAULT_REGION: ${{ secrets.AWS_DEFAULT_REGION }} - AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }} - -jobs: - runner_labels_setup: - name: Compute proper runner labels for the rest of the jobs - runs-on: ubuntu-latest - outputs: - runner_labels: ${{ steps.setVariables.outputs.runner_labels }} - steps: - - id: setVariables - name: Prepare runner_labels variables for the later steps - run: | - - # Prepend self-hosted - input="self-hosted, ${input}" - - # Remove all whitespace - input="$(echo ${input} | tr -d [:space:])" - # Make something like a JSON array from comma-separated list - input="[ '${input//\,/\'\, \'}' ]" - - echo "runner_labels=$input" >> ${GITHUB_OUTPUT} - env: - input: ${{ inputs.runner_type }} - - Test: - needs: [runner_labels_setup] - runs-on: ${{ fromJson(needs.runner_labels_setup.outputs.runner_labels) }} - name: ${{inputs.test_name}}${{ fromJson(inputs.data).jobs_data.jobs_params[inputs.test_name].num_batches > 1 && format('-{0}',matrix.batch) || '' }} - env: - GITHUB_JOB_OVERRIDDEN: ${{inputs.test_name}}${{ fromJson(inputs.data).jobs_data.jobs_params[inputs.test_name].num_batches > 1 && format('-{0}',matrix.batch) || '' }} - strategy: - fail-fast: false # we always wait for entire matrix - matrix: - batch: ${{ fromJson(inputs.data).jobs_data.jobs_params[inputs.test_name].batches }} - steps: - - name: Check out repository code - uses: Altinity/checkout@19599efdf36c4f3f30eb55d5bb388896faea69f6 - with: - clear-repository: true - ref: ${{ fromJson(inputs.data).git_ref }} - submodules: ${{inputs.submodules}} - fetch-depth: ${{inputs.checkout_depth}} - filter: tree:0 - - name: Set build envs - run: | - cat >> "$GITHUB_ENV" << 'EOF' - CHECK_NAME=${{ inputs.test_name }} - ${{inputs.additional_envs}} - ${{secrets.secret_envs}} - DOCKER_TAG< 1 }} - run: | - cat >> "$GITHUB_ENV" << 'EOF' - RUN_BY_HASH_NUM=${{matrix.batch}} - RUN_BY_HASH_TOTAL=${{ fromJson(inputs.data).jobs_data.jobs_params[inputs.test_name].num_batches }} - EOF - - name: Pre run - run: | - python3 "$GITHUB_WORKSPACE/tests/ci/ci.py" --infile ${{ toJson(inputs.data) }} --pre --job-name '${{inputs.test_name}}' - - name: Sign release - env: - GPG_BINARY_SIGNING_KEY: ${{ secrets.GPG_BINARY_SIGNING_KEY }} - GPG_BINARY_SIGNING_PASSPHRASE: ${{ secrets.GPG_BINARY_SIGNING_PASSPHRASE }} - run: | - cd "${{ inputs.working-directory }}" - python3 "$GITHUB_WORKSPACE/tests/ci/ci.py" \ - --infile ${{ toJson(inputs.data) }} \ - --job-name '${{inputs.test_name}}' \ - --run \ - --force \ - --run-command '''python3 sign_release.py''' - - name: Post run - if: ${{ !cancelled() }} - run: | - python3 "$GITHUB_WORKSPACE/tests/ci/ci.py" --infile ${{ toJson(inputs.data) }} --post --job-name '${{inputs.test_name}}' - - name: Mark as done - if: ${{ !cancelled() }} - run: | - python3 "$GITHUB_WORKSPACE/tests/ci/ci.py" --infile ${{ toJson(inputs.data) }} --mark-success --job-name '${{inputs.test_name}}' --batch ${{matrix.batch}} - - name: Upload signed hashes - uses: actions/upload-artifact@v4 - with: - name: ${{inputs.test_name}} signed-hashes - path: ${{ env.TEMP_PATH }}/*.gpg - - name: Clean - if: always() - uses: ./.github/actions/clean From 5587afc19f9f24e7acc0263ca0e957282cfef692 Mon Sep 17 00:00:00 2001 From: Vitaly Baranov Date: Thu, 22 Aug 2024 13:45:13 +0200 Subject: [PATCH 063/123] Fix test for role expiration in RoleCache. --- tests/integration/test_role/test.py | 81 +++++++++-------------------- 1 file changed, 26 insertions(+), 55 deletions(-) diff --git a/tests/integration/test_role/test.py b/tests/integration/test_role/test.py index 0e1bbea9cea0..6f2929c656b5 100644 --- a/tests/integration/test_role/test.py +++ b/tests/integration/test_role/test.py @@ -415,72 +415,43 @@ def test_function_current_roles(): ) -def test_role_expiration(): - instance.query("CREATE USER ure") +@pytest.mark.parametrize("with_extra_role", [False, True]) +def test_role_expiration(with_extra_role): instance.query("CREATE ROLE rre") - instance.query("GRANT rre TO ure") + instance.query("CREATE USER ure DEFAULT ROLE rre") - instance.query("CREATE TABLE IF NOT EXISTS tre (id Int) Engine=Log") - instance.query("INSERT INTO tre VALUES (0)") + instance.query("CREATE TABLE table1 (id Int) Engine=Log") + instance.query("CREATE TABLE table2 (id Int) Engine=Log") + instance.query("INSERT INTO table1 VALUES (1)") + instance.query("INSERT INTO table2 VALUES (2)") + instance.query("GRANT SELECT ON table1 TO rre") + + assert instance.query("SELECT * FROM table1", user="ure") == "1\n" assert "Not enough privileges" in instance.query_and_get_error( - "SELECT * FROM tre", user="ure" + "SELECT * FROM table2", user="ure" ) - instance.query("GRANT SELECT ON tre TO rre") - - assert instance.query("SELECT * FROM tre", user="ure") == "0\n" - # access_control_improvements/role_cache_expiration_time_seconds value is 2 for the test # so we wait >2 seconds until the role is expired time.sleep(5) - instance.query("CREATE TABLE IF NOT EXISTS tre1 (id Int) Engine=Log") - instance.query("INSERT INTO tre1 VALUES (0)") - instance.query("GRANT SELECT ON tre1 TO rre") + if with_extra_role: + # Expiration of role "rre" from the role cache can be caused by another role being used. + instance.query("CREATE ROLE extra_role") + instance.query("CREATE USER extra_user DEFAULT ROLE extra_role") + instance.query("GRANT SELECT ON table1 TO extra_role") + assert instance.query("SELECT * FROM table1", user="extra_user") == "1\n" - assert instance.query("SELECT * from tre1", user="ure") == "0\n" + instance.query("GRANT SELECT ON table2 TO rre") + assert instance.query("SELECT * FROM table1", user="ure") == "1\n" + assert instance.query("SELECT * FROM table2", user="ure") == "2\n" - instance.query("DROP USER ure") instance.query("DROP ROLE rre") - instance.query("DROP TABLE tre") - instance.query("DROP TABLE tre1") - - -def test_two_roles_expiration(): - instance.query("CREATE USER ure") - instance.query("CREATE ROLE rre") - instance.query("GRANT rre TO ure") - - instance.query("CREATE ROLE rre_second") - - instance.query("CREATE TABLE IF NOT EXISTS tre (id Int) Engine=Log") - instance.query("INSERT INTO tre VALUES (0)") - - assert "Not enough privileges" in instance.query_and_get_error( - "SELECT * FROM tre", user="ure" - ) - - instance.query("GRANT SELECT ON tre TO rre") - - assert instance.query("SELECT * FROM tre", user="ure") == "0\n" - - # access_control_improvements/role_cache_expiration_time_seconds value is 2 for the test - # so we wait >2 seconds until the roles are expired - time.sleep(5) - - instance.query( - "GRANT SELECT ON tre1 TO rre_second" - ) # we expect that both rre and rre_second are gone from cache upon this operation - - instance.query("CREATE TABLE IF NOT EXISTS tre1 (id Int) Engine=Log") - instance.query("INSERT INTO tre1 VALUES (0)") - instance.query("GRANT SELECT ON tre1 TO rre") - - assert instance.query("SELECT * from tre1", user="ure") == "0\n" - instance.query("DROP USER ure") - instance.query("DROP ROLE rre") - instance.query("DROP ROLE rre_second") - instance.query("DROP TABLE tre") - instance.query("DROP TABLE tre1") + instance.query("DROP TABLE table1") + instance.query("DROP TABLE table2") + + if with_extra_role: + instance.query("DROP ROLE extra_role") + instance.query("DROP USER extra_user") From ad3ff98109d713f6e513f923f6bf1f70827cf76e Mon Sep 17 00:00:00 2001 From: Vitaly Baranov Date: Sun, 4 Aug 2024 13:30:41 +0200 Subject: [PATCH 064/123] Add one more test. --- tests/integration/test_role/test.py | 173 ++++++++++++++++++++++++++++ 1 file changed, 173 insertions(+) diff --git a/tests/integration/test_role/test.py b/tests/integration/test_role/test.py index 6f2929c656b5..85618b905128 100644 --- a/tests/integration/test_role/test.py +++ b/tests/integration/test_role/test.py @@ -1,5 +1,6 @@ import time import pytest +import random from helpers.client import QueryRuntimeException from helpers.cluster import ClickHouseCluster from helpers.test_tools import TSV @@ -455,3 +456,175 @@ def test_role_expiration(with_extra_role): if with_extra_role: instance.query("DROP ROLE extra_role") instance.query("DROP USER extra_user") + + +def test_roles_cache(): + # This test takes 20 seconds. + test_time = 20 + + # Three users A, B, C. + users = ["A", "B", "C"] + instance.query("CREATE USER " + ", ".join(users)) + + # Table "tbl" has 10 columns. Each of the users has access to a different set of columns. + num_columns = 10 + columns = [f"x{i}" for i in range(1, num_columns + 1)] + columns_with_types = [column + " Int64" for column in columns] + columns_with_types_comma_separated = ", ".join(columns_with_types) + values = list(range(1, num_columns + 1)) + values_comma_separated = ", ".join([str(value) for value in values]) + instance.query( + f"CREATE TABLE tbl ({columns_with_types_comma_separated}) ENGINE=MergeTree ORDER BY tuple()" + ) + instance.query(f"INSERT INTO tbl VALUES ({values_comma_separated})") + columns_to_values = dict([(f"x{i}", i) for i in range(1, num_columns + 1)]) + + # In this test we create and modify roles multiple times along with updating the following variables. + # Then we check that each of the users has access to the expected set of columns. + roles = [] + users_to_roles = dict([(user, []) for user in users]) + roles_to_columns = {} + + # Checks that each of the users can access the expected set of columns and can't access other columns. + def check(): + for user in random.sample(users, len(users)): + expected_roles = users_to_roles[user] + expected_columns = list( + set(sum([roles_to_columns[role] for role in expected_roles], [])) + ) + expected_result = sorted( + [columns_to_values[column] for column in expected_columns] + ) + query = " UNION ALL ".join( + [ + f"SELECT * FROM viewIfPermitted(SELECT {column} AS c FROM tbl ELSE null('c Int64'))" + for column in columns + ] + ) + result = instance.query(query, user=user).splitlines() + result = sorted([int(value) for value in result]) + ok = result == expected_result + if not ok: + print(f"Show grants for {user}:") + print( + instance.query( + "SHOW GRANTS FOR " + ", ".join([user] + expected_roles) + ) + ) + print(f"Expected result: {expected_result}") + print(f"Got unexpected result: {result}") + assert ok + + # Grants one of our roles a permission to access one of the columns. + def grant_column(): + columns_used_in_roles = sum(roles_to_columns.values(), []) + columns_to_choose = [ + column for column in columns if column not in columns_used_in_roles + ] + if not columns_to_choose or not roles: + return False + column = random.choice(columns_to_choose) + role = random.choice(roles) + instance.query(f"GRANT SELECT({column}) ON tbl TO {role}") + roles_to_columns[role].append(column) + return True + + # Revokes a permission to access one of the granted column from all our roles. + def revoke_column(): + columns_used_in_roles = sum(roles_to_columns.values(), []) + columns_to_choose = list(set(columns_used_in_roles)) + if not columns_to_choose or not roles: + return False + column = random.choice(columns_to_choose) + roles_str = ", ".join(roles) + instance.query(f"REVOKE SELECT({column}) ON tbl FROM {roles_str}") + for role in roles_to_columns: + if column in roles_to_columns[role]: + roles_to_columns[role].remove(column) + return True + + # Creates a role and grants it to one of the users. + def create_role(): + for role in ["R1", "R2", "R3"]: + if role not in roles: + instance.query(f"CREATE ROLE {role}") + roles.append(role) + if role not in roles_to_columns: + roles_to_columns[role] = [] + if "R1" not in users_to_roles["A"]: + instance.query("GRANT R1 TO A") + users_to_roles["A"].append("R1") + elif "R2" not in users_to_roles["B"]: + instance.query("GRANT R2 TO B") + users_to_roles["B"].append("R2") + elif "R3" not in users_to_roles["B"]: + instance.query("GRANT R3 TO R2") + users_to_roles["B"].append("R3") + elif "R3" not in users_to_roles["C"]: + instance.query("GRANT R3 TO C") + users_to_roles["C"].append("R3") + else: + return False + return True + + # Drops one of our roles. + def drop_role(): + if not roles: + return False + role = random.choice(roles) + instance.query(f"DROP ROLE {role}") + roles.remove(role) + for u in users_to_roles: + if role in users_to_roles[u]: + users_to_roles[u].remove(role) + del roles_to_columns[role] + if (role == "R2") and ("R3" in users_to_roles["B"]): + users_to_roles["B"].remove("R3") + return True + + # Modifies some grants or roles randomly. + def modify(): + while True: + rnd = random.random() + if rnd < 0.4: + if grant_column(): + break + elif rnd < 0.5: + if revoke_column(): + break + elif rnd < 0.9: + if create_role(): + break + else: + if drop_role(): + break + + def maybe_modify(): + if random.random() < 0.9: + modify() + modify() + + # Sleeping is necessary in this test because the role cache in ClickHouse has expiration timeout. + def maybe_sleep(): + if random.random() < 0.1: + # "role_cache_expiration_time_seconds" is set to 2 seconds in the test configuration. + # We need a sleep longer than that in this test sometimes. + seconds = random.random() * 5 + print(f"Sleeping {seconds} seconds") + time.sleep(seconds) + + # Main part of the test. + start_time = time.time() + end_time = start_time + test_time + + while time.time() < end_time: + check() + maybe_sleep() + maybe_modify() + maybe_sleep() + + check() + + instance.query("DROP USER " + ", ".join(users)) + instance.query("DROP ROLE " + ", ".join(roles)) + instance.query("DROP TABLE tbl") From 8b42292fee5f1bc9137484c0bcc1554ec7f61bf6 Mon Sep 17 00:00:00 2001 From: Vitaly Baranov Date: Sun, 4 Aug 2024 13:33:50 +0200 Subject: [PATCH 065/123] Fix expiration in RoleCache. --- src/Access/RoleCache.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/Access/RoleCache.cpp b/src/Access/RoleCache.cpp index 2d94df2eea5a..cc1f1520b673 100644 --- a/src/Access/RoleCache.cpp +++ b/src/Access/RoleCache.cpp @@ -120,7 +120,7 @@ void RoleCache::collectEnabledRoles(EnabledRoles & enabled_roles, SubscriptionsO SubscriptionsOnRoles new_subscriptions_on_roles; new_subscriptions_on_roles.reserve(subscriptions_on_roles.size()); - auto get_role_function = [this, &subscriptions_on_roles](const UUID & id) TSA_NO_THREAD_SAFETY_ANALYSIS { return getRole(id, subscriptions_on_roles); }; + auto get_role_function = [this, &new_subscriptions_on_roles](const UUID & id) TSA_NO_THREAD_SAFETY_ANALYSIS { return getRole(id, new_subscriptions_on_roles); }; for (const auto & current_role : enabled_roles.params.current_roles) collectRoles(*new_info, skip_ids, get_role_function, current_role, true, false); From e2161db2735cac0d0e40c9d1f67be14d5e35f9d9 Mon Sep 17 00:00:00 2001 From: Ilya Golshtein Date: Wed, 24 May 2023 21:59:46 +0000 Subject: [PATCH 066/123] merge_row_policy: initial --- src/Interpreters/InterpreterSelectQuery.cpp | 14 ++++++++- src/Storages/StorageMerge.cpp | 33 ++++++++++++++++++++- 2 files changed, 45 insertions(+), 2 deletions(-) diff --git a/src/Interpreters/InterpreterSelectQuery.cpp b/src/Interpreters/InterpreterSelectQuery.cpp index a268faa5cd6f..21b6f737436e 100644 --- a/src/Interpreters/InterpreterSelectQuery.cpp +++ b/src/Interpreters/InterpreterSelectQuery.cpp @@ -93,6 +93,7 @@ #include #include +#include namespace ProfileEvents { @@ -546,7 +547,18 @@ InterpreterSelectQuery::InterpreterSelectQuery( std::shared_ptr table_join = joined_tables.makeTableJoin(query); if (storage) + { + + LOG_TRACE(&Poco::Logger::get("InterpretSelectQuery ctor"), "table name: {}", table_id.getTableName()); + row_policy_filter = context->getRowPolicyFilter(table_id.getDatabaseName(), table_id.getTableName(), RowPolicyFilterType::SELECT_FILTER); + } + else + { + LOG_TRACE(&Poco::Logger::get("InterpretSelectQuery ctor"), "no storage"); + } + + StorageView * view = nullptr; if (storage) @@ -832,7 +844,7 @@ InterpreterSelectQuery::InterpreterSelectQuery( if (query.prewhere() && !query.where()) analysis_result.prewhere_info->need_filter = true; - if (table_id && got_storage_from_query && !joined_tables.isLeftTableFunction()) + if (table_id && got_storage_from_query /* && !joined_tables.isLeftTableFunction() */) { /// The current user should have the SELECT privilege. If this table_id is for a table /// function we don't check access rights here because in this case they have been already diff --git a/src/Storages/StorageMerge.cpp b/src/Storages/StorageMerge.cpp index 272f35303bd1..86a9e8c743f2 100644 --- a/src/Storages/StorageMerge.cpp +++ b/src/Storages/StorageMerge.cpp @@ -43,6 +43,8 @@ #include #include +#include + namespace { @@ -515,6 +517,8 @@ void ReadFromMerge::initializePipeline(QueryPipelineBuilder & pipeline, const Bu } } + LOG_TRACE(&Poco::Logger::get("ReadFromMerge::initializePipeline"), "table name: {}", storage->getStorageID().getTableName()); + auto source_pipeline = createSources( nested_storage_snaphsot, modified_query_info, @@ -574,6 +578,8 @@ SelectQueryInfo ReadFromMerge::getModifiedQueryInfo(const SelectQueryInfo & quer modified_query_info.table_expression = replacement_table_expression; modified_query_info.planner_context->getOrCreateTableExpressionData(replacement_table_expression); + + auto get_column_options = GetColumnsOptions(GetColumnsOptions::All).withExtendedObjects().withVirtuals(); if (storage_snapshot->storage.supportsSubcolumns()) get_column_options.withSubcolumns(); @@ -594,6 +600,10 @@ SelectQueryInfo ReadFromMerge::getModifiedQueryInfo(const SelectQueryInfo & quer } modified_query_info.query = queryNodeToSelectQuery(modified_query_info.query_tree); + TreeRewriterResult new_analyzer_res = *modified_query_info.syntax_analyzer_result; + new_analyzer_res.has_explicit_columns = false; + + modified_query_info.syntax_analyzer_result = std::make_shared(std::move(new_analyzer_res)); } else { @@ -656,8 +666,9 @@ QueryPipelineBuilderPtr ReadFromMerge::createSources( QueryPlan & plan = child_plans.emplace_back(); StorageView * view = dynamic_cast(storage.get()); - if (!view || allow_experimental_analyzer) + if (/* !view || */ allow_experimental_analyzer) { + LOG_TRACE(&Poco::Logger::get("ReadFromMerge::createSources"), "direct storage->read"); storage->read(plan, real_column_names, storage_snapshot, @@ -667,6 +678,24 @@ QueryPipelineBuilderPtr ReadFromMerge::createSources( max_block_size, UInt32(streams_num)); } + else if (!view) + { + /// For view storage, we need to rewrite the `modified_query_info.view_query` to optimize read. + /// The most intuitive way is to use InterpreterSelectQuery. + + /// Intercept the settings + modified_context->setSetting("max_threads", streams_num); + modified_context->setSetting("max_streams_to_max_threads_ratio", 1); + modified_context->setSetting("max_block_size", max_block_size); + + LOG_TRACE(&Poco::Logger::get("ReadFromMerge::createSources"), "creating InterpreterSelectQuery 1.0"); + InterpreterSelectQuery interpreter(modified_query_info.query, + modified_context, + storage, + storage->getInMemoryMetadataPtr(), // view->getInMemoryMetadataPtr(), + SelectQueryOptions(/* processed_stage*/)); + interpreter.buildQueryPlan(plan); + } else { /// For view storage, we need to rewrite the `modified_query_info.view_query` to optimize read. @@ -677,6 +706,7 @@ QueryPipelineBuilderPtr ReadFromMerge::createSources( modified_context->setSetting("max_streams_to_max_threads_ratio", 1); modified_context->setSetting("max_block_size", max_block_size); + LOG_TRACE(&Poco::Logger::get("ReadFromMerge::createSources"), "creating InterpreterSelectQuery 1.5"); InterpreterSelectQuery interpreter(modified_query_info.query, modified_context, storage, @@ -719,6 +749,7 @@ QueryPipelineBuilderPtr ReadFromMerge::createSources( { modified_select.replaceDatabaseAndTable(database_name, table_name); /// TODO: Find a way to support projections for StorageMerge + LOG_TRACE(&Poco::Logger::get("ReadFromMerge::createSources"), "creating InterpreterSelectQuery 2"); InterpreterSelectQuery interpreter{modified_query_info.query, modified_context, SelectQueryOptions(processed_stage).ignoreProjections()}; From 1fc3ede6eac5fc70d65f4d7239e70b81f13f1b08 Mon Sep 17 00:00:00 2001 From: Ilya Golshtein Date: Thu, 25 May 2023 10:51:45 +0000 Subject: [PATCH 067/123] merge_row_policy: tiny cleanup --- src/Interpreters/InterpreterSelectQuery.cpp | 4 +--- src/Storages/StorageMerge.cpp | 7 ------- 2 files changed, 1 insertion(+), 10 deletions(-) diff --git a/src/Interpreters/InterpreterSelectQuery.cpp b/src/Interpreters/InterpreterSelectQuery.cpp index 21b6f737436e..11b2ab7d0d39 100644 --- a/src/Interpreters/InterpreterSelectQuery.cpp +++ b/src/Interpreters/InterpreterSelectQuery.cpp @@ -548,9 +548,7 @@ InterpreterSelectQuery::InterpreterSelectQuery( if (storage) { - LOG_TRACE(&Poco::Logger::get("InterpretSelectQuery ctor"), "table name: {}", table_id.getTableName()); - row_policy_filter = context->getRowPolicyFilter(table_id.getDatabaseName(), table_id.getTableName(), RowPolicyFilterType::SELECT_FILTER); } else @@ -844,7 +842,7 @@ InterpreterSelectQuery::InterpreterSelectQuery( if (query.prewhere() && !query.where()) analysis_result.prewhere_info->need_filter = true; - if (table_id && got_storage_from_query /* && !joined_tables.isLeftTableFunction() */) + if (table_id && got_storage_from_query && !joined_tables.isLeftTableFunction()) { /// The current user should have the SELECT privilege. If this table_id is for a table /// function we don't check access rights here because in this case they have been already diff --git a/src/Storages/StorageMerge.cpp b/src/Storages/StorageMerge.cpp index 86a9e8c743f2..144a6c687963 100644 --- a/src/Storages/StorageMerge.cpp +++ b/src/Storages/StorageMerge.cpp @@ -45,7 +45,6 @@ #include - namespace { @@ -578,8 +577,6 @@ SelectQueryInfo ReadFromMerge::getModifiedQueryInfo(const SelectQueryInfo & quer modified_query_info.table_expression = replacement_table_expression; modified_query_info.planner_context->getOrCreateTableExpressionData(replacement_table_expression); - - auto get_column_options = GetColumnsOptions(GetColumnsOptions::All).withExtendedObjects().withVirtuals(); if (storage_snapshot->storage.supportsSubcolumns()) get_column_options.withSubcolumns(); @@ -600,10 +597,6 @@ SelectQueryInfo ReadFromMerge::getModifiedQueryInfo(const SelectQueryInfo & quer } modified_query_info.query = queryNodeToSelectQuery(modified_query_info.query_tree); - TreeRewriterResult new_analyzer_res = *modified_query_info.syntax_analyzer_result; - new_analyzer_res.has_explicit_columns = false; - - modified_query_info.syntax_analyzer_result = std::make_shared(std::move(new_analyzer_res)); } else { From 1f6d372e65ea8a6ba8471a1196290559640feab7 Mon Sep 17 00:00:00 2001 From: Ilya Golshtein Date: Sat, 27 May 2023 22:23:19 +0000 Subject: [PATCH 068/123] merge_row_policy: with QueryProcessingStage::Complete --- src/Interpreters/InterpreterSelectQuery.cpp | 9 +++-- src/Storages/StorageMerge.cpp | 38 +++++++++++++++++++-- 2 files changed, 42 insertions(+), 5 deletions(-) diff --git a/src/Interpreters/InterpreterSelectQuery.cpp b/src/Interpreters/InterpreterSelectQuery.cpp index 11b2ab7d0d39..ba79df4853e9 100644 --- a/src/Interpreters/InterpreterSelectQuery.cpp +++ b/src/Interpreters/InterpreterSelectQuery.cpp @@ -100,6 +100,7 @@ namespace ProfileEvents extern const Event SelectQueriesWithSubqueries; extern const Event QueriesWithSubqueries; } +#pragma GCC diagnostic ignored "-Wold-style-cast" namespace DB { @@ -548,12 +549,12 @@ InterpreterSelectQuery::InterpreterSelectQuery( if (storage) { - LOG_TRACE(&Poco::Logger::get("InterpretSelectQuery ctor"), "table name: {}", table_id.getTableName()); + LOG_TRACE(&Poco::Logger::get("InterpretSelectQuery ctor"), " {}, table name: {}", (void*)this, table_id.getTableName()); row_policy_filter = context->getRowPolicyFilter(table_id.getDatabaseName(), table_id.getTableName(), RowPolicyFilterType::SELECT_FILTER); } else { - LOG_TRACE(&Poco::Logger::get("InterpretSelectQuery ctor"), "no storage"); + LOG_TRACE(&Poco::Logger::get("InterpretSelectQuery ctor"), " {}, no storage", (void*)this); } @@ -1451,7 +1452,7 @@ void InterpreterSelectQuery::executeImpl(QueryPlan & query_plan, std::optional

{}", QueryProcessingStage::toString(from_stage), QueryProcessingStage::toString(options.to_stage)); + LOG_TRACE(log, "executeImpl {}, {} -> {}", (void*) this, QueryProcessingStage::toString(from_stage), QueryProcessingStage::toString(options.to_stage)); } if (query_info.projection && query_info.projection->input_order_info && query_info.input_order_info) @@ -1523,6 +1524,8 @@ void InterpreterSelectQuery::executeImpl(QueryPlan & query_plan, std::optional

column_name, expressions.filter_info->do_remove_column); + LOG_TRACE(log, "executeImpl, adding Row-level security filter"); + row_level_security_step->setStepDescription("Row-level security filter"); query_plan.addStep(std::move(row_level_security_step)); } diff --git a/src/Storages/StorageMerge.cpp b/src/Storages/StorageMerge.cpp index 144a6c687963..e1db448c608b 100644 --- a/src/Storages/StorageMerge.cpp +++ b/src/Storages/StorageMerge.cpp @@ -45,6 +45,10 @@ #include +#pragma GCC diagnostic ignored "-Wunused-parameter" +#pragma GCC diagnostic ignored "-Wunused-but-set-variable" + + namespace { @@ -264,6 +268,8 @@ QueryProcessingStage::Enum StorageMerge::getQueryProcessingStage( /// (see removeJoin()) /// /// And for this we need to return FetchColumns. + LOG_TRACE(&Poco::Logger::get("StorageMerge::getQueryProcessingStage"), "to_stage {}", to_stage); + if (const auto * select = query_info.query->as(); select && hasJoin(*select)) return QueryProcessingStage::FetchColumns; @@ -287,13 +293,15 @@ QueryProcessingStage::Enum StorageMerge::getQueryProcessingStage( stage_in_source_tables, table->getQueryProcessingStage(local_context, to_stage, table->getStorageSnapshot(table->getInMemoryMetadataPtr(), local_context), query_info)); + LOG_TRACE(&Poco::Logger::get("StorageMerge::getQueryProcessingStage"), "stage_in_source_tables {}", stage_in_source_tables); } iterator->next(); } } - return selected_table_size == 1 ? stage_in_source_tables : std::min(stage_in_source_tables, QueryProcessingStage::WithMergeableState); + // return selected_table_size == 1 ? stage_in_source_tables : std::min(stage_in_source_tables, QueryProcessingStage::WithMergeableState); + return QueryProcessingStage::Complete; } void StorageMerge::read( @@ -312,6 +320,9 @@ void StorageMerge::read( auto modified_context = Context::createCopy(local_context); modified_context->setSetting("optimize_move_to_prewhere", false); + LOG_TRACE(&Poco::Logger::get("StorageMerge::read"), "processed_stage {}", QueryProcessingStage::toString(processed_stage)); + + bool has_database_virtual_column = false; bool has_table_virtual_column = false; Names real_column_names; @@ -324,7 +335,10 @@ void StorageMerge::read( else if (column_name == "_table" && isVirtualColumn(column_name, storage_snapshot->metadata)) has_table_virtual_column = true; else + { real_column_names.push_back(column_name); + LOG_TRACE(&Poco::Logger::get("StorageMerge::read"), "column_name {}", column_name); + } } StorageListWithLocks selected_tables @@ -353,7 +367,7 @@ void StorageMerge::read( query_plan.addInterpreterContext(modified_context); /// What will be result structure depending on query processed stage in source tables? - Block common_header = getHeaderForProcessingStage(column_names, storage_snapshot, query_info, local_context, processed_stage); + Block common_header = getHeaderForProcessingStage(column_names, storage_snapshot, query_info, local_context, QueryProcessingStage::Complete /* processed_stage */); auto step = std::make_unique( common_header, @@ -477,6 +491,7 @@ void ReadFromMerge::initializePipeline(QueryPipelineBuilder & pipeline, const Bu bool with_aliases = common_processed_stage == QueryProcessingStage::FetchColumns && !storage_columns.getAliases().empty(); if (with_aliases) { + LOG_TRACE(&Poco::Logger::get("ReadFromMerge::initializePipeline"), "with_aliases"); ASTPtr required_columns_expr_list = std::make_shared(); ASTPtr column_expr; @@ -650,6 +665,7 @@ QueryPipelineBuilderPtr ReadFromMerge::createSources( storage_snapshot, modified_query_info); +#pragma GCC diagnostic ignored "-Wunreachable-code" if (processed_stage <= storage_stage || (allow_experimental_analyzer && processed_stage == QueryProcessingStage::FetchColumns)) { /// If there are only virtual columns in query, you must request at least one other column. @@ -660,6 +676,7 @@ QueryPipelineBuilderPtr ReadFromMerge::createSources( StorageView * view = dynamic_cast(storage.get()); if (/* !view || */ allow_experimental_analyzer) + // if (!view || allow_experimental_analyzer) { LOG_TRACE(&Poco::Logger::get("ReadFromMerge::createSources"), "direct storage->read"); storage->read(plan, @@ -687,6 +704,8 @@ QueryPipelineBuilderPtr ReadFromMerge::createSources( storage, storage->getInMemoryMetadataPtr(), // view->getInMemoryMetadataPtr(), SelectQueryOptions(/* processed_stage*/)); + // SelectQueryOptions(processed_stage)); + // SelectQueryOptions(QueryProcessingStage::WithMergeableState)); interpreter.buildQueryPlan(plan); } else @@ -807,6 +826,7 @@ QueryPipelineBuilderPtr ReadFromMerge::createSources( /// Subordinary tables could have different but convertible types, like numeric types of different width. /// We must return streams with structure equals to structure of Merge table. + convertingSourceStream(header, storage_snapshot->metadata, aliases, modified_context, *builder, processed_stage); } @@ -1018,6 +1038,20 @@ void ReadFromMerge::convertingSourceStream( if (local_context->getSettingsRef().allow_experimental_analyzer && processed_stage != QueryProcessingStage::FetchColumns) convert_actions_match_columns_mode = ActionsDAG::MatchColumnsMode::Position; + + for (const auto & column_with_type_and_name : builder.getHeader().getColumnsWithTypeAndName()) + { + LOG_TRACE(&Poco::Logger::get("ReadFromMerge::convertinfSourceStream"), "column name: {} (builder.getHeader().getColumnsWithTypeAndName())", column_with_type_and_name.name); + } + + for (const auto & column_with_type_and_name : header.getColumnsWithTypeAndName()) + { + LOG_TRACE(&Poco::Logger::get("ReadFromMerge::convertinfSourceStream"), "column name: {} (header.getColumnsWithTypeAndName())", column_with_type_and_name.name); + } + + + + auto convert_actions_dag = ActionsDAG::makeConvertingActions(builder.getHeader().getColumnsWithTypeAndName(), header.getColumnsWithTypeAndName(), convert_actions_match_columns_mode); From 1fec9b77a30f4ee6328be64dd8af7f19ca5c693f Mon Sep 17 00:00:00 2001 From: Ilya Golshtein Date: Mon, 29 May 2023 11:15:42 +0000 Subject: [PATCH 069/123] merge_row_policy - extra debug --- src/Interpreters/InterpreterSelectQuery.cpp | 2 ++ src/Storages/StorageMerge.cpp | 4 ++++ 2 files changed, 6 insertions(+) diff --git a/src/Interpreters/InterpreterSelectQuery.cpp b/src/Interpreters/InterpreterSelectQuery.cpp index ba79df4853e9..c2556c19ded3 100644 --- a/src/Interpreters/InterpreterSelectQuery.cpp +++ b/src/Interpreters/InterpreterSelectQuery.cpp @@ -861,6 +861,8 @@ InterpreterSelectQuery::InterpreterSelectQuery( /// Add prewhere actions with alias columns and record needed columns from storage. if (storage) { + LOG_TRACE(log, "calling addPrewhereAliasActions"); + addPrewhereAliasActions(); analysis_result.required_columns = required_columns; } diff --git a/src/Storages/StorageMerge.cpp b/src/Storages/StorageMerge.cpp index e1db448c608b..def1e051ae70 100644 --- a/src/Storages/StorageMerge.cpp +++ b/src/Storages/StorageMerge.cpp @@ -732,9 +732,13 @@ QueryPipelineBuilderPtr ReadFromMerge::createSources( if (auto * read_from_merge_tree = typeid_cast(plan.getRootNode()->step.get())) { + LOG_TRACE(&Poco::Logger::get("ReadFromMerge::createSources"), "ReadFromMergeTree detected"); size_t filters_dags_size = filter_dags.size(); for (size_t i = 0; i < filters_dags_size; ++i) + { + LOG_TRACE(&Poco::Logger::get("ReadFromMerge::createSources"), "adding filter"); read_from_merge_tree->addFilter(filter_dags[i], filter_nodes.nodes[i]); + } } builder = plan.buildQueryPipeline( From 4c7e3c60c747c575dd2f5d7eac7696f2f3038ba5 Mon Sep 17 00:00:00 2001 From: Ilya Golshtein Date: Wed, 31 May 2023 11:48:20 +0000 Subject: [PATCH 070/123] merge_row_policy: original behavior restored + extra debug --- src/Interpreters/InterpreterSelectQuery.cpp | 10 +++++++++- src/Storages/StorageMerge.cpp | 13 +++++++------ 2 files changed, 16 insertions(+), 7 deletions(-) diff --git a/src/Interpreters/InterpreterSelectQuery.cpp b/src/Interpreters/InterpreterSelectQuery.cpp index c2556c19ded3..3d297d599830 100644 --- a/src/Interpreters/InterpreterSelectQuery.cpp +++ b/src/Interpreters/InterpreterSelectQuery.cpp @@ -134,6 +134,9 @@ FilterDAGInfoPtr generateFilterActions( Names & prerequisite_columns, PreparedSetsPtr prepared_sets) { + LOG_TRACE(&Poco::Logger::get("generateFilterActions"), "top of"); + + auto filter_info = std::make_shared(); const auto & db_name = table_id.getDatabaseName(); @@ -549,7 +552,7 @@ InterpreterSelectQuery::InterpreterSelectQuery( if (storage) { - LOG_TRACE(&Poco::Logger::get("InterpretSelectQuery ctor"), " {}, table name: {}", (void*)this, table_id.getTableName()); + LOG_TRACE(&Poco::Logger::get("InterpretSelectQuery ctor"), " {}, table name: {}, calling getRowPolicyFilter", (void*)this, table_id.getTableName()); row_policy_filter = context->getRowPolicyFilter(table_id.getDatabaseName(), table_id.getTableName(), RowPolicyFilterType::SELECT_FILTER); } else @@ -2065,11 +2068,15 @@ void InterpreterSelectQuery::addPrewhereAliasActions() auto & expressions = analysis_result; if (expressions.filter_info) { + LOG_TRACE(&Poco::Logger::get("addPrewhereAliasActions"), " {}, expressions.filter_info", (void*)this); + if (!expressions.prewhere_info) { + LOG_TRACE(&Poco::Logger::get("addPrewhereAliasActions"), " {}, expressions.filter_info 1", (void*)this); const bool does_storage_support_prewhere = !input_pipe && storage && storage->supportsPrewhere(); if (does_storage_support_prewhere && shouldMoveToPrewhere()) { + LOG_TRACE(&Poco::Logger::get("addPrewhereAliasActions"), " {}, expressions.filter_info 1.5", (void*)this); /// Execute row level filter in prewhere as a part of "move to prewhere" optimization. expressions.prewhere_info = std::make_shared( std::move(expressions.filter_info->actions), @@ -2082,6 +2089,7 @@ void InterpreterSelectQuery::addPrewhereAliasActions() } else { + LOG_TRACE(&Poco::Logger::get("addPrewhereAliasActions"), " {}, expressions.filter_info 2", (void*)this); /// Add row level security actions to prewhere. expressions.prewhere_info->row_level_filter = std::move(expressions.filter_info->actions); expressions.prewhere_info->row_level_column_name = std::move(expressions.filter_info->column_name); diff --git a/src/Storages/StorageMerge.cpp b/src/Storages/StorageMerge.cpp index def1e051ae70..f5aaf4614b87 100644 --- a/src/Storages/StorageMerge.cpp +++ b/src/Storages/StorageMerge.cpp @@ -300,8 +300,8 @@ QueryProcessingStage::Enum StorageMerge::getQueryProcessingStage( } } - // return selected_table_size == 1 ? stage_in_source_tables : std::min(stage_in_source_tables, QueryProcessingStage::WithMergeableState); - return QueryProcessingStage::Complete; + return selected_table_size == 1 ? stage_in_source_tables : std::min(stage_in_source_tables, QueryProcessingStage::WithMergeableState); + // return QueryProcessingStage::Complete; } void StorageMerge::read( @@ -367,7 +367,8 @@ void StorageMerge::read( query_plan.addInterpreterContext(modified_context); /// What will be result structure depending on query processed stage in source tables? - Block common_header = getHeaderForProcessingStage(column_names, storage_snapshot, query_info, local_context, QueryProcessingStage::Complete /* processed_stage */); + // Block common_header = getHeaderForProcessingStage(column_names, storage_snapshot, query_info, local_context, QueryProcessingStage::Complete /* processed_stage */); + Block common_header = getHeaderForProcessingStage(column_names, storage_snapshot, query_info, local_context, processed_stage ); auto step = std::make_unique( common_header, @@ -703,8 +704,8 @@ QueryPipelineBuilderPtr ReadFromMerge::createSources( modified_context, storage, storage->getInMemoryMetadataPtr(), // view->getInMemoryMetadataPtr(), - SelectQueryOptions(/* processed_stage*/)); - // SelectQueryOptions(processed_stage)); + // SelectQueryOptions(/* processed_stage*/)); + SelectQueryOptions(processed_stage)); // SelectQueryOptions(QueryProcessingStage::WithMergeableState)); interpreter.buildQueryPlan(plan); } @@ -732,8 +733,8 @@ QueryPipelineBuilderPtr ReadFromMerge::createSources( if (auto * read_from_merge_tree = typeid_cast(plan.getRootNode()->step.get())) { - LOG_TRACE(&Poco::Logger::get("ReadFromMerge::createSources"), "ReadFromMergeTree detected"); size_t filters_dags_size = filter_dags.size(); + LOG_TRACE(&Poco::Logger::get("ReadFromMerge::createSources"), "ReadFromMergeTree detected, DAG size {}", filters_dags_size); for (size_t i = 0; i < filters_dags_size; ++i) { LOG_TRACE(&Poco::Logger::get("ReadFromMerge::createSources"), "adding filter"); From f304c4a85a7e5fae9f375f165f186e2e881953f6 Mon Sep 17 00:00:00 2001 From: Ilya Golshtein Date: Fri, 2 Jun 2023 23:06:10 +0000 Subject: [PATCH 071/123] merge_row_policy: FilterTransform --- src/Interpreters/InterpreterSelectQuery.cpp | 21 +++- .../MergeTree/MergeTreeSequentialSource.cpp | 2 +- src/Storages/StorageMerge.cpp | 98 ++++++++++++++++++- src/Storages/StorageMerge.h | 4 +- 4 files changed, 117 insertions(+), 8 deletions(-) diff --git a/src/Interpreters/InterpreterSelectQuery.cpp b/src/Interpreters/InterpreterSelectQuery.cpp index 3d297d599830..367ebd279855 100644 --- a/src/Interpreters/InterpreterSelectQuery.cpp +++ b/src/Interpreters/InterpreterSelectQuery.cpp @@ -961,6 +961,8 @@ Block InterpreterSelectQuery::getSampleBlockImpl() analysis_result = ExpressionAnalysisResult( *query_analyzer, metadata_snapshot, first_stage, second_stage, options.only_analyze, filter_info, additional_filter_info, source_header); + LOG_TRACE(log, "getSampleBlockImpl {} : source_header after ExpressionAnalysisResult {}", (void*) this, source_header.dumpStructure()); + if (options.to_stage == QueryProcessingStage::Enum::FetchColumns) { @@ -970,8 +972,12 @@ Block InterpreterSelectQuery::getSampleBlockImpl() { header = analysis_result.prewhere_info->prewhere_actions->updateHeader(header); if (analysis_result.prewhere_info->remove_prewhere_column) + { + LOG_TRACE(log, "getSampleBlockImpl {} : erasing column {}", (void*) this, analysis_result.prewhere_info->prewhere_column_name); header.erase(analysis_result.prewhere_info->prewhere_column_name); + } } + LOG_TRACE(log, "getSampleBlockImpl {} : returning header", (void*) this); return header; } @@ -1523,13 +1529,15 @@ void InterpreterSelectQuery::executeImpl(QueryPlan & query_plan, std::optional

column_name, query_plan.getCurrentDataStream().header.dumpStructure()); + auto row_level_security_step = std::make_unique( query_plan.getCurrentDataStream(), expressions.filter_info->actions, expressions.filter_info->column_name, expressions.filter_info->do_remove_column); - LOG_TRACE(log, "executeImpl, adding Row-level security filter"); row_level_security_step->setStepDescription("Row-level security filter"); query_plan.addStep(std::move(row_level_security_step)); @@ -2072,8 +2080,9 @@ void InterpreterSelectQuery::addPrewhereAliasActions() if (!expressions.prewhere_info) { - LOG_TRACE(&Poco::Logger::get("addPrewhereAliasActions"), " {}, expressions.filter_info 1", (void*)this); const bool does_storage_support_prewhere = !input_pipe && storage && storage->supportsPrewhere(); + LOG_TRACE(&Poco::Logger::get("addPrewhereAliasActions"), " {}, expressions.filter_info 1 - does_storage_support_prewhere {} shouldMoveToPrewhere() {}", + (void*)this, does_storage_support_prewhere, shouldMoveToPrewhere()); if (does_storage_support_prewhere && shouldMoveToPrewhere()) { LOG_TRACE(&Poco::Logger::get("addPrewhereAliasActions"), " {}, expressions.filter_info 1.5", (void*)this); @@ -2096,6 +2105,14 @@ void InterpreterSelectQuery::addPrewhereAliasActions() expressions.prewhere_info->row_level_filter->projectInput(false); expressions.filter_info = nullptr; } + if (expressions.prewhere_info) + { + LOG_TRACE(&Poco::Logger::get("addPrewhereAliasActions"), " {} dump: {}", (void*)this, expressions.prewhere_info->dump()); + } + else + { + LOG_TRACE(&Poco::Logger::get("addPrewhereAliasActions"), " no prewhere_info"); + } } auto & prewhere_info = analysis_result.prewhere_info; diff --git a/src/Storages/MergeTree/MergeTreeSequentialSource.cpp b/src/Storages/MergeTree/MergeTreeSequentialSource.cpp index 8a9faa5cee49..16b825d1d7fb 100644 --- a/src/Storages/MergeTree/MergeTreeSequentialSource.cpp +++ b/src/Storages/MergeTree/MergeTreeSequentialSource.cpp @@ -306,7 +306,7 @@ class ReadFromPart final : public ISourceStep const auto & primary_key = storage_snapshot->metadata->getPrimaryKey(); const Names & primary_key_column_names = primary_key.column_names; KeyCondition key_condition(filter, context, primary_key_column_names, primary_key.expression, NameSet{}); - LOG_DEBUG(log, "Key condition: {}", key_condition.toString()); + LOG_DEBUG(log, "ReadFromPart (MergeTreeSequentialSource) Key condition: {}", key_condition.toString()); if (!key_condition.alwaysFalse()) mark_ranges = MergeTreeDataSelectExecutor::markRangesFromPKRange( diff --git a/src/Storages/StorageMerge.cpp b/src/Storages/StorageMerge.cpp index f5aaf4614b87..c4276c220181 100644 --- a/src/Storages/StorageMerge.cpp +++ b/src/Storages/StorageMerge.cpp @@ -37,6 +37,7 @@ #include #include #include +#include #include #include #include @@ -318,7 +319,7 @@ void StorageMerge::read( * since there is no certainty that it works when one of table is MergeTree and other is not. */ auto modified_context = Context::createCopy(local_context); - modified_context->setSetting("optimize_move_to_prewhere", false); + // modified_context->setSetting("optimize_move_to_prewhere", false); LOG_TRACE(&Poco::Logger::get("StorageMerge::read"), "processed_stage {}", QueryProcessingStage::toString(processed_stage)); @@ -832,7 +833,7 @@ QueryPipelineBuilderPtr ReadFromMerge::createSources( /// Subordinary tables could have different but convertible types, like numeric types of different width. /// We must return streams with structure equals to structure of Merge table. - convertingSourceStream(header, storage_snapshot->metadata, aliases, modified_context, *builder, processed_stage); + convertingSourceStream(header, storage_snapshot->metadata, aliases, modified_context, *builder, processed_stage, database_name, table_name); } return builder; @@ -1014,7 +1015,9 @@ void ReadFromMerge::convertingSourceStream( const Aliases & aliases, ContextPtr local_context, QueryPipelineBuilder & builder, - const QueryProcessingStage::Enum & processed_stage) + const QueryProcessingStage::Enum & processed_stage, + const String & database_name, + const String & table_name) { Block before_block_header = builder.getHeader(); @@ -1051,7 +1054,7 @@ void ReadFromMerge::convertingSourceStream( for (const auto & column_with_type_and_name : header.getColumnsWithTypeAndName()) { - LOG_TRACE(&Poco::Logger::get("ReadFromMerge::convertinfSourceStream"), "column name: {} (header.getColumnsWithTypeAndName())", column_with_type_and_name.name); + LOG_TRACE(&Poco::Logger::get("ReadFromMerge::convertingSourceStream"), "column name: {} (header.getColumnsWithTypeAndName())", column_with_type_and_name.name); } @@ -1068,6 +1071,93 @@ void ReadFromMerge::convertingSourceStream( { return std::make_shared(stream_header, actions); }); + + + bool explicit_row_policy_filter_needed = true; + + if (explicit_row_policy_filter_needed) + { + + auto row_policy_filter = local_context->getRowPolicyFilter(database_name, table_name, RowPolicyFilterType::SELECT_FILTER); + + // row_policy_filter->expression + // auto pipe_columns = builder.getHeader().getNamesAndTypesList(); + + + ASTPtr expr = row_policy_filter->expression; + + // auto * select_ast = expr /* query_ast */ ->as(); + // assert(select_ast); + + // select_ast->setExpression(ASTSelectQuery::Expression::SELECT, std::make_shared()); + // auto expr_list = select_ast->select(); + // expr_list->children.push_back(expr); + // String filter_column_name = expr_list->children.at(0)->getColumnName(); + // LOG_TRACE(&Poco::Logger::get("ReadFromMerge::convertinfSourceStream"), "filter_column_name: {} ", filter_column_name); + + auto syntax_result = TreeRewriter(local_context).analyze(expr, pipe_columns); + // auto syntax_result = TreeRewriter(local_context).analyze(expr, NamesAndTypesList()); + auto expression_analyzer = ExpressionAnalyzer{row_policy_filter->expression, syntax_result, local_context}; + + auto actions_dag = expression_analyzer.getActionsDAG(true, false); + LOG_TRACE(&Poco::Logger::get("ReadFromMerge::convertinfSourceStream"), "actions_dag: {},<> {}", actions_dag->dumpNames(), actions_dag->dumpDAG()); + + + + auto filter_actions = std::make_shared(actions_dag, ExpressionActionsSettings::fromContext(local_context, CompileExpressions::yes)); + auto required_columns = filter_actions->getRequiredColumns(); + for (const auto & req_col : required_columns) + { + LOG_TRACE(&Poco::Logger::get("ReadFromMerge::convertinfSourceStream"), "req_col: {}", req_col); + } + + + + LOG_TRACE(&Poco::Logger::get("ReadFromMerge::convertinfSourceStream"), "filter_actions_dag: {},<> {}, <> {}", + filter_actions->getActionsDAG().dumpNames(), filter_actions->getActionsDAG().dumpDAG(), filter_actions->getSampleBlock().dumpStructure()); + + + auto fa_actions_columns_sorted = filter_actions->getSampleBlock().getNames(); + std::sort(fa_actions_columns_sorted.begin(), fa_actions_columns_sorted.end()); + + Names required_columns_sorted = required_columns; + std::sort(required_columns_sorted.begin(), required_columns_sorted.end()); + + Names filter_columns; + + + std::set_difference(fa_actions_columns_sorted.begin(), fa_actions_columns_sorted.end(), + required_columns.begin(), required_columns.end(), + std::inserter(filter_columns, filter_columns.begin())); + + for (const auto & filter_column : filter_columns) + { + LOG_TRACE(&Poco::Logger::get("ReadFromMerge::convertinfSourceStream"), "filter_column: {}", filter_column); + } + + // Block block; + // block = filter_actions->getActionsDAG().updateHeader(std::move(block)); + // LOG_TRACE(&Poco::Logger::get("ReadFromMerge::convertingSourceStream"), "block from updateHeader {}", block.dumpStructure()); + + + + builder.addSimpleTransform([&](const Block & stream_header) + { + LOG_TRACE(&Poco::Logger::get("ReadFromMerge::convertinfSourceStream"), "stream_header {}", stream_header.dumpStructure()); + return std::make_shared(stream_header, filter_actions, filter_columns.front(), true /* remove fake column */); + }); + + + // auto row_level_filter_step = std::make_unique( + // query_plan.getCurrentDataStream(), + // expressions.prewhere_info->row_level_filter, + // expressions.prewhere_info->row_level_column_name, + // true); + + // row_level_filter_step->setStepDescription("Row-level security filter (PREWHERE)"); + // query_plan.addStep(std::move(row_level_filter_step)); + + } } bool ReadFromMerge::requestReadingInOrder(InputOrderInfoPtr order_info_) diff --git a/src/Storages/StorageMerge.h b/src/Storages/StorageMerge.h index babf0dd92e86..fbe6dcec2980 100644 --- a/src/Storages/StorageMerge.h +++ b/src/Storages/StorageMerge.h @@ -201,7 +201,9 @@ class ReadFromMerge final : public SourceStepWithFilter const Aliases & aliases, ContextPtr context, QueryPipelineBuilder & builder, - const QueryProcessingStage::Enum & processed_stage); + const QueryProcessingStage::Enum & processed_stage, + const String & database_name, + const String & table_name); }; } From bf382287a1d6db339c3ff179d31523aa70d32287 Mon Sep 17 00:00:00 2001 From: Ilya Golshtein Date: Mon, 5 Jun 2023 16:00:31 +0000 Subject: [PATCH 072/123] merge_row_policy: row_policy_storage_merge test --- src/Storages/StorageMerge.cpp | 106 ++++++++-------- .../02763_row_policy_storage_merge.reference | 114 ++++++++++++++++++ .../02763_row_policy_storage_merge.sql.j2 | 60 +++++++++ 3 files changed, 230 insertions(+), 50 deletions(-) create mode 100644 tests/queries/0_stateless/02763_row_policy_storage_merge.reference create mode 100644 tests/queries/0_stateless/02763_row_policy_storage_merge.sql.j2 diff --git a/src/Storages/StorageMerge.cpp b/src/Storages/StorageMerge.cpp index c4276c220181..7135fb102539 100644 --- a/src/Storages/StorageMerge.cpp +++ b/src/Storages/StorageMerge.cpp @@ -1080,82 +1080,88 @@ void ReadFromMerge::convertingSourceStream( auto row_policy_filter = local_context->getRowPolicyFilter(database_name, table_name, RowPolicyFilterType::SELECT_FILTER); - // row_policy_filter->expression - // auto pipe_columns = builder.getHeader().getNamesAndTypesList(); + if (row_policy_filter) + { - ASTPtr expr = row_policy_filter->expression; + // row_policy_filter->expression + // auto pipe_columns = builder.getHeader().getNamesAndTypesList(); - // auto * select_ast = expr /* query_ast */ ->as(); - // assert(select_ast); - // select_ast->setExpression(ASTSelectQuery::Expression::SELECT, std::make_shared()); - // auto expr_list = select_ast->select(); - // expr_list->children.push_back(expr); - // String filter_column_name = expr_list->children.at(0)->getColumnName(); - // LOG_TRACE(&Poco::Logger::get("ReadFromMerge::convertinfSourceStream"), "filter_column_name: {} ", filter_column_name); + ASTPtr expr = row_policy_filter->expression; - auto syntax_result = TreeRewriter(local_context).analyze(expr, pipe_columns); - // auto syntax_result = TreeRewriter(local_context).analyze(expr, NamesAndTypesList()); - auto expression_analyzer = ExpressionAnalyzer{row_policy_filter->expression, syntax_result, local_context}; + // auto * select_ast = expr /* query_ast */ ->as(); + // assert(select_ast); - auto actions_dag = expression_analyzer.getActionsDAG(true, false); - LOG_TRACE(&Poco::Logger::get("ReadFromMerge::convertinfSourceStream"), "actions_dag: {},<> {}", actions_dag->dumpNames(), actions_dag->dumpDAG()); + // select_ast->setExpression(ASTSelectQuery::Expression::SELECT, std::make_shared()); + // auto expr_list = select_ast->select(); + // expr_list->children.push_back(expr); + // String filter_column_name = expr_list->children.at(0)->getColumnName(); + // LOG_TRACE(&Poco::Logger::get("ReadFromMerge::convertinfSourceStream"), "filter_column_name: {} ", filter_column_name); + auto syntax_result = TreeRewriter(local_context).analyze(expr, pipe_columns); + // auto syntax_result = TreeRewriter(local_context).analyze(expr, NamesAndTypesList()); + auto expression_analyzer = ExpressionAnalyzer{row_policy_filter->expression, syntax_result, local_context}; + auto actions_dag = expression_analyzer.getActionsDAG(true, false); + LOG_TRACE(&Poco::Logger::get("ReadFromMerge::convertinfSourceStream"), "actions_dag: {},<> {}", actions_dag->dumpNames(), actions_dag->dumpDAG()); - auto filter_actions = std::make_shared(actions_dag, ExpressionActionsSettings::fromContext(local_context, CompileExpressions::yes)); - auto required_columns = filter_actions->getRequiredColumns(); - for (const auto & req_col : required_columns) - { - LOG_TRACE(&Poco::Logger::get("ReadFromMerge::convertinfSourceStream"), "req_col: {}", req_col); - } + auto filter_actions = std::make_shared(actions_dag, ExpressionActionsSettings::fromContext(local_context, CompileExpressions::yes)); + auto required_columns = filter_actions->getRequiredColumns(); + for (const auto & req_col : required_columns) + { + LOG_TRACE(&Poco::Logger::get("ReadFromMerge::convertinfSourceStream"), "req_col: {}", req_col); + } - LOG_TRACE(&Poco::Logger::get("ReadFromMerge::convertinfSourceStream"), "filter_actions_dag: {},<> {}, <> {}", - filter_actions->getActionsDAG().dumpNames(), filter_actions->getActionsDAG().dumpDAG(), filter_actions->getSampleBlock().dumpStructure()); - auto fa_actions_columns_sorted = filter_actions->getSampleBlock().getNames(); - std::sort(fa_actions_columns_sorted.begin(), fa_actions_columns_sorted.end()); + LOG_TRACE(&Poco::Logger::get("ReadFromMerge::convertinfSourceStream"), "filter_actions_dag: {},<> {}, <> {}", + filter_actions->getActionsDAG().dumpNames(), filter_actions->getActionsDAG().dumpDAG(), filter_actions->getSampleBlock().dumpStructure()); - Names required_columns_sorted = required_columns; - std::sort(required_columns_sorted.begin(), required_columns_sorted.end()); - Names filter_columns; + auto fa_actions_columns_sorted = filter_actions->getSampleBlock().getNames(); + std::sort(fa_actions_columns_sorted.begin(), fa_actions_columns_sorted.end()); + Names required_columns_sorted = required_columns; + std::sort(required_columns_sorted.begin(), required_columns_sorted.end()); - std::set_difference(fa_actions_columns_sorted.begin(), fa_actions_columns_sorted.end(), - required_columns.begin(), required_columns.end(), - std::inserter(filter_columns, filter_columns.begin())); + Names filter_columns; - for (const auto & filter_column : filter_columns) - { - LOG_TRACE(&Poco::Logger::get("ReadFromMerge::convertinfSourceStream"), "filter_column: {}", filter_column); - } - // Block block; - // block = filter_actions->getActionsDAG().updateHeader(std::move(block)); - // LOG_TRACE(&Poco::Logger::get("ReadFromMerge::convertingSourceStream"), "block from updateHeader {}", block.dumpStructure()); + std::set_difference(fa_actions_columns_sorted.begin(), fa_actions_columns_sorted.end(), + required_columns.begin(), required_columns.end(), + std::inserter(filter_columns, filter_columns.begin())); + + for (const auto & filter_column : filter_columns) + { + LOG_TRACE(&Poco::Logger::get("ReadFromMerge::convertinfSourceStream"), "filter_column: {}", filter_column); + } + // Block block; + // block = filter_actions->getActionsDAG().updateHeader(std::move(block)); + // LOG_TRACE(&Poco::Logger::get("ReadFromMerge::convertingSourceStream"), "block from updateHeader {}", block.dumpStructure()); - builder.addSimpleTransform([&](const Block & stream_header) - { - LOG_TRACE(&Poco::Logger::get("ReadFromMerge::convertinfSourceStream"), "stream_header {}", stream_header.dumpStructure()); - return std::make_shared(stream_header, filter_actions, filter_columns.front(), true /* remove fake column */); - }); + + builder.addSimpleTransform([&](const Block & stream_header) + { + LOG_TRACE(&Poco::Logger::get("ReadFromMerge::convertinfSourceStream"), "stream_header {}", stream_header.dumpStructure()); + return std::make_shared(stream_header, filter_actions, filter_columns.front(), true /* remove fake column */); + }); - // auto row_level_filter_step = std::make_unique( - // query_plan.getCurrentDataStream(), - // expressions.prewhere_info->row_level_filter, - // expressions.prewhere_info->row_level_column_name, - // true); + // auto row_level_filter_step = std::make_unique( + // query_plan.getCurrentDataStream(), + // expressions.prewhere_info->row_level_filter, + // expressions.prewhere_info->row_level_column_name, + // true); + + // row_level_filter_step->setStepDescription("Row-level security filter (PREWHERE)"); + // query_plan.addStep(std::move(row_level_filter_step)); + } - // row_level_filter_step->setStepDescription("Row-level security filter (PREWHERE)"); - // query_plan.addStep(std::move(row_level_filter_step)); } } diff --git a/tests/queries/0_stateless/02763_row_policy_storage_merge.reference b/tests/queries/0_stateless/02763_row_policy_storage_merge.reference new file mode 100644 index 000000000000..ab531c5e300a --- /dev/null +++ b/tests/queries/0_stateless/02763_row_policy_storage_merge.reference @@ -0,0 +1,114 @@ +1 +1 +1 +1 +2 +2 +2 +2 +3 +3 +3 +3 +4 +4 +4 +4 +1 +2 +3 +4 +SETTINGS optimize_move_to_prewhere= 0 +SELECT * FROM 02763_merge_log_1 +3 +SELECT * FROM merge(currentDatabase(), 02763_merge_log_1) +3 +SELECT * FROM merge(currentDatabase(), 02763_merge_log) +1 +2 +3 +3 +4 +SELECT * FROM merge(currentDatabase(), 02763_merge_log) WHERE x>2 +3 +3 +4 +SELECT * FROM 02763_merge_merge_1 +4 +SELECT * FROM merge(currentDatabase(), 02763_merge_merge_1 +4 +SELECT * FROM merge(currentDatabase(), 02763_merge_merge) +1 +2 +3 +4 +4 +SELECT * FROM merge(currentDatabase(), 02763_merge_merge) WHERE x>2 +3 +4 +4 +SELECT * FROM merge(currentDatabase(), 02763_merge) +1 +1 +2 +2 +3 +3 +3 +4 +4 +4 +SELECT * FROM merge(currentDatabase(), 02763_merge) WHEER x>2 +3 +3 +3 +4 +4 +4 +SETTINGS optimize_move_to_prewhere= 1 +SELECT * FROM 02763_merge_log_1 +3 +SELECT * FROM merge(currentDatabase(), 02763_merge_log_1) +3 +SELECT * FROM merge(currentDatabase(), 02763_merge_log) +1 +2 +3 +3 +4 +SELECT * FROM merge(currentDatabase(), 02763_merge_log) WHERE x>2 +3 +3 +4 +SELECT * FROM 02763_merge_merge_1 +4 +SELECT * FROM merge(currentDatabase(), 02763_merge_merge_1 +4 +SELECT * FROM merge(currentDatabase(), 02763_merge_merge) +1 +2 +3 +4 +4 +SELECT * FROM merge(currentDatabase(), 02763_merge_merge) WHERE x>2 +3 +4 +4 +SELECT * FROM merge(currentDatabase(), 02763_merge) +1 +1 +2 +2 +3 +3 +3 +4 +4 +4 +SELECT * FROM merge(currentDatabase(), 02763_merge) WHEER x>2 +3 +3 +3 +4 +4 +4 diff --git a/tests/queries/0_stateless/02763_row_policy_storage_merge.sql.j2 b/tests/queries/0_stateless/02763_row_policy_storage_merge.sql.j2 new file mode 100644 index 000000000000..3883b357602d --- /dev/null +++ b/tests/queries/0_stateless/02763_row_policy_storage_merge.sql.j2 @@ -0,0 +1,60 @@ +DROP TABLE IF EXISTS 02763_merge_log_1; +DROP TABLE IF EXISTS 02763_merge_log_2; +DROP TABLE IF EXISTS 02763_merge_merge_1; +DROP TABLE IF EXISTS 02763_merge_merge_2; +DROP ROW POLICY IF EXISTS 02763_filter_1 ON 02763_merge_log_1; +DROP ROW POLICY IF EXISTS 02763_filter_2 ON 02763_merge_merge_1; + + +CREATE TABLE 02763_merge_log_1 (x UInt8) ENGINE = Log; +CREATE TABLE 02763_merge_log_2 (x UInt8) ENGINE = Log; + +CREATE TABLE 02763_merge_merge_1 (x UInt8) ENGINE = MergeTree ORDER BY x; +CREATE TABLE 02763_merge_merge_2 (x UInt8) ENGINE = MergeTree ORDER BY x; + +INSERT INTO 02763_merge_log_1 VALUES (1), (2), (3), (4); +INSERT INTO 02763_merge_log_2 VALUES (1), (2), (3), (4); +INSERT INTO 02763_merge_merge_1 VALUES (1), (2), (3), (4); +INSERT INTO 02763_merge_merge_2 VALUES (1), (2), (3), (4); + +SELECT * FROM merge(currentDatabase(), '02763_merge') ORDER BY x; + +SELECT * FROM 02763_merge_log_1 ORDER BY x; + + +{% for prew in [0 , 1] -%} + +SELECT 'SETTINGS optimize_move_to_prewhere= {{prew}}'; + +CREATE ROW POLICY 02763_filter_1 ON 02763_merge_log_1 USING x=3 AS permissive TO ALL; + +SELECT 'SELECT * FROM 02763_merge_log_1'; +SELECT * FROM 02763_merge_log_1 ORDER BY x SETTINGS optimize_move_to_prewhere= {{prew}}; +SELECT 'SELECT * FROM merge(currentDatabase(), 02763_merge_log_1)'; +SELECT * FROM merge(currentDatabase(), '02763_merge_log_1') ORDER BY x SETTINGS optimize_move_to_prewhere= {{prew}};; +SELECT 'SELECT * FROM merge(currentDatabase(), 02763_merge_log)'; +SELECT * FROM merge(currentDatabase(), '02763_merge_log') ORDER BY x SETTINGS optimize_move_to_prewhere= {{prew}};; +SELECT 'SELECT * FROM merge(currentDatabase(), 02763_merge_log) WHERE x>2'; +SELECT * FROM merge(currentDatabase(), '02763_merge_log') WHERE x>2 ORDER BY x SETTINGS optimize_move_to_prewhere= {{prew}};; + +CREATE ROW POLICY 02763_filter_2 ON 02763_merge_merge_1 USING x=4 AS permissive TO ALL; + +SELECT 'SELECT * FROM 02763_merge_merge_1'; +SELECT * FROM 02763_merge_merge_1 ORDER BY x SETTINGS optimize_move_to_prewhere= {{prew}};; +SELECT 'SELECT * FROM merge(currentDatabase(), 02763_merge_merge_1'; +SELECT * FROM merge(currentDatabase(), '02763_merge_merge_1') ORDER BY x SETTINGS optimize_move_to_prewhere= {{prew}};; +SELECT 'SELECT * FROM merge(currentDatabase(), 02763_merge_merge)'; +SELECT * FROM merge(currentDatabase(), '02763_merge_merge') ORDER BY x SETTINGS optimize_move_to_prewhere= {{prew}};; +SELECT 'SELECT * FROM merge(currentDatabase(), 02763_merge_merge) WHERE x>2'; +SELECT * FROM merge(currentDatabase(), '02763_merge_merge') WHERE x>2 ORDER BY x SETTINGS optimize_move_to_prewhere= {{prew}};; + + +SELECT 'SELECT * FROM merge(currentDatabase(), 02763_merge)'; +SELECT * FROM merge(currentDatabase(), '02763_merge') ORDER BY x SETTINGS optimize_move_to_prewhere= {{prew}};; +SELECT 'SELECT * FROM merge(currentDatabase(), 02763_merge) WHEER x>2'; +SELECT * FROM merge(currentDatabase(), '02763_merge') WHERE x>2 ORDER BY x SETTINGS optimize_move_to_prewhere= {{prew}};; + +DROP ROW POLICY 02763_filter_1 ON 02763_merge_log_1; +DROP ROW POLICY 02763_filter_2 ON 02763_merge_merge_1; + +{% endfor %} From e6caf500a8d71774fccb6916716aafebf6d2320e Mon Sep 17 00:00:00 2001 From: Ilya Golshtein Date: Tue, 6 Jun 2023 20:45:48 +0000 Subject: [PATCH 073/123] merge_row_policy: style fixes and back to storage->read --- src/Interpreters/InterpreterSelectQuery.cpp | 2 +- src/Storages/StorageMerge.cpp | 44 ++++++++++----------- 2 files changed, 23 insertions(+), 23 deletions(-) diff --git a/src/Interpreters/InterpreterSelectQuery.cpp b/src/Interpreters/InterpreterSelectQuery.cpp index 367ebd279855..7a574ffecab3 100644 --- a/src/Interpreters/InterpreterSelectQuery.cpp +++ b/src/Interpreters/InterpreterSelectQuery.cpp @@ -1529,7 +1529,7 @@ void InterpreterSelectQuery::executeImpl(QueryPlan & query_plan, std::optional

column_name, query_plan.getCurrentDataStream().header.dumpStructure()); auto row_level_security_step = std::make_unique( diff --git a/src/Storages/StorageMerge.cpp b/src/Storages/StorageMerge.cpp index 7135fb102539..7208dc3a6eda 100644 --- a/src/Storages/StorageMerge.cpp +++ b/src/Storages/StorageMerge.cpp @@ -369,7 +369,7 @@ void StorageMerge::read( /// What will be result structure depending on query processed stage in source tables? // Block common_header = getHeaderForProcessingStage(column_names, storage_snapshot, query_info, local_context, QueryProcessingStage::Complete /* processed_stage */); - Block common_header = getHeaderForProcessingStage(column_names, storage_snapshot, query_info, local_context, processed_stage ); + Block common_header = getHeaderForProcessingStage(column_names, storage_snapshot, query_info, local_context, processed_stage); auto step = std::make_unique( common_header, @@ -677,7 +677,7 @@ QueryPipelineBuilderPtr ReadFromMerge::createSources( QueryPlan & plan = child_plans.emplace_back(); StorageView * view = dynamic_cast(storage.get()); - if (/* !view || */ allow_experimental_analyzer) + if ( !view || allow_experimental_analyzer) // if (!view || allow_experimental_analyzer) { LOG_TRACE(&Poco::Logger::get("ReadFromMerge::createSources"), "direct storage->read"); @@ -690,26 +690,26 @@ QueryPipelineBuilderPtr ReadFromMerge::createSources( max_block_size, UInt32(streams_num)); } - else if (!view) - { - /// For view storage, we need to rewrite the `modified_query_info.view_query` to optimize read. - /// The most intuitive way is to use InterpreterSelectQuery. - - /// Intercept the settings - modified_context->setSetting("max_threads", streams_num); - modified_context->setSetting("max_streams_to_max_threads_ratio", 1); - modified_context->setSetting("max_block_size", max_block_size); - - LOG_TRACE(&Poco::Logger::get("ReadFromMerge::createSources"), "creating InterpreterSelectQuery 1.0"); - InterpreterSelectQuery interpreter(modified_query_info.query, - modified_context, - storage, - storage->getInMemoryMetadataPtr(), // view->getInMemoryMetadataPtr(), - // SelectQueryOptions(/* processed_stage*/)); - SelectQueryOptions(processed_stage)); - // SelectQueryOptions(QueryProcessingStage::WithMergeableState)); - interpreter.buildQueryPlan(plan); - } + // else if (!view) + // { + // /// For view storage, we need to rewrite the `modified_query_info.view_query` to optimize read. + // /// The most intuitive way is to use InterpreterSelectQuery. + + // /// Intercept the settings + // modified_context->setSetting("max_threads", streams_num); + // modified_context->setSetting("max_streams_to_max_threads_ratio", 1); + // modified_context->setSetting("max_block_size", max_block_size); + + // LOG_TRACE(&Poco::Logger::get("ReadFromMerge::createSources"), "creating InterpreterSelectQuery 1.0"); + // InterpreterSelectQuery interpreter(modified_query_info.query, + // modified_context, + // storage, + // storage->getInMemoryMetadataPtr(), // view->getInMemoryMetadataPtr(), + // // SelectQueryOptions(/* processed_stage*/)); + // SelectQueryOptions(processed_stage)); + // // SelectQueryOptions(QueryProcessingStage::WithMergeableState)); + // interpreter.buildQueryPlan(plan); + // } else { /// For view storage, we need to rewrite the `modified_query_info.view_query` to optimize read. From e4de8a3dd3ae634cfd403f3a9c227b439054f55f Mon Sep 17 00:00:00 2001 From: Ilya Golshtein Date: Fri, 9 Jun 2023 09:03:47 +0000 Subject: [PATCH 074/123] merge_row_policy: addFilter() instead of FilterTransform --- src/Storages/StorageMerge.cpp | 71 ++++++++++++++++++++++++++++++++++- 1 file changed, 69 insertions(+), 2 deletions(-) diff --git a/src/Storages/StorageMerge.cpp b/src/Storages/StorageMerge.cpp index 7208dc3a6eda..9674ce8c0b0b 100644 --- a/src/Storages/StorageMerge.cpp +++ b/src/Storages/StorageMerge.cpp @@ -677,6 +677,8 @@ QueryPipelineBuilderPtr ReadFromMerge::createSources( QueryPlan & plan = child_plans.emplace_back(); StorageView * view = dynamic_cast(storage.get()); + bool direct_read = false; + if ( !view || allow_experimental_analyzer) // if (!view || allow_experimental_analyzer) { @@ -689,6 +691,7 @@ QueryPipelineBuilderPtr ReadFromMerge::createSources( processed_stage, max_block_size, UInt32(streams_num)); + direct_read = true; } // else if (!view) // { @@ -742,10 +745,74 @@ QueryPipelineBuilderPtr ReadFromMerge::createSources( read_from_merge_tree->addFilter(filter_dags[i], filter_nodes.nodes[i]); } } - builder = plan.buildQueryPipeline( QueryPlanOptimizationSettings::fromContext(modified_context), BuildQueryPipelineSettings::fromContext(modified_context)); + if (auto * source_step_with_filter = typeid_cast(plan.getRootNode()->step.get())) + { + auto row_policy_filter = modified_context->getRowPolicyFilter(database_name, table_name, RowPolicyFilterType::SELECT_FILTER); + + if (row_policy_filter) + { + + + // row_policy_filter->expression + // auto pipe_columns = builder.getHeader().getNamesAndTypesList(); + + + ASTPtr expr = row_policy_filter->expression; + + // auto * select_ast = expr /* query_ast */ ->as(); + // assert(select_ast); + + // select_ast->setExpression(ASTSelectQuery::Expression::SELECT, std::make_shared()); + // auto expr_list = select_ast->select(); + // expr_list->children.push_back(expr); + // String filter_column_name = expr_list->children.at(0)->getColumnName(); + // LOG_TRACE(&Poco::Logger::get("ReadFromMerge::convertinfSourceStream"), "filter_column_name: {} ", filter_column_name); + + auto syntax_result = TreeRewriter(modified_context).analyze(expr, builder->getHeader().getNamesAndTypesList() /* pipe_columns*/); + // auto syntax_result = TreeRewriter(local_context).analyze(expr, NamesAndTypesList()); + auto expression_analyzer = ExpressionAnalyzer{row_policy_filter->expression, syntax_result, modified_context}; + + auto filter_dag_ptr = expression_analyzer.getActionsDAG(true, false); + + + auto filter_actions = std::make_shared(filter_dag_ptr, ExpressionActionsSettings::fromContext(modified_context, CompileExpressions::yes)); + auto required_columns = filter_actions->getRequiredColumns(); + + for (const auto & req_col : required_columns) + { + LOG_TRACE(&Poco::Logger::get("ReadFromMerge::convertinfSourceStream"), "req_col: {}", req_col); + } + + + + LOG_TRACE(&Poco::Logger::get("ReadFromMerge::convertinfSourceStream"), "filter_actions_dag: {},<> {}, <> {}", + filter_actions->getActionsDAG().dumpNames(), filter_actions->getActionsDAG().dumpDAG(), filter_actions->getSampleBlock().dumpStructure()); + + + auto fa_actions_columns_sorted = filter_actions->getSampleBlock().getNames(); + std::sort(fa_actions_columns_sorted.begin(), fa_actions_columns_sorted.end()); + + Names required_columns_sorted = required_columns; + std::sort(required_columns_sorted.begin(), required_columns_sorted.end()); + + Names filter_columns; + + + std::set_difference(fa_actions_columns_sorted.begin(), fa_actions_columns_sorted.end(), + required_columns.begin(), required_columns.end(), + std::inserter(filter_columns, filter_columns.begin())); + + + + LOG_TRACE(&Poco::Logger::get("ReadFromMerge::createSources"), "SourceStepWithFilter detected"); + source_step_with_filter->addFilter(filter_dag_ptr, filter_columns.front()); + } + } + + } else if (processed_stage > storage_stage || (allow_experimental_analyzer && processed_stage != QueryProcessingStage::FetchColumns)) { @@ -1073,7 +1140,7 @@ void ReadFromMerge::convertingSourceStream( }); - bool explicit_row_policy_filter_needed = true; + bool explicit_row_policy_filter_needed = false; if (explicit_row_policy_filter_needed) { From 13dd5a168a68de1142f3d37dfad4e25757cd193c Mon Sep 17 00:00:00 2001 From: Ilya Golshtein Date: Tue, 13 Jun 2023 17:19:20 +0000 Subject: [PATCH 075/123] merge_row_policy: add_filter() actually works --- src/Interpreters/InterpreterSelectQuery.cpp | 1 - src/Storages/StorageMerge.cpp | 39 ++++++++------------- 2 files changed, 15 insertions(+), 25 deletions(-) diff --git a/src/Interpreters/InterpreterSelectQuery.cpp b/src/Interpreters/InterpreterSelectQuery.cpp index 7a574ffecab3..9716957bfe29 100644 --- a/src/Interpreters/InterpreterSelectQuery.cpp +++ b/src/Interpreters/InterpreterSelectQuery.cpp @@ -561,7 +561,6 @@ InterpreterSelectQuery::InterpreterSelectQuery( } - StorageView * view = nullptr; if (storage) view = dynamic_cast(storage.get()); diff --git a/src/Storages/StorageMerge.cpp b/src/Storages/StorageMerge.cpp index 9674ce8c0b0b..8b21bf76287a 100644 --- a/src/Storages/StorageMerge.cpp +++ b/src/Storages/StorageMerge.cpp @@ -34,6 +34,7 @@ #include #include #include +#include #include #include #include @@ -668,6 +669,7 @@ QueryPipelineBuilderPtr ReadFromMerge::createSources( modified_query_info); #pragma GCC diagnostic ignored "-Wunreachable-code" +#pragma GCC diagnostic ignored "-Wunused-variable" if (processed_stage <= storage_stage || (allow_experimental_analyzer && processed_stage == QueryProcessingStage::FetchColumns)) { /// If there are only virtual columns in query, you must request at least one other column. @@ -679,7 +681,7 @@ QueryPipelineBuilderPtr ReadFromMerge::createSources( StorageView * view = dynamic_cast(storage.get()); bool direct_read = false; - if ( !view || allow_experimental_analyzer) + if (!view || allow_experimental_analyzer) // if (!view || allow_experimental_analyzer) { LOG_TRACE(&Poco::Logger::get("ReadFromMerge::createSources"), "direct storage->read"); @@ -741,25 +743,18 @@ QueryPipelineBuilderPtr ReadFromMerge::createSources( LOG_TRACE(&Poco::Logger::get("ReadFromMerge::createSources"), "ReadFromMergeTree detected, DAG size {}", filters_dags_size); for (size_t i = 0; i < filters_dags_size; ++i) { - LOG_TRACE(&Poco::Logger::get("ReadFromMerge::createSources"), "adding filter"); + LOG_TRACE(&Poco::Logger::get("ReadFromMerge::createSources"), "adding filter {}", filter_dags[i]->dumpDAG()); read_from_merge_tree->addFilter(filter_dags[i], filter_nodes.nodes[i]); } } - builder = plan.buildQueryPipeline( - QueryPlanOptimizationSettings::fromContext(modified_context), - BuildQueryPipelineSettings::fromContext(modified_context)); - if (auto * source_step_with_filter = typeid_cast(plan.getRootNode()->step.get())) + if (auto * source_step_with_filter = dynamic_cast((plan.getRootNode()->step.get()))) { auto row_policy_filter = modified_context->getRowPolicyFilter(database_name, table_name, RowPolicyFilterType::SELECT_FILTER); if (row_policy_filter) { - - // row_policy_filter->expression // auto pipe_columns = builder.getHeader().getNamesAndTypesList(); - - ASTPtr expr = row_policy_filter->expression; // auto * select_ast = expr /* query_ast */ ->as(); @@ -771,7 +766,7 @@ QueryPipelineBuilderPtr ReadFromMerge::createSources( // String filter_column_name = expr_list->children.at(0)->getColumnName(); // LOG_TRACE(&Poco::Logger::get("ReadFromMerge::convertinfSourceStream"), "filter_column_name: {} ", filter_column_name); - auto syntax_result = TreeRewriter(modified_context).analyze(expr, builder->getHeader().getNamesAndTypesList() /* pipe_columns*/); + auto syntax_result = TreeRewriter(modified_context).analyze(expr, header/*builder->getHeader().*/.getNamesAndTypesList() /* pipe_columns*/); // auto syntax_result = TreeRewriter(local_context).analyze(expr, NamesAndTypesList()); auto expression_analyzer = ExpressionAnalyzer{row_policy_filter->expression, syntax_result, modified_context}; @@ -787,7 +782,6 @@ QueryPipelineBuilderPtr ReadFromMerge::createSources( } - LOG_TRACE(&Poco::Logger::get("ReadFromMerge::convertinfSourceStream"), "filter_actions_dag: {},<> {}, <> {}", filter_actions->getActionsDAG().dumpNames(), filter_actions->getActionsDAG().dumpDAG(), filter_actions->getSampleBlock().dumpStructure()); @@ -806,12 +800,18 @@ QueryPipelineBuilderPtr ReadFromMerge::createSources( std::inserter(filter_columns, filter_columns.begin())); - LOG_TRACE(&Poco::Logger::get("ReadFromMerge::createSources"), "SourceStepWithFilter detected"); - source_step_with_filter->addFilter(filter_dag_ptr, filter_columns.front()); + auto found_column = filter_dag_ptr->tryFindInOutputs(filter_columns.front()); + assert(found_column); + + // LOG_TRACE(&Poco::Logger::get("ReadFromMerge::createSources"), "found column {}", found_column->dumpDAG()); + + source_step_with_filter->addFilter(/* filter_actions */ filter_dag_ptr, filter_columns.front()); } } - + builder = plan.buildQueryPipeline( + QueryPlanOptimizationSettings::fromContext(modified_context), + BuildQueryPipelineSettings::fromContext(modified_context)); } else if (processed_stage > storage_stage || (allow_experimental_analyzer && processed_stage != QueryProcessingStage::FetchColumns)) @@ -1124,9 +1124,6 @@ void ReadFromMerge::convertingSourceStream( LOG_TRACE(&Poco::Logger::get("ReadFromMerge::convertingSourceStream"), "column name: {} (header.getColumnsWithTypeAndName())", column_with_type_and_name.name); } - - - auto convert_actions_dag = ActionsDAG::makeConvertingActions(builder.getHeader().getColumnsWithTypeAndName(), header.getColumnsWithTypeAndName(), convert_actions_match_columns_mode); @@ -1150,7 +1147,6 @@ void ReadFromMerge::convertingSourceStream( if (row_policy_filter) { - // row_policy_filter->expression // auto pipe_columns = builder.getHeader().getNamesAndTypesList(); @@ -1174,7 +1170,6 @@ void ReadFromMerge::convertingSourceStream( LOG_TRACE(&Poco::Logger::get("ReadFromMerge::convertinfSourceStream"), "actions_dag: {},<> {}", actions_dag->dumpNames(), actions_dag->dumpDAG()); - auto filter_actions = std::make_shared(actions_dag, ExpressionActionsSettings::fromContext(local_context, CompileExpressions::yes)); auto required_columns = filter_actions->getRequiredColumns(); for (const auto & req_col : required_columns) @@ -1183,7 +1178,6 @@ void ReadFromMerge::convertingSourceStream( } - LOG_TRACE(&Poco::Logger::get("ReadFromMerge::convertinfSourceStream"), "filter_actions_dag: {},<> {}, <> {}", filter_actions->getActionsDAG().dumpNames(), filter_actions->getActionsDAG().dumpDAG(), filter_actions->getSampleBlock().dumpStructure()); @@ -1210,8 +1204,6 @@ void ReadFromMerge::convertingSourceStream( // block = filter_actions->getActionsDAG().updateHeader(std::move(block)); // LOG_TRACE(&Poco::Logger::get("ReadFromMerge::convertingSourceStream"), "block from updateHeader {}", block.dumpStructure()); - - builder.addSimpleTransform([&](const Block & stream_header) { LOG_TRACE(&Poco::Logger::get("ReadFromMerge::convertinfSourceStream"), "stream_header {}", stream_header.dumpStructure()); @@ -1229,7 +1221,6 @@ void ReadFromMerge::convertingSourceStream( // query_plan.addStep(std::move(row_level_filter_step)); } - } } From 1b01f039f8199fb5bbcb86a3a6f395856c5cf9f5 Mon Sep 17 00:00:00 2001 From: Ilya Golshtein Date: Wed, 14 Jun 2023 06:29:41 +0000 Subject: [PATCH 076/123] merge_row_policy: addFilter(() together with FilterTransform --- src/Storages/StorageMerge.cpp | 2 +- .../02763_row_policy_storage_merge.reference | 28 +++++++++++++++++-- .../02763_row_policy_storage_merge.sql.j2 | 23 ++++++++------- 3 files changed, 40 insertions(+), 13 deletions(-) diff --git a/src/Storages/StorageMerge.cpp b/src/Storages/StorageMerge.cpp index 8b21bf76287a..ae3dbcbc9539 100644 --- a/src/Storages/StorageMerge.cpp +++ b/src/Storages/StorageMerge.cpp @@ -1137,7 +1137,7 @@ void ReadFromMerge::convertingSourceStream( }); - bool explicit_row_policy_filter_needed = false; + bool explicit_row_policy_filter_needed = true; if (explicit_row_policy_filter_needed) { diff --git a/tests/queries/0_stateless/02763_row_policy_storage_merge.reference b/tests/queries/0_stateless/02763_row_policy_storage_merge.reference index ab531c5e300a..9dcc5f449ab1 100644 --- a/tests/queries/0_stateless/02763_row_policy_storage_merge.reference +++ b/tests/queries/0_stateless/02763_row_policy_storage_merge.reference @@ -58,7 +58,19 @@ SELECT * FROM merge(currentDatabase(), 02763_merge) 4 4 4 -SELECT * FROM merge(currentDatabase(), 02763_merge) WHEER x>2 +SELECT * FROM merge(currentDatabase(), 02763_merge) WHERE x>2 +3 +3 +3 +4 +4 +4 +aaa 6 +aaa 6 +aaa 6 +aaa 8 +aaa 8 +aaa 8 3 3 3 @@ -105,7 +117,19 @@ SELECT * FROM merge(currentDatabase(), 02763_merge) 4 4 4 -SELECT * FROM merge(currentDatabase(), 02763_merge) WHEER x>2 +SELECT * FROM merge(currentDatabase(), 02763_merge) WHERE x>2 +3 +3 +3 +4 +4 +4 +aaa 6 +aaa 6 +aaa 6 +aaa 8 +aaa 8 +aaa 8 3 3 3 diff --git a/tests/queries/0_stateless/02763_row_policy_storage_merge.sql.j2 b/tests/queries/0_stateless/02763_row_policy_storage_merge.sql.j2 index 3883b357602d..33b02275d4a8 100644 --- a/tests/queries/0_stateless/02763_row_policy_storage_merge.sql.j2 +++ b/tests/queries/0_stateless/02763_row_policy_storage_merge.sql.j2 @@ -31,28 +31,31 @@ CREATE ROW POLICY 02763_filter_1 ON 02763_merge_log_1 USING x=3 AS permissive TO SELECT 'SELECT * FROM 02763_merge_log_1'; SELECT * FROM 02763_merge_log_1 ORDER BY x SETTINGS optimize_move_to_prewhere= {{prew}}; SELECT 'SELECT * FROM merge(currentDatabase(), 02763_merge_log_1)'; -SELECT * FROM merge(currentDatabase(), '02763_merge_log_1') ORDER BY x SETTINGS optimize_move_to_prewhere= {{prew}};; +SELECT * FROM merge(currentDatabase(), '02763_merge_log_1') ORDER BY x SETTINGS optimize_move_to_prewhere= {{prew}}; SELECT 'SELECT * FROM merge(currentDatabase(), 02763_merge_log)'; -SELECT * FROM merge(currentDatabase(), '02763_merge_log') ORDER BY x SETTINGS optimize_move_to_prewhere= {{prew}};; +SELECT * FROM merge(currentDatabase(), '02763_merge_log') ORDER BY x SETTINGS optimize_move_to_prewhere= {{prew}}; SELECT 'SELECT * FROM merge(currentDatabase(), 02763_merge_log) WHERE x>2'; -SELECT * FROM merge(currentDatabase(), '02763_merge_log') WHERE x>2 ORDER BY x SETTINGS optimize_move_to_prewhere= {{prew}};; +SELECT * FROM merge(currentDatabase(), '02763_merge_log') WHERE x>2 ORDER BY x SETTINGS optimize_move_to_prewhere= {{prew}}; CREATE ROW POLICY 02763_filter_2 ON 02763_merge_merge_1 USING x=4 AS permissive TO ALL; SELECT 'SELECT * FROM 02763_merge_merge_1'; -SELECT * FROM 02763_merge_merge_1 ORDER BY x SETTINGS optimize_move_to_prewhere= {{prew}};; +SELECT * FROM 02763_merge_merge_1 ORDER BY x SETTINGS optimize_move_to_prewhere= {{prew}}; SELECT 'SELECT * FROM merge(currentDatabase(), 02763_merge_merge_1'; -SELECT * FROM merge(currentDatabase(), '02763_merge_merge_1') ORDER BY x SETTINGS optimize_move_to_prewhere= {{prew}};; +SELECT * FROM merge(currentDatabase(), '02763_merge_merge_1') ORDER BY x SETTINGS optimize_move_to_prewhere= {{prew}}; SELECT 'SELECT * FROM merge(currentDatabase(), 02763_merge_merge)'; -SELECT * FROM merge(currentDatabase(), '02763_merge_merge') ORDER BY x SETTINGS optimize_move_to_prewhere= {{prew}};; +SELECT * FROM merge(currentDatabase(), '02763_merge_merge') ORDER BY x SETTINGS optimize_move_to_prewhere= {{prew}}; SELECT 'SELECT * FROM merge(currentDatabase(), 02763_merge_merge) WHERE x>2'; -SELECT * FROM merge(currentDatabase(), '02763_merge_merge') WHERE x>2 ORDER BY x SETTINGS optimize_move_to_prewhere= {{prew}};; +SELECT * FROM merge(currentDatabase(), '02763_merge_merge') WHERE x>2 ORDER BY x SETTINGS optimize_move_to_prewhere= {{prew}}; SELECT 'SELECT * FROM merge(currentDatabase(), 02763_merge)'; -SELECT * FROM merge(currentDatabase(), '02763_merge') ORDER BY x SETTINGS optimize_move_to_prewhere= {{prew}};; -SELECT 'SELECT * FROM merge(currentDatabase(), 02763_merge) WHEER x>2'; -SELECT * FROM merge(currentDatabase(), '02763_merge') WHERE x>2 ORDER BY x SETTINGS optimize_move_to_prewhere= {{prew}};; +SELECT * FROM merge(currentDatabase(), '02763_merge') ORDER BY x SETTINGS optimize_move_to_prewhere= {{prew}}; +SELECT 'SELECT * FROM merge(currentDatabase(), 02763_merge) WHERE x>2'; +SELECT * FROM merge(currentDatabase(), '02763_merge') WHERE x>2 ORDER BY x SETTINGS optimize_move_to_prewhere= {{prew}}; + +SELECT 'aaa', x*2 as x_2 FROM merge(currentDatabase(), '02763_merge') WHERE x>2 ORDER BY x_2 SETTINGS optimize_move_to_prewhere= {{prew}}; +SELECT x FROM (SELECT * FROM merge(currentDatabase(), '02763_merge') WHERE x IN (3,4)) ORDER BY x SETTINGS optimize_move_to_prewhere= {{prew}}; DROP ROW POLICY 02763_filter_1 ON 02763_merge_log_1; DROP ROW POLICY 02763_filter_2 ON 02763_merge_merge_1; From 1de1fbe201d728b19a6329d460ad4f72f11cf6d2 Mon Sep 17 00:00:00 2001 From: Ilya Golshtein Date: Wed, 14 Jun 2023 21:31:17 +0000 Subject: [PATCH 077/123] merge_row_policy: cleanup, remove some debug output --- src/Interpreters/InterpreterSelectQuery.cpp | 41 +--- .../MergeTree/MergeTreeSequentialSource.cpp | 2 +- src/Storages/StorageMerge.cpp | 202 ++++-------------- 3 files changed, 43 insertions(+), 202 deletions(-) diff --git a/src/Interpreters/InterpreterSelectQuery.cpp b/src/Interpreters/InterpreterSelectQuery.cpp index 9716957bfe29..a268faa5cd6f 100644 --- a/src/Interpreters/InterpreterSelectQuery.cpp +++ b/src/Interpreters/InterpreterSelectQuery.cpp @@ -93,14 +93,12 @@ #include #include -#include namespace ProfileEvents { extern const Event SelectQueriesWithSubqueries; extern const Event QueriesWithSubqueries; } -#pragma GCC diagnostic ignored "-Wold-style-cast" namespace DB { @@ -134,9 +132,6 @@ FilterDAGInfoPtr generateFilterActions( Names & prerequisite_columns, PreparedSetsPtr prepared_sets) { - LOG_TRACE(&Poco::Logger::get("generateFilterActions"), "top of"); - - auto filter_info = std::make_shared(); const auto & db_name = table_id.getDatabaseName(); @@ -551,15 +546,7 @@ InterpreterSelectQuery::InterpreterSelectQuery( std::shared_ptr table_join = joined_tables.makeTableJoin(query); if (storage) - { - LOG_TRACE(&Poco::Logger::get("InterpretSelectQuery ctor"), " {}, table name: {}, calling getRowPolicyFilter", (void*)this, table_id.getTableName()); row_policy_filter = context->getRowPolicyFilter(table_id.getDatabaseName(), table_id.getTableName(), RowPolicyFilterType::SELECT_FILTER); - } - else - { - LOG_TRACE(&Poco::Logger::get("InterpretSelectQuery ctor"), " {}, no storage", (void*)this); - } - StorageView * view = nullptr; if (storage) @@ -863,8 +850,6 @@ InterpreterSelectQuery::InterpreterSelectQuery( /// Add prewhere actions with alias columns and record needed columns from storage. if (storage) { - LOG_TRACE(log, "calling addPrewhereAliasActions"); - addPrewhereAliasActions(); analysis_result.required_columns = required_columns; } @@ -960,8 +945,6 @@ Block InterpreterSelectQuery::getSampleBlockImpl() analysis_result = ExpressionAnalysisResult( *query_analyzer, metadata_snapshot, first_stage, second_stage, options.only_analyze, filter_info, additional_filter_info, source_header); - LOG_TRACE(log, "getSampleBlockImpl {} : source_header after ExpressionAnalysisResult {}", (void*) this, source_header.dumpStructure()); - if (options.to_stage == QueryProcessingStage::Enum::FetchColumns) { @@ -971,12 +954,8 @@ Block InterpreterSelectQuery::getSampleBlockImpl() { header = analysis_result.prewhere_info->prewhere_actions->updateHeader(header); if (analysis_result.prewhere_info->remove_prewhere_column) - { - LOG_TRACE(log, "getSampleBlockImpl {} : erasing column {}", (void*) this, analysis_result.prewhere_info->prewhere_column_name); header.erase(analysis_result.prewhere_info->prewhere_column_name); - } } - LOG_TRACE(log, "getSampleBlockImpl {} : returning header", (void*) this); return header; } @@ -1462,7 +1441,7 @@ void InterpreterSelectQuery::executeImpl(QueryPlan & query_plan, std::optional

{}", (void*) this, QueryProcessingStage::toString(from_stage), QueryProcessingStage::toString(options.to_stage)); + LOG_TRACE(log, "{} -> {}", QueryProcessingStage::toString(from_stage), QueryProcessingStage::toString(options.to_stage)); } if (query_info.projection && query_info.projection->input_order_info && query_info.input_order_info) @@ -1528,16 +1507,12 @@ void InterpreterSelectQuery::executeImpl(QueryPlan & query_plan, std::optional

column_name, query_plan.getCurrentDataStream().header.dumpStructure()); - auto row_level_security_step = std::make_unique( query_plan.getCurrentDataStream(), expressions.filter_info->actions, expressions.filter_info->column_name, expressions.filter_info->do_remove_column); - row_level_security_step->setStepDescription("Row-level security filter"); query_plan.addStep(std::move(row_level_security_step)); } @@ -2075,16 +2050,11 @@ void InterpreterSelectQuery::addPrewhereAliasActions() auto & expressions = analysis_result; if (expressions.filter_info) { - LOG_TRACE(&Poco::Logger::get("addPrewhereAliasActions"), " {}, expressions.filter_info", (void*)this); - if (!expressions.prewhere_info) { const bool does_storage_support_prewhere = !input_pipe && storage && storage->supportsPrewhere(); - LOG_TRACE(&Poco::Logger::get("addPrewhereAliasActions"), " {}, expressions.filter_info 1 - does_storage_support_prewhere {} shouldMoveToPrewhere() {}", - (void*)this, does_storage_support_prewhere, shouldMoveToPrewhere()); if (does_storage_support_prewhere && shouldMoveToPrewhere()) { - LOG_TRACE(&Poco::Logger::get("addPrewhereAliasActions"), " {}, expressions.filter_info 1.5", (void*)this); /// Execute row level filter in prewhere as a part of "move to prewhere" optimization. expressions.prewhere_info = std::make_shared( std::move(expressions.filter_info->actions), @@ -2097,21 +2067,12 @@ void InterpreterSelectQuery::addPrewhereAliasActions() } else { - LOG_TRACE(&Poco::Logger::get("addPrewhereAliasActions"), " {}, expressions.filter_info 2", (void*)this); /// Add row level security actions to prewhere. expressions.prewhere_info->row_level_filter = std::move(expressions.filter_info->actions); expressions.prewhere_info->row_level_column_name = std::move(expressions.filter_info->column_name); expressions.prewhere_info->row_level_filter->projectInput(false); expressions.filter_info = nullptr; } - if (expressions.prewhere_info) - { - LOG_TRACE(&Poco::Logger::get("addPrewhereAliasActions"), " {} dump: {}", (void*)this, expressions.prewhere_info->dump()); - } - else - { - LOG_TRACE(&Poco::Logger::get("addPrewhereAliasActions"), " no prewhere_info"); - } } auto & prewhere_info = analysis_result.prewhere_info; diff --git a/src/Storages/MergeTree/MergeTreeSequentialSource.cpp b/src/Storages/MergeTree/MergeTreeSequentialSource.cpp index 16b825d1d7fb..8a9faa5cee49 100644 --- a/src/Storages/MergeTree/MergeTreeSequentialSource.cpp +++ b/src/Storages/MergeTree/MergeTreeSequentialSource.cpp @@ -306,7 +306,7 @@ class ReadFromPart final : public ISourceStep const auto & primary_key = storage_snapshot->metadata->getPrimaryKey(); const Names & primary_key_column_names = primary_key.column_names; KeyCondition key_condition(filter, context, primary_key_column_names, primary_key.expression, NameSet{}); - LOG_DEBUG(log, "ReadFromPart (MergeTreeSequentialSource) Key condition: {}", key_condition.toString()); + LOG_DEBUG(log, "Key condition: {}", key_condition.toString()); if (!key_condition.alwaysFalse()) mark_ranges = MergeTreeDataSelectExecutor::markRangesFromPKRange( diff --git a/src/Storages/StorageMerge.cpp b/src/Storages/StorageMerge.cpp index ae3dbcbc9539..77880717b277 100644 --- a/src/Storages/StorageMerge.cpp +++ b/src/Storages/StorageMerge.cpp @@ -47,10 +47,6 @@ #include -#pragma GCC diagnostic ignored "-Wunused-parameter" -#pragma GCC diagnostic ignored "-Wunused-but-set-variable" - - namespace { @@ -270,7 +266,6 @@ QueryProcessingStage::Enum StorageMerge::getQueryProcessingStage( /// (see removeJoin()) /// /// And for this we need to return FetchColumns. - LOG_TRACE(&Poco::Logger::get("StorageMerge::getQueryProcessingStage"), "to_stage {}", to_stage); if (const auto * select = query_info.query->as(); select && hasJoin(*select)) return QueryProcessingStage::FetchColumns; @@ -295,7 +290,6 @@ QueryProcessingStage::Enum StorageMerge::getQueryProcessingStage( stage_in_source_tables, table->getQueryProcessingStage(local_context, to_stage, table->getStorageSnapshot(table->getInMemoryMetadataPtr(), local_context), query_info)); - LOG_TRACE(&Poco::Logger::get("StorageMerge::getQueryProcessingStage"), "stage_in_source_tables {}", stage_in_source_tables); } iterator->next(); @@ -322,9 +316,6 @@ void StorageMerge::read( auto modified_context = Context::createCopy(local_context); // modified_context->setSetting("optimize_move_to_prewhere", false); - LOG_TRACE(&Poco::Logger::get("StorageMerge::read"), "processed_stage {}", QueryProcessingStage::toString(processed_stage)); - - bool has_database_virtual_column = false; bool has_table_virtual_column = false; Names real_column_names; @@ -339,7 +330,6 @@ void StorageMerge::read( else { real_column_names.push_back(column_name); - LOG_TRACE(&Poco::Logger::get("StorageMerge::read"), "column_name {}", column_name); } } @@ -494,7 +484,6 @@ void ReadFromMerge::initializePipeline(QueryPipelineBuilder & pipeline, const Bu bool with_aliases = common_processed_stage == QueryProcessingStage::FetchColumns && !storage_columns.getAliases().empty(); if (with_aliases) { - LOG_TRACE(&Poco::Logger::get("ReadFromMerge::initializePipeline"), "with_aliases"); ASTPtr required_columns_expr_list = std::make_shared(); ASTPtr column_expr; @@ -534,8 +523,6 @@ void ReadFromMerge::initializePipeline(QueryPipelineBuilder & pipeline, const Bu } } - LOG_TRACE(&Poco::Logger::get("ReadFromMerge::initializePipeline"), "table name: {}", storage->getStorageID().getTableName()); - auto source_pipeline = createSources( nested_storage_snaphsot, modified_query_info, @@ -668,8 +655,6 @@ QueryPipelineBuilderPtr ReadFromMerge::createSources( storage_snapshot, modified_query_info); -#pragma GCC diagnostic ignored "-Wunreachable-code" -#pragma GCC diagnostic ignored "-Wunused-variable" if (processed_stage <= storage_stage || (allow_experimental_analyzer && processed_stage == QueryProcessingStage::FetchColumns)) { /// If there are only virtual columns in query, you must request at least one other column. @@ -679,12 +664,9 @@ QueryPipelineBuilderPtr ReadFromMerge::createSources( QueryPlan & plan = child_plans.emplace_back(); StorageView * view = dynamic_cast(storage.get()); - bool direct_read = false; if (!view || allow_experimental_analyzer) - // if (!view || allow_experimental_analyzer) { - LOG_TRACE(&Poco::Logger::get("ReadFromMerge::createSources"), "direct storage->read"); storage->read(plan, real_column_names, storage_snapshot, @@ -693,28 +675,45 @@ QueryPipelineBuilderPtr ReadFromMerge::createSources( processed_stage, max_block_size, UInt32(streams_num)); - direct_read = true; + + + if (!plan.isInitialized()) + return {}; + + if (auto * source_step_with_filter = dynamic_cast((plan.getRootNode()->step.get()))) + { + auto row_policy_filter = modified_context->getRowPolicyFilter(database_name, table_name, RowPolicyFilterType::SELECT_FILTER); + + if (row_policy_filter) + { + ASTPtr expr = row_policy_filter->expression; + + auto syntax_result = TreeRewriter(modified_context).analyze(expr, header.getNamesAndTypesList()); + auto expression_analyzer = ExpressionAnalyzer{row_policy_filter->expression, syntax_result, modified_context}; + + auto filter_dag_ptr = expression_analyzer.getActionsDAG(true, false); + auto filter_actions = std::make_shared(filter_dag_ptr, ExpressionActionsSettings::fromContext(modified_context, CompileExpressions::yes)); + auto required_columns = filter_actions->getRequiredColumns(); + + LOG_TRACE(&Poco::Logger::get("ReadFromMerge::convertinfSourceStream"), "filter_actions_dag: {},<> {}, <> {}", + filter_actions->getActionsDAG().dumpNames(), filter_actions->getActionsDAG().dumpDAG(), filter_actions->getSampleBlock().dumpStructure()); + + auto fa_actions_columns_sorted = filter_actions->getSampleBlock().getNames(); + std::sort(fa_actions_columns_sorted.begin(), fa_actions_columns_sorted.end()); + + Names required_columns_sorted = required_columns; + std::sort(required_columns_sorted.begin(), required_columns_sorted.end()); + + Names filter_columns; + + std::set_difference(fa_actions_columns_sorted.begin(), fa_actions_columns_sorted.end(), + required_columns.begin(), required_columns.end(), + std::inserter(filter_columns, filter_columns.begin())); + + source_step_with_filter->addFilter(filter_dag_ptr, filter_columns.front()); + } + } } - // else if (!view) - // { - // /// For view storage, we need to rewrite the `modified_query_info.view_query` to optimize read. - // /// The most intuitive way is to use InterpreterSelectQuery. - - // /// Intercept the settings - // modified_context->setSetting("max_threads", streams_num); - // modified_context->setSetting("max_streams_to_max_threads_ratio", 1); - // modified_context->setSetting("max_block_size", max_block_size); - - // LOG_TRACE(&Poco::Logger::get("ReadFromMerge::createSources"), "creating InterpreterSelectQuery 1.0"); - // InterpreterSelectQuery interpreter(modified_query_info.query, - // modified_context, - // storage, - // storage->getInMemoryMetadataPtr(), // view->getInMemoryMetadataPtr(), - // // SelectQueryOptions(/* processed_stage*/)); - // SelectQueryOptions(processed_stage)); - // // SelectQueryOptions(QueryProcessingStage::WithMergeableState)); - // interpreter.buildQueryPlan(plan); - // } else { /// For view storage, we need to rewrite the `modified_query_info.view_query` to optimize read. @@ -725,90 +724,25 @@ QueryPipelineBuilderPtr ReadFromMerge::createSources( modified_context->setSetting("max_streams_to_max_threads_ratio", 1); modified_context->setSetting("max_block_size", max_block_size); - LOG_TRACE(&Poco::Logger::get("ReadFromMerge::createSources"), "creating InterpreterSelectQuery 1.5"); InterpreterSelectQuery interpreter(modified_query_info.query, modified_context, storage, view->getInMemoryMetadataPtr(), SelectQueryOptions(processed_stage)); interpreter.buildQueryPlan(plan); - } - if (!plan.isInitialized()) - return {}; + if (!plan.isInitialized()) + return {}; + } if (auto * read_from_merge_tree = typeid_cast(plan.getRootNode()->step.get())) { size_t filters_dags_size = filter_dags.size(); - LOG_TRACE(&Poco::Logger::get("ReadFromMerge::createSources"), "ReadFromMergeTree detected, DAG size {}", filters_dags_size); for (size_t i = 0; i < filters_dags_size; ++i) { - LOG_TRACE(&Poco::Logger::get("ReadFromMerge::createSources"), "adding filter {}", filter_dags[i]->dumpDAG()); read_from_merge_tree->addFilter(filter_dags[i], filter_nodes.nodes[i]); } } - if (auto * source_step_with_filter = dynamic_cast((plan.getRootNode()->step.get()))) - { - auto row_policy_filter = modified_context->getRowPolicyFilter(database_name, table_name, RowPolicyFilterType::SELECT_FILTER); - - if (row_policy_filter) - { - // row_policy_filter->expression - // auto pipe_columns = builder.getHeader().getNamesAndTypesList(); - ASTPtr expr = row_policy_filter->expression; - - // auto * select_ast = expr /* query_ast */ ->as(); - // assert(select_ast); - - // select_ast->setExpression(ASTSelectQuery::Expression::SELECT, std::make_shared()); - // auto expr_list = select_ast->select(); - // expr_list->children.push_back(expr); - // String filter_column_name = expr_list->children.at(0)->getColumnName(); - // LOG_TRACE(&Poco::Logger::get("ReadFromMerge::convertinfSourceStream"), "filter_column_name: {} ", filter_column_name); - - auto syntax_result = TreeRewriter(modified_context).analyze(expr, header/*builder->getHeader().*/.getNamesAndTypesList() /* pipe_columns*/); - // auto syntax_result = TreeRewriter(local_context).analyze(expr, NamesAndTypesList()); - auto expression_analyzer = ExpressionAnalyzer{row_policy_filter->expression, syntax_result, modified_context}; - - auto filter_dag_ptr = expression_analyzer.getActionsDAG(true, false); - - - auto filter_actions = std::make_shared(filter_dag_ptr, ExpressionActionsSettings::fromContext(modified_context, CompileExpressions::yes)); - auto required_columns = filter_actions->getRequiredColumns(); - - for (const auto & req_col : required_columns) - { - LOG_TRACE(&Poco::Logger::get("ReadFromMerge::convertinfSourceStream"), "req_col: {}", req_col); - } - - - LOG_TRACE(&Poco::Logger::get("ReadFromMerge::convertinfSourceStream"), "filter_actions_dag: {},<> {}, <> {}", - filter_actions->getActionsDAG().dumpNames(), filter_actions->getActionsDAG().dumpDAG(), filter_actions->getSampleBlock().dumpStructure()); - - - auto fa_actions_columns_sorted = filter_actions->getSampleBlock().getNames(); - std::sort(fa_actions_columns_sorted.begin(), fa_actions_columns_sorted.end()); - - Names required_columns_sorted = required_columns; - std::sort(required_columns_sorted.begin(), required_columns_sorted.end()); - - Names filter_columns; - - - std::set_difference(fa_actions_columns_sorted.begin(), fa_actions_columns_sorted.end(), - required_columns.begin(), required_columns.end(), - std::inserter(filter_columns, filter_columns.begin())); - - - LOG_TRACE(&Poco::Logger::get("ReadFromMerge::createSources"), "SourceStepWithFilter detected"); - auto found_column = filter_dag_ptr->tryFindInOutputs(filter_columns.front()); - assert(found_column); - - // LOG_TRACE(&Poco::Logger::get("ReadFromMerge::createSources"), "found column {}", found_column->dumpDAG()); - - source_step_with_filter->addFilter(/* filter_actions */ filter_dag_ptr, filter_columns.front()); - } - } builder = plan.buildQueryPipeline( QueryPlanOptimizationSettings::fromContext(modified_context), BuildQueryPipelineSettings::fromContext(modified_context)); @@ -834,7 +768,6 @@ QueryPipelineBuilderPtr ReadFromMerge::createSources( { modified_select.replaceDatabaseAndTable(database_name, table_name); /// TODO: Find a way to support projections for StorageMerge - LOG_TRACE(&Poco::Logger::get("ReadFromMerge::createSources"), "creating InterpreterSelectQuery 2"); InterpreterSelectQuery interpreter{modified_query_info.query, modified_context, SelectQueryOptions(processed_stage).ignoreProjections()}; @@ -1114,16 +1047,6 @@ void ReadFromMerge::convertingSourceStream( convert_actions_match_columns_mode = ActionsDAG::MatchColumnsMode::Position; - for (const auto & column_with_type_and_name : builder.getHeader().getColumnsWithTypeAndName()) - { - LOG_TRACE(&Poco::Logger::get("ReadFromMerge::convertinfSourceStream"), "column name: {} (builder.getHeader().getColumnsWithTypeAndName())", column_with_type_and_name.name); - } - - for (const auto & column_with_type_and_name : header.getColumnsWithTypeAndName()) - { - LOG_TRACE(&Poco::Logger::get("ReadFromMerge::convertingSourceStream"), "column name: {} (header.getColumnsWithTypeAndName())", column_with_type_and_name.name); - } - auto convert_actions_dag = ActionsDAG::makeConvertingActions(builder.getHeader().getColumnsWithTypeAndName(), header.getColumnsWithTypeAndName(), convert_actions_match_columns_mode); @@ -1147,36 +1070,14 @@ void ReadFromMerge::convertingSourceStream( if (row_policy_filter) { - // row_policy_filter->expression - // auto pipe_columns = builder.getHeader().getNamesAndTypesList(); - - ASTPtr expr = row_policy_filter->expression; - // auto * select_ast = expr /* query_ast */ ->as(); - // assert(select_ast); - - // select_ast->setExpression(ASTSelectQuery::Expression::SELECT, std::make_shared()); - // auto expr_list = select_ast->select(); - // expr_list->children.push_back(expr); - // String filter_column_name = expr_list->children.at(0)->getColumnName(); - // LOG_TRACE(&Poco::Logger::get("ReadFromMerge::convertinfSourceStream"), "filter_column_name: {} ", filter_column_name); - auto syntax_result = TreeRewriter(local_context).analyze(expr, pipe_columns); - // auto syntax_result = TreeRewriter(local_context).analyze(expr, NamesAndTypesList()); auto expression_analyzer = ExpressionAnalyzer{row_policy_filter->expression, syntax_result, local_context}; auto actions_dag = expression_analyzer.getActionsDAG(true, false); - LOG_TRACE(&Poco::Logger::get("ReadFromMerge::convertinfSourceStream"), "actions_dag: {},<> {}", actions_dag->dumpNames(), actions_dag->dumpDAG()); - - auto filter_actions = std::make_shared(actions_dag, ExpressionActionsSettings::fromContext(local_context, CompileExpressions::yes)); auto required_columns = filter_actions->getRequiredColumns(); - for (const auto & req_col : required_columns) - { - LOG_TRACE(&Poco::Logger::get("ReadFromMerge::convertinfSourceStream"), "req_col: {}", req_col); - } - LOG_TRACE(&Poco::Logger::get("ReadFromMerge::convertinfSourceStream"), "filter_actions_dag: {},<> {}, <> {}", filter_actions->getActionsDAG().dumpNames(), filter_actions->getActionsDAG().dumpDAG(), filter_actions->getSampleBlock().dumpStructure()); @@ -1195,32 +1096,11 @@ void ReadFromMerge::convertingSourceStream( required_columns.begin(), required_columns.end(), std::inserter(filter_columns, filter_columns.begin())); - for (const auto & filter_column : filter_columns) - { - LOG_TRACE(&Poco::Logger::get("ReadFromMerge::convertinfSourceStream"), "filter_column: {}", filter_column); - } - - // Block block; - // block = filter_actions->getActionsDAG().updateHeader(std::move(block)); - // LOG_TRACE(&Poco::Logger::get("ReadFromMerge::convertingSourceStream"), "block from updateHeader {}", block.dumpStructure()); - builder.addSimpleTransform([&](const Block & stream_header) { - LOG_TRACE(&Poco::Logger::get("ReadFromMerge::convertinfSourceStream"), "stream_header {}", stream_header.dumpStructure()); return std::make_shared(stream_header, filter_actions, filter_columns.front(), true /* remove fake column */); }); - - - // auto row_level_filter_step = std::make_unique( - // query_plan.getCurrentDataStream(), - // expressions.prewhere_info->row_level_filter, - // expressions.prewhere_info->row_level_column_name, - // true); - - // row_level_filter_step->setStepDescription("Row-level security filter (PREWHERE)"); - // query_plan.addStep(std::move(row_level_filter_step)); } - } } From 8007f7f2089e20e03367f7178632e5c479359067 Mon Sep 17 00:00:00 2001 From: Ilya Golshtein Date: Wed, 14 Jun 2023 21:36:20 +0000 Subject: [PATCH 078/123] merge_row_policy: further cleanup --- src/Storages/StorageMerge.cpp | 58 +++++++++++++---------------------- 1 file changed, 22 insertions(+), 36 deletions(-) diff --git a/src/Storages/StorageMerge.cpp b/src/Storages/StorageMerge.cpp index 77880717b277..fa22b14e739a 100644 --- a/src/Storages/StorageMerge.cpp +++ b/src/Storages/StorageMerge.cpp @@ -297,7 +297,6 @@ QueryProcessingStage::Enum StorageMerge::getQueryProcessingStage( } return selected_table_size == 1 ? stage_in_source_tables : std::min(stage_in_source_tables, QueryProcessingStage::WithMergeableState); - // return QueryProcessingStage::Complete; } void StorageMerge::read( @@ -314,7 +313,6 @@ void StorageMerge::read( * since there is no certainty that it works when one of table is MergeTree and other is not. */ auto modified_context = Context::createCopy(local_context); - // modified_context->setSetting("optimize_move_to_prewhere", false); bool has_database_virtual_column = false; bool has_table_virtual_column = false; @@ -328,9 +326,7 @@ void StorageMerge::read( else if (column_name == "_table" && isVirtualColumn(column_name, storage_snapshot->metadata)) has_table_virtual_column = true; else - { real_column_names.push_back(column_name); - } } StorageListWithLocks selected_tables @@ -359,7 +355,6 @@ void StorageMerge::read( query_plan.addInterpreterContext(modified_context); /// What will be result structure depending on query processed stage in source tables? - // Block common_header = getHeaderForProcessingStage(column_names, storage_snapshot, query_info, local_context, QueryProcessingStage::Complete /* processed_stage */); Block common_header = getHeaderForProcessingStage(column_names, storage_snapshot, query_info, local_context, processed_stage); auto step = std::make_unique( @@ -664,7 +659,6 @@ QueryPipelineBuilderPtr ReadFromMerge::createSources( QueryPlan & plan = child_plans.emplace_back(); StorageView * view = dynamic_cast(storage.get()); - if (!view || allow_experimental_analyzer) { storage->read(plan, @@ -1046,7 +1040,6 @@ void ReadFromMerge::convertingSourceStream( if (local_context->getSettingsRef().allow_experimental_analyzer && processed_stage != QueryProcessingStage::FetchColumns) convert_actions_match_columns_mode = ActionsDAG::MatchColumnsMode::Position; - auto convert_actions_dag = ActionsDAG::makeConvertingActions(builder.getHeader().getColumnsWithTypeAndName(), header.getColumnsWithTypeAndName(), convert_actions_match_columns_mode); @@ -1060,47 +1053,40 @@ void ReadFromMerge::convertingSourceStream( }); - bool explicit_row_policy_filter_needed = true; + auto row_policy_filter = local_context->getRowPolicyFilter(database_name, table_name, RowPolicyFilterType::SELECT_FILTER); - if (explicit_row_policy_filter_needed) + if (row_policy_filter) { + ASTPtr expr = row_policy_filter->expression; - auto row_policy_filter = local_context->getRowPolicyFilter(database_name, table_name, RowPolicyFilterType::SELECT_FILTER); - - if (row_policy_filter) - { - - ASTPtr expr = row_policy_filter->expression; - - auto syntax_result = TreeRewriter(local_context).analyze(expr, pipe_columns); - auto expression_analyzer = ExpressionAnalyzer{row_policy_filter->expression, syntax_result, local_context}; + auto syntax_result = TreeRewriter(local_context).analyze(expr, pipe_columns); + auto expression_analyzer = ExpressionAnalyzer{row_policy_filter->expression, syntax_result, local_context}; - auto actions_dag = expression_analyzer.getActionsDAG(true, false); - auto filter_actions = std::make_shared(actions_dag, ExpressionActionsSettings::fromContext(local_context, CompileExpressions::yes)); - auto required_columns = filter_actions->getRequiredColumns(); + auto actions_dag = expression_analyzer.getActionsDAG(true, false); + auto filter_actions = std::make_shared(actions_dag, ExpressionActionsSettings::fromContext(local_context, CompileExpressions::yes)); + auto required_columns = filter_actions->getRequiredColumns(); - LOG_TRACE(&Poco::Logger::get("ReadFromMerge::convertinfSourceStream"), "filter_actions_dag: {},<> {}, <> {}", - filter_actions->getActionsDAG().dumpNames(), filter_actions->getActionsDAG().dumpDAG(), filter_actions->getSampleBlock().dumpStructure()); + LOG_TRACE(&Poco::Logger::get("ReadFromMerge::convertinfSourceStream"), "filter_actions_dag: {},<> {}, <> {}", + filter_actions->getActionsDAG().dumpNames(), filter_actions->getActionsDAG().dumpDAG(), filter_actions->getSampleBlock().dumpStructure()); - auto fa_actions_columns_sorted = filter_actions->getSampleBlock().getNames(); - std::sort(fa_actions_columns_sorted.begin(), fa_actions_columns_sorted.end()); + auto fa_actions_columns_sorted = filter_actions->getSampleBlock().getNames(); + std::sort(fa_actions_columns_sorted.begin(), fa_actions_columns_sorted.end()); - Names required_columns_sorted = required_columns; - std::sort(required_columns_sorted.begin(), required_columns_sorted.end()); + Names required_columns_sorted = required_columns; + std::sort(required_columns_sorted.begin(), required_columns_sorted.end()); - Names filter_columns; + Names filter_columns; - std::set_difference(fa_actions_columns_sorted.begin(), fa_actions_columns_sorted.end(), - required_columns.begin(), required_columns.end(), - std::inserter(filter_columns, filter_columns.begin())); + std::set_difference(fa_actions_columns_sorted.begin(), fa_actions_columns_sorted.end(), + required_columns.begin(), required_columns.end(), + std::inserter(filter_columns, filter_columns.begin())); - builder.addSimpleTransform([&](const Block & stream_header) - { - return std::make_shared(stream_header, filter_actions, filter_columns.front(), true /* remove fake column */); - }); - } + builder.addSimpleTransform([&](const Block & stream_header) + { + return std::make_shared(stream_header, filter_actions, filter_columns.front(), true /* remove fake column */); + }); } } From b5f39b782c91cdbd8b7787a9a3841a051b746daf Mon Sep 17 00:00:00 2001 From: Ilya Golshtein Date: Sun, 25 Jun 2023 21:22:58 +0000 Subject: [PATCH 079/123] merge_row_policy: more tests --- .../02763_row_policy_storage_merge.reference | 278 ++++++++++-------- .../02763_row_policy_storage_merge.sql.j2 | 44 ++- 2 files changed, 194 insertions(+), 128 deletions(-) diff --git a/tests/queries/0_stateless/02763_row_policy_storage_merge.reference b/tests/queries/0_stateless/02763_row_policy_storage_merge.reference index 9dcc5f449ab1..444513c6c201 100644 --- a/tests/queries/0_stateless/02763_row_policy_storage_merge.reference +++ b/tests/queries/0_stateless/02763_row_policy_storage_merge.reference @@ -1,138 +1,180 @@ -1 -1 -1 -1 -2 -2 -2 -2 -3 -3 -3 -3 -4 -4 -4 -4 -1 -2 -3 -4 +1 11 +1 11 +1 11 +1 11 +2 12 +2 12 +2 12 +2 12 +3 13 +3 13 +3 13 +3 13 +4 14 +4 14 +4 14 +4 14 +1 11 +2 12 +3 13 +4 14 SETTINGS optimize_move_to_prewhere= 0 SELECT * FROM 02763_merge_log_1 -3 +3 13 SELECT * FROM merge(currentDatabase(), 02763_merge_log_1) -3 +3 13 SELECT * FROM merge(currentDatabase(), 02763_merge_log) -1 -2 -3 -3 -4 +1 11 +2 12 +3 13 +3 13 +4 14 SELECT * FROM merge(currentDatabase(), 02763_merge_log) WHERE x>2 -3 -3 -4 +3 13 +3 13 +4 14 SELECT * FROM 02763_merge_merge_1 -4 -SELECT * FROM merge(currentDatabase(), 02763_merge_merge_1 -4 +4 14 +SELECT * FROM merge(currentDatabase(), 02763_merge_merge_1) +4 14 SELECT * FROM merge(currentDatabase(), 02763_merge_merge) -1 -2 -3 -4 -4 +1 11 +2 12 +3 13 +4 14 +4 14 SELECT * FROM merge(currentDatabase(), 02763_merge_merge) WHERE x>2 -3 -4 -4 +3 13 +4 14 +4 14 SELECT * FROM merge(currentDatabase(), 02763_merge) -1 -1 -2 -2 -3 -3 -3 -4 -4 -4 +1 11 +1 11 +2 12 +2 12 +3 13 +3 13 +3 13 +4 14 +4 14 +4 14 SELECT * FROM merge(currentDatabase(), 02763_merge) WHERE x>2 -3 -3 -3 -4 -4 -4 -aaa 6 -aaa 6 -aaa 6 -aaa 8 -aaa 8 -aaa 8 -3 -3 -3 -4 -4 -4 +3 13 +3 13 +3 13 +4 14 +4 14 +4 14 +aaa 6 39 +aaa 6 39 +aaa 6 39 +aaa 8 42 +aaa 8 42 +aaa 8 42 +3 +3 +3 +4 +4 +4 +SELECT * FROM merge(...) LEFT JOIN merge(...) +3 13 13 +3 13 13 +4 14 14 +4 14 14 +SELECT * FROM merge(...) UNION ALL SELECT * FROM merge(...) +1 11 +1 11 +2 12 +2 12 +3 13 +3 13 +3 13 +4 14 +4 14 +4 14 +SELECT x, SUM(x) FROM (SELECT * FROM merge(...) UNION ALL ...) GROUP BY x +1 22 +2 24 +3 39 +4 42 SETTINGS optimize_move_to_prewhere= 1 SELECT * FROM 02763_merge_log_1 -3 +3 13 SELECT * FROM merge(currentDatabase(), 02763_merge_log_1) -3 +3 13 SELECT * FROM merge(currentDatabase(), 02763_merge_log) -1 -2 -3 -3 -4 +1 11 +2 12 +3 13 +3 13 +4 14 SELECT * FROM merge(currentDatabase(), 02763_merge_log) WHERE x>2 -3 -3 -4 +3 13 +3 13 +4 14 SELECT * FROM 02763_merge_merge_1 -4 -SELECT * FROM merge(currentDatabase(), 02763_merge_merge_1 -4 +4 14 +SELECT * FROM merge(currentDatabase(), 02763_merge_merge_1) +4 14 SELECT * FROM merge(currentDatabase(), 02763_merge_merge) -1 -2 -3 -4 -4 +1 11 +2 12 +3 13 +4 14 +4 14 SELECT * FROM merge(currentDatabase(), 02763_merge_merge) WHERE x>2 -3 -4 -4 +3 13 +4 14 +4 14 SELECT * FROM merge(currentDatabase(), 02763_merge) -1 -1 -2 -2 -3 -3 -3 -4 -4 -4 +1 11 +1 11 +2 12 +2 12 +3 13 +3 13 +3 13 +4 14 +4 14 +4 14 SELECT * FROM merge(currentDatabase(), 02763_merge) WHERE x>2 -3 -3 -3 -4 -4 -4 -aaa 6 -aaa 6 -aaa 6 -aaa 8 -aaa 8 -aaa 8 -3 -3 -3 -4 -4 -4 +3 13 +3 13 +3 13 +4 14 +4 14 +4 14 +aaa 6 39 +aaa 6 39 +aaa 6 39 +aaa 8 42 +aaa 8 42 +aaa 8 42 +3 +3 +3 +4 +4 +4 +SELECT * FROM merge(...) LEFT JOIN merge(...) +3 13 13 +3 13 13 +4 14 14 +4 14 14 +SELECT * FROM merge(...) UNION ALL SELECT * FROM merge(...) +1 11 +1 11 +2 12 +2 12 +3 13 +3 13 +3 13 +4 14 +4 14 +4 14 +SELECT x, SUM(x) FROM (SELECT * FROM merge(...) UNION ALL ...) GROUP BY x +1 22 +2 24 +3 39 +4 42 diff --git a/tests/queries/0_stateless/02763_row_policy_storage_merge.sql.j2 b/tests/queries/0_stateless/02763_row_policy_storage_merge.sql.j2 index 33b02275d4a8..b5094f927f44 100644 --- a/tests/queries/0_stateless/02763_row_policy_storage_merge.sql.j2 +++ b/tests/queries/0_stateless/02763_row_policy_storage_merge.sql.j2 @@ -6,16 +6,16 @@ DROP ROW POLICY IF EXISTS 02763_filter_1 ON 02763_merge_log_1; DROP ROW POLICY IF EXISTS 02763_filter_2 ON 02763_merge_merge_1; -CREATE TABLE 02763_merge_log_1 (x UInt8) ENGINE = Log; -CREATE TABLE 02763_merge_log_2 (x UInt8) ENGINE = Log; +CREATE TABLE 02763_merge_log_1 (x UInt8, y UInt64) ENGINE = Log; +CREATE TABLE 02763_merge_log_2 (x UInt8, y UInt64) ENGINE = Log; -CREATE TABLE 02763_merge_merge_1 (x UInt8) ENGINE = MergeTree ORDER BY x; -CREATE TABLE 02763_merge_merge_2 (x UInt8) ENGINE = MergeTree ORDER BY x; +CREATE TABLE 02763_merge_merge_1 (x UInt8, y UInt64) ENGINE = MergeTree ORDER BY x; +CREATE TABLE 02763_merge_merge_2 (x UInt8, y UInt64) ENGINE = MergeTree ORDER BY x; -INSERT INTO 02763_merge_log_1 VALUES (1), (2), (3), (4); -INSERT INTO 02763_merge_log_2 VALUES (1), (2), (3), (4); -INSERT INTO 02763_merge_merge_1 VALUES (1), (2), (3), (4); -INSERT INTO 02763_merge_merge_2 VALUES (1), (2), (3), (4); +INSERT INTO 02763_merge_log_1 VALUES (1, 11), (2, 12), (3, 13), (4, 14); +INSERT INTO 02763_merge_log_2 VALUES (1, 11), (2, 12), (3, 13), (4, 14); +INSERT INTO 02763_merge_merge_1 VALUES (1, 11), (2, 12), (3, 13), (4, 14); +INSERT INTO 02763_merge_merge_2 VALUES (1, 11), (2, 12), (3, 13), (4, 14); SELECT * FROM merge(currentDatabase(), '02763_merge') ORDER BY x; @@ -41,7 +41,7 @@ CREATE ROW POLICY 02763_filter_2 ON 02763_merge_merge_1 USING x=4 AS permissive SELECT 'SELECT * FROM 02763_merge_merge_1'; SELECT * FROM 02763_merge_merge_1 ORDER BY x SETTINGS optimize_move_to_prewhere= {{prew}}; -SELECT 'SELECT * FROM merge(currentDatabase(), 02763_merge_merge_1'; +SELECT 'SELECT * FROM merge(currentDatabase(), 02763_merge_merge_1)'; SELECT * FROM merge(currentDatabase(), '02763_merge_merge_1') ORDER BY x SETTINGS optimize_move_to_prewhere= {{prew}}; SELECT 'SELECT * FROM merge(currentDatabase(), 02763_merge_merge)'; SELECT * FROM merge(currentDatabase(), '02763_merge_merge') ORDER BY x SETTINGS optimize_move_to_prewhere= {{prew}}; @@ -54,9 +54,33 @@ SELECT * FROM merge(currentDatabase(), '02763_merge') ORDER BY x SETTINGS optimi SELECT 'SELECT * FROM merge(currentDatabase(), 02763_merge) WHERE x>2'; SELECT * FROM merge(currentDatabase(), '02763_merge') WHERE x>2 ORDER BY x SETTINGS optimize_move_to_prewhere= {{prew}}; -SELECT 'aaa', x*2 as x_2 FROM merge(currentDatabase(), '02763_merge') WHERE x>2 ORDER BY x_2 SETTINGS optimize_move_to_prewhere= {{prew}}; +SELECT 'aaa', x*2 as x_2, y*3 as y_3 FROM merge(currentDatabase(), '02763_merge') WHERE x>2 ORDER BY x_2 SETTINGS optimize_move_to_prewhere= {{prew}}; SELECT x FROM (SELECT * FROM merge(currentDatabase(), '02763_merge') WHERE x IN (3,4)) ORDER BY x SETTINGS optimize_move_to_prewhere= {{prew}}; +SELECT 'SELECT * FROM merge(...) LEFT JOIN merge(...)'; +SELECT * FROM merge(currentDatabase(), '02763_merge.*1') as a +LEFT JOIN +merge(currentDatabase(), '02763_merge.*2') as b +USING (x) +ORDER BY x SETTINGS optimize_move_to_prewhere= {{prew}}; + +SELECT 'SELECT * FROM merge(...) UNION ALL SELECT * FROM merge(...)'; +SELECT * FROM +( +SELECT * FROM merge(currentDatabase(), '02763_merge.*1') +UNION ALL +SELECT * FROM merge(currentDatabase(), '02763_merge.*2') +) +ORDER BY x SETTINGS optimize_move_to_prewhere= {{prew}}; + +SELECT 'SELECT x, SUM(x) FROM (SELECT * FROM merge(...) UNION ALL ...) GROUP BY x'; +SELECT x, SUM(y) FROM +(SELECT * FROM merge(currentDatabase(), '02763_merge.*1') +UNION ALL +SELECT * FROM merge(currentDatabase(), '02763_merge.*2')) +GROUP BY x +ORDER BY x; + DROP ROW POLICY 02763_filter_1 ON 02763_merge_log_1; DROP ROW POLICY 02763_filter_2 ON 02763_merge_merge_1; From 265ca05457ed88366f9a746f7630f3309c507419 Mon Sep 17 00:00:00 2001 From: Ilya Golshtein Date: Tue, 11 Jul 2023 11:46:37 +0000 Subject: [PATCH 080/123] merge_row_policy: namesDifference, try to handle nonselected columns --- src/Storages/StorageMerge.cpp | 75 ++++++++++++------- .../02763_row_policy_storage_merge.reference | 54 ++++++++++++- .../02763_row_policy_storage_merge.sql.j2 | 23 +++++- 3 files changed, 118 insertions(+), 34 deletions(-) diff --git a/src/Storages/StorageMerge.cpp b/src/Storages/StorageMerge.cpp index fa22b14e739a..5be809a3593d 100644 --- a/src/Storages/StorageMerge.cpp +++ b/src/Storages/StorageMerge.cpp @@ -66,6 +66,26 @@ bool columnDefaultKindHasSameType(ColumnDefaultKind lhs, ColumnDefaultKind rhs) return false; } +std::string namesDifference(Names && outer_set, Names && inner_set) +{ + std::sort(outer_set.begin(), outer_set.end()); + + std::sort(inner_set.begin(), inner_set.end()); + + Names result; + + std::set_difference(outer_set.begin(), outer_set.end(), + inner_set.begin(), inner_set.end(), std::inserter(result, result.begin())); + + if (result.size() != 1) + { + throw Exception(ErrorCodes::LOGICAL_ERROR, + "Cannot determine row level filter"); + } + + return result.front(); +} + } namespace DB @@ -682,29 +702,22 @@ QueryPipelineBuilderPtr ReadFromMerge::createSources( { ASTPtr expr = row_policy_filter->expression; - auto syntax_result = TreeRewriter(modified_context).analyze(expr, header.getNamesAndTypesList()); + auto storage_metadata_snapshot = storage->getInMemoryMetadataPtr(); + auto storage_columns = storage_metadata_snapshot->getColumns(); + auto needed_columns = storage_columns.getAllPhysical(); // header.getNamesAndTypesList() + + auto syntax_result = TreeRewriter(modified_context).analyze(expr, needed_columns); auto expression_analyzer = ExpressionAnalyzer{row_policy_filter->expression, syntax_result, modified_context}; - auto filter_dag_ptr = expression_analyzer.getActionsDAG(true, false); - auto filter_actions = std::make_shared(filter_dag_ptr, ExpressionActionsSettings::fromContext(modified_context, CompileExpressions::yes)); + auto actions_dag = expression_analyzer.getActionsDAG(true, false); + auto filter_actions = std::make_shared(actions_dag, ExpressionActionsSettings::fromContext(modified_context, CompileExpressions::yes)); auto required_columns = filter_actions->getRequiredColumns(); - LOG_TRACE(&Poco::Logger::get("ReadFromMerge::convertinfSourceStream"), "filter_actions_dag: {},<> {}, <> {}", + LOG_TRACE(&Poco::Logger::get("ReadFromMerge::convertingSourceStream"), "filter_actions_dag: {},<> {}, <> {}", filter_actions->getActionsDAG().dumpNames(), filter_actions->getActionsDAG().dumpDAG(), filter_actions->getSampleBlock().dumpStructure()); - auto fa_actions_columns_sorted = filter_actions->getSampleBlock().getNames(); - std::sort(fa_actions_columns_sorted.begin(), fa_actions_columns_sorted.end()); - - Names required_columns_sorted = required_columns; - std::sort(required_columns_sorted.begin(), required_columns_sorted.end()); - - Names filter_columns; - - std::set_difference(fa_actions_columns_sorted.begin(), fa_actions_columns_sorted.end(), - required_columns.begin(), required_columns.end(), - std::inserter(filter_columns, filter_columns.begin())); - - source_step_with_filter->addFilter(filter_dag_ptr, filter_columns.front()); + auto filter_column_name = namesDifference(filter_actions->getSampleBlock().getNames(), filter_actions->getRequiredColumns()); + source_step_with_filter->addFilter(actions_dag, filter_column_name); } } } @@ -1059,33 +1072,39 @@ void ReadFromMerge::convertingSourceStream( { ASTPtr expr = row_policy_filter->expression; - auto syntax_result = TreeRewriter(local_context).analyze(expr, pipe_columns); + auto storage_columns = metadata_snapshot->getColumns(); + auto needed_columns = storage_columns.getAllPhysical(); // header.getNamesAndTypesList() + + + auto syntax_result = TreeRewriter(local_context).analyze(expr, needed_columns /* pipe_columns */); auto expression_analyzer = ExpressionAnalyzer{row_policy_filter->expression, syntax_result, local_context}; auto actions_dag = expression_analyzer.getActionsDAG(true, false); auto filter_actions = std::make_shared(actions_dag, ExpressionActionsSettings::fromContext(local_context, CompileExpressions::yes)); - auto required_columns = filter_actions->getRequiredColumns(); LOG_TRACE(&Poco::Logger::get("ReadFromMerge::convertinfSourceStream"), "filter_actions_dag: {},<> {}, <> {}", filter_actions->getActionsDAG().dumpNames(), filter_actions->getActionsDAG().dumpDAG(), filter_actions->getSampleBlock().dumpStructure()); - auto fa_actions_columns_sorted = filter_actions->getSampleBlock().getNames(); - std::sort(fa_actions_columns_sorted.begin(), fa_actions_columns_sorted.end()); - Names required_columns_sorted = required_columns; - std::sort(required_columns_sorted.begin(), required_columns_sorted.end()); + for (auto & colname : filter_actions->getSampleBlock().getNames()) + { + LOG_TRACE(&Poco::Logger::get("ReadFromMerge::convertinfSourceStream"), "filter_actions->getSampleBlock().getNames(): {}", colname); + } + + for (auto & colname : filter_actions->getRequiredColumns()) + { + LOG_TRACE(&Poco::Logger::get("ReadFromMerge::convertinfSourceStream"), "filter_actions->getRequiredColumns(): {}", colname); + } + - Names filter_columns; + auto filter_column_name = namesDifference(filter_actions->getSampleBlock().getNames(), filter_actions->getRequiredColumns()); - std::set_difference(fa_actions_columns_sorted.begin(), fa_actions_columns_sorted.end(), - required_columns.begin(), required_columns.end(), - std::inserter(filter_columns, filter_columns.begin())); builder.addSimpleTransform([&](const Block & stream_header) { - return std::make_shared(stream_header, filter_actions, filter_columns.front(), true /* remove fake column */); + return std::make_shared(stream_header, filter_actions, filter_column_name, true /* remove fake column */); }); } } diff --git a/tests/queries/0_stateless/02763_row_policy_storage_merge.reference b/tests/queries/0_stateless/02763_row_policy_storage_merge.reference index 444513c6c201..55890a117836 100644 --- a/tests/queries/0_stateless/02763_row_policy_storage_merge.reference +++ b/tests/queries/0_stateless/02763_row_policy_storage_merge.reference @@ -1,3 +1,9 @@ +SELECT * FROM 02763_merge_log_1 ORDER BY x +1 11 +2 12 +3 13 +4 14 +SELECT * FROM merge(currentDatabase(), 02763_merge) ORDER BY x 1 11 1 11 1 11 @@ -14,10 +20,6 @@ 4 14 4 14 4 14 -1 11 -2 12 -3 13 -4 14 SETTINGS optimize_move_to_prewhere= 0 SELECT * FROM 02763_merge_log_1 3 13 @@ -98,6 +100,28 @@ SELECT x, SUM(x) FROM (SELECT * FROM merge(...) UNION ALL ...) GROUP BY x 2 24 3 39 4 42 +1 11 0 +2 12 0 +3 13 0 +4 14 1 +4 14 1 +SELECT * FROM merge(currentDatabase(), 02763_merge_log) WHERE x>1 -- with y>12 +2 12 +3 13 +3 13 +4 14 +4 14 +SELECT * FROM merge(currentDatabase(), 02763_merge_merge) WHERE x>1 -- with y>12 +2 12 +3 13 +3 13 +4 14 +4 14 +2 12 0 +3 13 1 +3 13 1 +4 14 1 +4 14 1 SETTINGS optimize_move_to_prewhere= 1 SELECT * FROM 02763_merge_log_1 3 13 @@ -178,3 +202,25 @@ SELECT x, SUM(x) FROM (SELECT * FROM merge(...) UNION ALL ...) GROUP BY x 2 24 3 39 4 42 +1 11 0 +2 12 0 +3 13 0 +4 14 1 +4 14 1 +SELECT * FROM merge(currentDatabase(), 02763_merge_log) WHERE x>1 -- with y>12 +2 12 +3 13 +3 13 +4 14 +4 14 +SELECT * FROM merge(currentDatabase(), 02763_merge_merge) WHERE x>1 -- with y>12 +2 12 +3 13 +3 13 +4 14 +4 14 +2 12 0 +3 13 1 +3 13 1 +4 14 1 +4 14 1 diff --git a/tests/queries/0_stateless/02763_row_policy_storage_merge.sql.j2 b/tests/queries/0_stateless/02763_row_policy_storage_merge.sql.j2 index b5094f927f44..94872dfd7b6a 100644 --- a/tests/queries/0_stateless/02763_row_policy_storage_merge.sql.j2 +++ b/tests/queries/0_stateless/02763_row_policy_storage_merge.sql.j2 @@ -17,10 +17,12 @@ INSERT INTO 02763_merge_log_2 VALUES (1, 11), (2, 12), (3, 13), (4, 14); INSERT INTO 02763_merge_merge_1 VALUES (1, 11), (2, 12), (3, 13), (4, 14); INSERT INTO 02763_merge_merge_2 VALUES (1, 11), (2, 12), (3, 13), (4, 14); -SELECT * FROM merge(currentDatabase(), '02763_merge') ORDER BY x; - +SELECT 'SELECT * FROM 02763_merge_log_1 ORDER BY x'; SELECT * FROM 02763_merge_log_1 ORDER BY x; +SELECT 'SELECT * FROM merge(currentDatabase(), 02763_merge) ORDER BY x'; +SELECT * FROM merge(currentDatabase(), '02763_merge') ORDER BY x; + {% for prew in [0 , 1] -%} @@ -81,7 +83,24 @@ SELECT * FROM merge(currentDatabase(), '02763_merge.*2')) GROUP BY x ORDER BY x; +SELECT *, x=4 FROM merge(currentDatabase(), '02763_merge_merge') ORDER BY x SETTINGS optimize_move_to_prewhere= {{prew}}; + + +CREATE ROW POLICY 02763_filter_3 ON 02763_merge_log_1 USING y>12 AS permissive TO ALL; +SELECT 'SELECT * FROM merge(currentDatabase(), 02763_merge_log) WHERE x>1 -- with y>12'; +SELECT * FROM merge(currentDatabase(), '02763_merge_log') WHERE x>1 ORDER BY x SETTINGS optimize_move_to_prewhere= {{prew}}; + +CREATE ROW POLICY 02763_filter_4 ON 02763_merge_merge_1 USING y>12 AS permissive TO ALL; +SELECT 'SELECT * FROM merge(currentDatabase(), 02763_merge_merge) WHERE x>1 -- with y>12'; +SELECT * FROM merge(currentDatabase(), '02763_merge_merge') WHERE x>1 ORDER BY x SETTINGS optimize_move_to_prewhere= {{prew}}; + +SELECT *, (x=4 OR y>12) FROM merge(currentDatabase(), '02763_merge_merge') WHERE x>1 ORDER BY x SETTINGS optimize_move_to_prewhere= {{prew}}; + + DROP ROW POLICY 02763_filter_1 ON 02763_merge_log_1; DROP ROW POLICY 02763_filter_2 ON 02763_merge_merge_1; +DROP ROW POLICY 02763_filter_3 ON 02763_merge_log_1; +DROP ROW POLICY 02763_filter_4 ON 02763_merge_merge_1; + {% endfor %} From 8f632048eb3c07dd602d20784646e7f73516c7fc Mon Sep 17 00:00:00 2001 From: Ilya Golshtein Date: Tue, 11 Jul 2023 21:16:54 +0000 Subject: [PATCH 081/123] merge_row_policy: works again with adding missed columns --- src/Storages/StorageMerge.cpp | 119 +++++++++++++++++++++------------- 1 file changed, 73 insertions(+), 46 deletions(-) diff --git a/src/Storages/StorageMerge.cpp b/src/Storages/StorageMerge.cpp index 5be809a3593d..c08bb692d3a4 100644 --- a/src/Storages/StorageMerge.cpp +++ b/src/Storages/StorageMerge.cpp @@ -16,6 +16,7 @@ #include #include #include +#include #include #include #include @@ -681,6 +682,29 @@ QueryPipelineBuilderPtr ReadFromMerge::createSources( StorageView * view = dynamic_cast(storage.get()); if (!view || allow_experimental_analyzer) { + + auto row_policy_filter = modified_context->getRowPolicyFilter(database_name, table_name, RowPolicyFilterType::SELECT_FILTER); + if (row_policy_filter) + { + ASTPtr expr = row_policy_filter->expression; + + RequiredSourceColumnsVisitor::Data columns_context; + RequiredSourceColumnsVisitor(columns_context).visit(expr); + + auto req_columns = columns_context.requiredColumns(); + for (const auto & req_column : req_columns) + { + LOG_TRACE(&Poco::Logger::get("ReadFromMerge::createSources"), "req.column: {}", req_column); + std::sort(real_column_names.begin(), real_column_names.end()); + + if (!std::binary_search(real_column_names.begin(), real_column_names.end(), req_column)) + { + real_column_names.push_back(req_column); + } + } + } + + storage->read(plan, real_column_names, storage_snapshot, @@ -694,13 +718,12 @@ QueryPipelineBuilderPtr ReadFromMerge::createSources( if (!plan.isInitialized()) return {}; - if (auto * source_step_with_filter = dynamic_cast((plan.getRootNode()->step.get()))) - { - auto row_policy_filter = modified_context->getRowPolicyFilter(database_name, table_name, RowPolicyFilterType::SELECT_FILTER); - if (row_policy_filter) + if (row_policy_filter) + { + ASTPtr expr = row_policy_filter->expression; + if (auto * source_step_with_filter = dynamic_cast((plan.getRootNode()->step.get()))) { - ASTPtr expr = row_policy_filter->expression; auto storage_metadata_snapshot = storage->getInMemoryMetadataPtr(); auto storage_columns = storage_metadata_snapshot->getColumns(); @@ -713,12 +736,13 @@ QueryPipelineBuilderPtr ReadFromMerge::createSources( auto filter_actions = std::make_shared(actions_dag, ExpressionActionsSettings::fromContext(modified_context, CompileExpressions::yes)); auto required_columns = filter_actions->getRequiredColumns(); - LOG_TRACE(&Poco::Logger::get("ReadFromMerge::convertingSourceStream"), "filter_actions_dag: {},<> {}, <> {}", + LOG_TRACE(&Poco::Logger::get("ReadFromMerge::createSources"), "filter_actions_dag: {},<> {}, <> {}", filter_actions->getActionsDAG().dumpNames(), filter_actions->getActionsDAG().dumpDAG(), filter_actions->getSampleBlock().dumpStructure()); auto filter_column_name = namesDifference(filter_actions->getSampleBlock().getNames(), filter_actions->getRequiredColumns()); source_step_with_filter->addFilter(actions_dag, filter_column_name); } + } } else @@ -1026,46 +1050,6 @@ void ReadFromMerge::convertingSourceStream( const String & database_name, const String & table_name) { - Block before_block_header = builder.getHeader(); - - auto storage_sample_block = metadata_snapshot->getSampleBlock(); - auto pipe_columns = builder.getHeader().getNamesAndTypesList(); - - for (const auto & alias : aliases) - { - pipe_columns.emplace_back(NameAndTypePair(alias.name, alias.type)); - ASTPtr expr = alias.expression; - auto syntax_result = TreeRewriter(local_context).analyze(expr, pipe_columns); - auto expression_analyzer = ExpressionAnalyzer{alias.expression, syntax_result, local_context}; - - auto dag = std::make_shared(pipe_columns); - auto actions_dag = expression_analyzer.getActionsDAG(true, false); - auto actions = std::make_shared(actions_dag, ExpressionActionsSettings::fromContext(local_context, CompileExpressions::yes)); - - builder.addSimpleTransform([&](const Block & stream_header) - { - return std::make_shared(stream_header, actions); - }); - } - - ActionsDAG::MatchColumnsMode convert_actions_match_columns_mode = ActionsDAG::MatchColumnsMode::Name; - - if (local_context->getSettingsRef().allow_experimental_analyzer && processed_stage != QueryProcessingStage::FetchColumns) - convert_actions_match_columns_mode = ActionsDAG::MatchColumnsMode::Position; - - auto convert_actions_dag = ActionsDAG::makeConvertingActions(builder.getHeader().getColumnsWithTypeAndName(), - header.getColumnsWithTypeAndName(), - convert_actions_match_columns_mode); - auto actions = std::make_shared( - std::move(convert_actions_dag), - ExpressionActionsSettings::fromContext(local_context, CompileExpressions::yes)); - - builder.addSimpleTransform([&](const Block & stream_header) - { - return std::make_shared(stream_header, actions); - }); - - auto row_policy_filter = local_context->getRowPolicyFilter(database_name, table_name, RowPolicyFilterType::SELECT_FILTER); if (row_policy_filter) @@ -1107,6 +1091,49 @@ void ReadFromMerge::convertingSourceStream( return std::make_shared(stream_header, filter_actions, filter_column_name, true /* remove fake column */); }); } + + + + Block before_block_header = builder.getHeader(); + + auto storage_sample_block = metadata_snapshot->getSampleBlock(); + auto pipe_columns = builder.getHeader().getNamesAndTypesList(); + + for (const auto & alias : aliases) + { + pipe_columns.emplace_back(NameAndTypePair(alias.name, alias.type)); + ASTPtr expr = alias.expression; + auto syntax_result = TreeRewriter(local_context).analyze(expr, pipe_columns); + auto expression_analyzer = ExpressionAnalyzer{alias.expression, syntax_result, local_context}; + + auto dag = std::make_shared(pipe_columns); + auto actions_dag = expression_analyzer.getActionsDAG(true, false); + auto actions = std::make_shared(actions_dag, ExpressionActionsSettings::fromContext(local_context, CompileExpressions::yes)); + + builder.addSimpleTransform([&](const Block & stream_header) + { + return std::make_shared(stream_header, actions); + }); + } + + ActionsDAG::MatchColumnsMode convert_actions_match_columns_mode = ActionsDAG::MatchColumnsMode::Name; + + if (local_context->getSettingsRef().allow_experimental_analyzer && processed_stage != QueryProcessingStage::FetchColumns) + convert_actions_match_columns_mode = ActionsDAG::MatchColumnsMode::Position; + + auto convert_actions_dag = ActionsDAG::makeConvertingActions(builder.getHeader().getColumnsWithTypeAndName(), + header.getColumnsWithTypeAndName(), + convert_actions_match_columns_mode); + auto actions = std::make_shared( + std::move(convert_actions_dag), + ExpressionActionsSettings::fromContext(local_context, CompileExpressions::yes)); + + builder.addSimpleTransform([&](const Block & stream_header) + { + return std::make_shared(stream_header, actions); + }); + + } bool ReadFromMerge::requestReadingInOrder(InputOrderInfoPtr order_info_) From dd57a8e3456230ed55fd1ddbbee0a4d15d9507aa Mon Sep 17 00:00:00 2001 From: Ilya Golshtein Date: Wed, 12 Jul 2023 14:42:32 +0000 Subject: [PATCH 082/123] merge_row_policy: refactored with ReadFromMerge::RowPolicyData --- src/Storages/StorageMerge.cpp | 246 ++++++++++-------- src/Storages/StorageMerge.h | 5 +- .../02763_row_policy_storage_merge.reference | 26 ++ .../02763_row_policy_storage_merge.sql.j2 | 3 +- 4 files changed, 173 insertions(+), 107 deletions(-) diff --git a/src/Storages/StorageMerge.cpp b/src/Storages/StorageMerge.cpp index c08bb692d3a4..21fab8d9495c 100644 --- a/src/Storages/StorageMerge.cpp +++ b/src/Storages/StorageMerge.cpp @@ -67,26 +67,6 @@ bool columnDefaultKindHasSameType(ColumnDefaultKind lhs, ColumnDefaultKind rhs) return false; } -std::string namesDifference(Names && outer_set, Names && inner_set) -{ - std::sort(outer_set.begin(), outer_set.end()); - - std::sort(inner_set.begin(), inner_set.end()); - - Names result; - - std::set_difference(outer_set.begin(), outer_set.end(), - inner_set.begin(), inner_set.end(), std::inserter(result, result.begin())); - - if (result.size() != 1) - { - throw Exception(ErrorCodes::LOGICAL_ERROR, - "Cannot determine row level filter"); - } - - return result.front(); -} - } namespace DB @@ -395,6 +375,28 @@ void StorageMerge::read( query_plan.addStep(std::move(step)); } +class ReadFromMerge::RowPolicyData +{ +public: + bool needCare() + { + return static_cast(row_policy_filter_ptr); + } + void init(RowPolicyFilterPtr, + const std::shared_ptr, + ContextPtr); + void extendNames(Names &); + void addStorageFilter(SourceStepWithFilter *); + void addFilterTransform(QueryPipelineBuilder &); +private: + static std::string namesDifference(Names && outer_set, Names && inner_set); + RowPolicyFilterPtr row_policy_filter_ptr; + std::string filter_column_name; + ActionsDAGPtr actions_dag; + ExpressionActionsPtr filter_actions; +}; + + ReadFromMerge::ReadFromMerge( Block common_header_, StorageListWithLocks selected_tables_, @@ -671,6 +673,8 @@ QueryPipelineBuilderPtr ReadFromMerge::createSources( storage_snapshot, modified_query_info); + RowPolicyData row_policy_data; + if (processed_stage <= storage_stage || (allow_experimental_analyzer && processed_stage == QueryProcessingStage::FetchColumns)) { /// If there are only virtual columns in query, you must request at least one other column. @@ -682,29 +686,15 @@ QueryPipelineBuilderPtr ReadFromMerge::createSources( StorageView * view = dynamic_cast(storage.get()); if (!view || allow_experimental_analyzer) { + row_policy_data.init(modified_context->getRowPolicyFilter(database_name, table_name, RowPolicyFilterType::SELECT_FILTER), + storage, + modified_context); - auto row_policy_filter = modified_context->getRowPolicyFilter(database_name, table_name, RowPolicyFilterType::SELECT_FILTER); - if (row_policy_filter) + if (row_policy_data.needCare()) { - ASTPtr expr = row_policy_filter->expression; - - RequiredSourceColumnsVisitor::Data columns_context; - RequiredSourceColumnsVisitor(columns_context).visit(expr); - - auto req_columns = columns_context.requiredColumns(); - for (const auto & req_column : req_columns) - { - LOG_TRACE(&Poco::Logger::get("ReadFromMerge::createSources"), "req.column: {}", req_column); - std::sort(real_column_names.begin(), real_column_names.end()); - - if (!std::binary_search(real_column_names.begin(), real_column_names.end(), req_column)) - { - real_column_names.push_back(req_column); - } - } + row_policy_data.extendNames(real_column_names); } - storage->read(plan, real_column_names, storage_snapshot, @@ -714,35 +704,16 @@ QueryPipelineBuilderPtr ReadFromMerge::createSources( max_block_size, UInt32(streams_num)); - if (!plan.isInitialized()) return {}; - if (row_policy_filter) + if (row_policy_data.needCare()) { - ASTPtr expr = row_policy_filter->expression; if (auto * source_step_with_filter = dynamic_cast((plan.getRootNode()->step.get()))) { - - auto storage_metadata_snapshot = storage->getInMemoryMetadataPtr(); - auto storage_columns = storage_metadata_snapshot->getColumns(); - auto needed_columns = storage_columns.getAllPhysical(); // header.getNamesAndTypesList() - - auto syntax_result = TreeRewriter(modified_context).analyze(expr, needed_columns); - auto expression_analyzer = ExpressionAnalyzer{row_policy_filter->expression, syntax_result, modified_context}; - - auto actions_dag = expression_analyzer.getActionsDAG(true, false); - auto filter_actions = std::make_shared(actions_dag, ExpressionActionsSettings::fromContext(modified_context, CompileExpressions::yes)); - auto required_columns = filter_actions->getRequiredColumns(); - - LOG_TRACE(&Poco::Logger::get("ReadFromMerge::createSources"), "filter_actions_dag: {},<> {}, <> {}", - filter_actions->getActionsDAG().dumpNames(), filter_actions->getActionsDAG().dumpDAG(), filter_actions->getSampleBlock().dumpStructure()); - - auto filter_column_name = namesDifference(filter_actions->getSampleBlock().getNames(), filter_actions->getRequiredColumns()); - source_step_with_filter->addFilter(actions_dag, filter_column_name); + row_policy_data.addStorageFilter(source_step_with_filter); } - } } else @@ -864,12 +835,119 @@ QueryPipelineBuilderPtr ReadFromMerge::createSources( /// Subordinary tables could have different but convertible types, like numeric types of different width. /// We must return streams with structure equals to structure of Merge table. - convertingSourceStream(header, storage_snapshot->metadata, aliases, modified_context, *builder, processed_stage, database_name, table_name); + convertingSourceStream(header, + storage_snapshot->metadata, + aliases, + modified_context, + *builder, + processed_stage, + row_policy_data); } return builder; } +void ReadFromMerge::RowPolicyData::init(RowPolicyFilterPtr row_policy_filter_ptr_, + const std::shared_ptr storage, + ContextPtr local_context) +{ + + if (row_policy_filter_ptr_) + { + row_policy_filter_ptr = row_policy_filter_ptr_; + + ASTPtr expr = row_policy_filter_ptr->expression; + + auto storage_metadata_snapshot = storage->getInMemoryMetadataPtr(); + auto storage_columns = storage_metadata_snapshot->getColumns(); + auto needed_columns = storage_columns.getAllPhysical(); // header.getNamesAndTypesList() + + + auto syntax_result = TreeRewriter(local_context).analyze(expr, needed_columns /* pipe_columns */); + auto expression_analyzer = ExpressionAnalyzer{expr, syntax_result, local_context}; + + actions_dag = expression_analyzer.getActionsDAG(true, false); + filter_actions = std::make_shared(actions_dag, ExpressionActionsSettings::fromContext(local_context, CompileExpressions::yes)); + filter_column_name = namesDifference(filter_actions->getSampleBlock().getNames(), filter_actions->getRequiredColumns()); + } + +} + +void ReadFromMerge::RowPolicyData::extendNames(Names & names) +{ + assert(row_policy_filter_ptr); + ASTPtr expr = row_policy_filter_ptr->expression; + + RequiredSourceColumnsVisitor::Data columns_context; + RequiredSourceColumnsVisitor(columns_context).visit(expr); + + auto req_columns = columns_context.requiredColumns(); + for (const auto & req_column : req_columns) + { + LOG_TRACE(&Poco::Logger::get("ReadFromMerge::RowPolicyData::extendNames"), "req.column: {}", req_column); + std::sort(names.begin(), names.end()); + + if (!std::binary_search(names.begin(), names.end(), req_column)) + { + names.push_back(req_column); + } + } +} + +void ReadFromMerge::RowPolicyData::addStorageFilter(SourceStepWithFilter * step) +{ + assert(row_policy_filter_ptr); + LOG_TRACE(&Poco::Logger::get("ReadFromMerge::createSources"), "filter_actions_dag: {},<> {}, <> {}", + filter_actions->getActionsDAG().dumpNames(), filter_actions->getActionsDAG().dumpDAG(), filter_actions->getSampleBlock().dumpStructure()); + step->addFilter(actions_dag, filter_column_name); +} + +void ReadFromMerge::RowPolicyData::addFilterTransform(QueryPipelineBuilder & builder) +{ + assert(row_policy_filter_ptr); + + LOG_TRACE(&Poco::Logger::get("ReadFromMerge::convertinfSourceStream"), "filter_actions_dag: {},<> {}, <> {}", + filter_actions->getActionsDAG().dumpNames(), filter_actions->getActionsDAG().dumpDAG(), filter_actions->getSampleBlock().dumpStructure()); + + + for (auto & colname : filter_actions->getSampleBlock().getNames()) + { + LOG_TRACE(&Poco::Logger::get("ReadFromMerge::convertingSourceStream"), "filter_actions->getSampleBlock().getNames(): {}", colname); + } + + for (auto & colname : filter_actions->getRequiredColumns()) + { + LOG_TRACE(&Poco::Logger::get("ReadFromMerge::convertingSourceStream"), "filter_actions->getRequiredColumns(): {}", colname); + } + + // auto filter_column_name = namesDifference(filter_actions->getSampleBlock().getNames(), filter_actions->getRequiredColumns()); + + builder.addSimpleTransform([&](const Block & stream_header) + { + return std::make_shared(stream_header, filter_actions, filter_column_name, true /* remove fake column */); + }); +} + +std::string ReadFromMerge::RowPolicyData::namesDifference(Names && outer_set, Names && inner_set) +{ + std::sort(outer_set.begin(), outer_set.end()); + + std::sort(inner_set.begin(), inner_set.end()); + + Names result; + + std::set_difference(outer_set.begin(), outer_set.end(), + inner_set.begin(), inner_set.end(), std::inserter(result, result.begin())); + + if (result.size() != 1) + { + throw Exception(ErrorCodes::LOGICAL_ERROR, + "Cannot determine row level filter"); + } + + return result.front(); +} + StorageMerge::StorageListWithLocks StorageMerge::getSelectedTables( ContextPtr query_context, const ASTPtr & query /* = nullptr */, @@ -1047,53 +1125,13 @@ void ReadFromMerge::convertingSourceStream( ContextPtr local_context, QueryPipelineBuilder & builder, const QueryProcessingStage::Enum & processed_stage, - const String & database_name, - const String & table_name) + RowPolicyData & row_policy_data) { - auto row_policy_filter = local_context->getRowPolicyFilter(database_name, table_name, RowPolicyFilterType::SELECT_FILTER); - - if (row_policy_filter) + if (row_policy_data.needCare()) { - ASTPtr expr = row_policy_filter->expression; - - auto storage_columns = metadata_snapshot->getColumns(); - auto needed_columns = storage_columns.getAllPhysical(); // header.getNamesAndTypesList() - - - auto syntax_result = TreeRewriter(local_context).analyze(expr, needed_columns /* pipe_columns */); - auto expression_analyzer = ExpressionAnalyzer{row_policy_filter->expression, syntax_result, local_context}; - - auto actions_dag = expression_analyzer.getActionsDAG(true, false); - auto filter_actions = std::make_shared(actions_dag, ExpressionActionsSettings::fromContext(local_context, CompileExpressions::yes)); - - LOG_TRACE(&Poco::Logger::get("ReadFromMerge::convertinfSourceStream"), "filter_actions_dag: {},<> {}, <> {}", - filter_actions->getActionsDAG().dumpNames(), filter_actions->getActionsDAG().dumpDAG(), filter_actions->getSampleBlock().dumpStructure()); - - - - for (auto & colname : filter_actions->getSampleBlock().getNames()) - { - LOG_TRACE(&Poco::Logger::get("ReadFromMerge::convertinfSourceStream"), "filter_actions->getSampleBlock().getNames(): {}", colname); - } - - for (auto & colname : filter_actions->getRequiredColumns()) - { - LOG_TRACE(&Poco::Logger::get("ReadFromMerge::convertinfSourceStream"), "filter_actions->getRequiredColumns(): {}", colname); - } - - - - auto filter_column_name = namesDifference(filter_actions->getSampleBlock().getNames(), filter_actions->getRequiredColumns()); - - - builder.addSimpleTransform([&](const Block & stream_header) - { - return std::make_shared(stream_header, filter_actions, filter_column_name, true /* remove fake column */); - }); + row_policy_data.addFilterTransform(builder); } - - Block before_block_header = builder.getHeader(); auto storage_sample_block = metadata_snapshot->getSampleBlock(); diff --git a/src/Storages/StorageMerge.h b/src/Storages/StorageMerge.h index fbe6dcec2980..4dc7cb77c488 100644 --- a/src/Storages/StorageMerge.h +++ b/src/Storages/StorageMerge.h @@ -177,6 +177,8 @@ class ReadFromMerge final : public SourceStepWithFilter using Aliases = std::vector; + class RowPolicyData; + static SelectQueryInfo getModifiedQueryInfo(const SelectQueryInfo & query_info, const ContextPtr & modified_context, const StorageWithLockAndName & storage_with_lock_and_name, @@ -202,8 +204,7 @@ class ReadFromMerge final : public SourceStepWithFilter ContextPtr context, QueryPipelineBuilder & builder, const QueryProcessingStage::Enum & processed_stage, - const String & database_name, - const String & table_name); + RowPolicyData & row_policy_data); }; } diff --git a/tests/queries/0_stateless/02763_row_policy_storage_merge.reference b/tests/queries/0_stateless/02763_row_policy_storage_merge.reference index 55890a117836..070520bb392a 100644 --- a/tests/queries/0_stateless/02763_row_policy_storage_merge.reference +++ b/tests/queries/0_stateless/02763_row_policy_storage_merge.reference @@ -122,6 +122,19 @@ SELECT * FROM merge(currentDatabase(), 02763_merge_merge) WHERE x>1 -- with y>12 3 13 1 4 14 1 4 14 1 +SELECT y from merge(currentDatabase(), 02763_merge) +11 +11 +12 +12 +13 +13 +13 +13 +14 +14 +14 +14 SETTINGS optimize_move_to_prewhere= 1 SELECT * FROM 02763_merge_log_1 3 13 @@ -224,3 +237,16 @@ SELECT * FROM merge(currentDatabase(), 02763_merge_merge) WHERE x>1 -- with y>12 3 13 1 4 14 1 4 14 1 +SELECT y from merge(currentDatabase(), 02763_merge) +11 +11 +12 +12 +13 +13 +13 +13 +14 +14 +14 +14 diff --git a/tests/queries/0_stateless/02763_row_policy_storage_merge.sql.j2 b/tests/queries/0_stateless/02763_row_policy_storage_merge.sql.j2 index 94872dfd7b6a..3030be2539fe 100644 --- a/tests/queries/0_stateless/02763_row_policy_storage_merge.sql.j2 +++ b/tests/queries/0_stateless/02763_row_policy_storage_merge.sql.j2 @@ -85,7 +85,6 @@ ORDER BY x; SELECT *, x=4 FROM merge(currentDatabase(), '02763_merge_merge') ORDER BY x SETTINGS optimize_move_to_prewhere= {{prew}}; - CREATE ROW POLICY 02763_filter_3 ON 02763_merge_log_1 USING y>12 AS permissive TO ALL; SELECT 'SELECT * FROM merge(currentDatabase(), 02763_merge_log) WHERE x>1 -- with y>12'; SELECT * FROM merge(currentDatabase(), '02763_merge_log') WHERE x>1 ORDER BY x SETTINGS optimize_move_to_prewhere= {{prew}}; @@ -96,6 +95,8 @@ SELECT * FROM merge(currentDatabase(), '02763_merge_merge') WHERE x>1 ORDER BY x SELECT *, (x=4 OR y>12) FROM merge(currentDatabase(), '02763_merge_merge') WHERE x>1 ORDER BY x SETTINGS optimize_move_to_prewhere= {{prew}}; +SELECT 'SELECT y from merge(currentDatabase(), 02763_merge)'; +SELECT y from merge(currentDatabase(), '02763_merge') ORDER BY y SETTINGS optimize_move_to_prewhere= {{prew}}; DROP ROW POLICY 02763_filter_1 ON 02763_merge_log_1; DROP ROW POLICY 02763_filter_2 ON 02763_merge_merge_1; From edc67fb18a20137b4b7075b4d583bad1b2a62f91 Mon Sep 17 00:00:00 2001 From: Ilya Golshtein Date: Wed, 12 Jul 2023 15:11:59 +0000 Subject: [PATCH 083/123] merge_row_policy: stylecheck --- src/Storages/StorageMerge.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/Storages/StorageMerge.cpp b/src/Storages/StorageMerge.cpp index 21fab8d9495c..b18ef08f30bd 100644 --- a/src/Storages/StorageMerge.cpp +++ b/src/Storages/StorageMerge.cpp @@ -937,7 +937,7 @@ std::string ReadFromMerge::RowPolicyData::namesDifference(Names && outer_set, Na Names result; std::set_difference(outer_set.begin(), outer_set.end(), - inner_set.begin(), inner_set.end(), std::inserter(result, result.begin())); + inner_set.begin(), inner_set.end(), std::inserter(result, result.begin())); if (result.size() != 1) { From ee7188afcba733ad6c9af5a59520dff9e52b9fea Mon Sep 17 00:00:00 2001 From: Ilya Golshtein Date: Wed, 12 Jul 2023 15:52:48 +0000 Subject: [PATCH 084/123] merge_row_policy: stylecheck again --- src/Storages/StorageMerge.cpp | 1 + 1 file changed, 1 insertion(+) diff --git a/src/Storages/StorageMerge.cpp b/src/Storages/StorageMerge.cpp index b18ef08f30bd..5a9a4b2f1da5 100644 --- a/src/Storages/StorageMerge.cpp +++ b/src/Storages/StorageMerge.cpp @@ -81,6 +81,7 @@ namespace ErrorCodes extern const int SAMPLING_NOT_SUPPORTED; extern const int ALTER_OF_COLUMN_IS_FORBIDDEN; extern const int CANNOT_EXTRACT_TABLE_STRUCTURE; + extern const int LOGICAL_ERROR; } StorageMerge::StorageMerge( From 9f1655bcfb2a6a5565c4780157c9d43c278ef029 Mon Sep 17 00:00:00 2001 From: Ilya Golshtein Date: Fri, 14 Jul 2023 13:39:43 +0000 Subject: [PATCH 085/123] merge_row_policy: cleanup, comments, new tests --- src/Storages/StorageMerge.cpp | 78 ++++++++++--------- .../02763_row_policy_storage_merge.reference | 54 +++++++++++++ .../02763_row_policy_storage_merge.sql.j2 | 25 ++++++ 3 files changed, 121 insertions(+), 36 deletions(-) diff --git a/src/Storages/StorageMerge.cpp b/src/Storages/StorageMerge.cpp index 5a9a4b2f1da5..49cc923359d2 100644 --- a/src/Storages/StorageMerge.cpp +++ b/src/Storages/StorageMerge.cpp @@ -268,7 +268,6 @@ QueryProcessingStage::Enum StorageMerge::getQueryProcessingStage( /// (see removeJoin()) /// /// And for this we need to return FetchColumns. - if (const auto * select = query_info.query->as(); select && hasJoin(*select)) return QueryProcessingStage::FetchColumns; @@ -315,6 +314,7 @@ void StorageMerge::read( * since there is no certainty that it works when one of table is MergeTree and other is not. */ auto modified_context = Context::createCopy(local_context); + modified_context->setSetting("optimize_move_to_prewhere", false); bool has_database_virtual_column = false; bool has_table_virtual_column = false; @@ -376,9 +376,13 @@ void StorageMerge::read( query_plan.addStep(std::move(step)); } +/// A transient object of this helper class is created +/// when processing a Merge table data source (subordinary table) +/// to guarantee that row policies are applied class ReadFromMerge::RowPolicyData { public: + /// Row policy requires extra filtering bool needCare() { return static_cast(row_policy_filter_ptr); @@ -386,18 +390,28 @@ class ReadFromMerge::RowPolicyData void init(RowPolicyFilterPtr, const std::shared_ptr, ContextPtr); + + /// Add columns that needed for row policies to data stream + /// SELECT x from T if T has row policy y=42 + /// required y in data pipeline void extendNames(Names &); + + /// Use storage facilities to filter data + /// does not guarantee accuracy, but reduce number of rows void addStorageFilter(SourceStepWithFilter *); + + /// Create explicit filter transform to stop + /// rows that are not conform to row level policy void addFilterTransform(QueryPipelineBuilder &); + private: static std::string namesDifference(Names && outer_set, Names && inner_set); RowPolicyFilterPtr row_policy_filter_ptr; - std::string filter_column_name; + std::string filter_column_name; // complex filer, may contain logic operations ActionsDAGPtr actions_dag; ExpressionActionsPtr filter_actions; }; - ReadFromMerge::ReadFromMerge( Block common_header_, StorageListWithLocks selected_tables_, @@ -687,7 +701,11 @@ QueryPipelineBuilderPtr ReadFromMerge::createSources( StorageView * view = dynamic_cast(storage.get()); if (!view || allow_experimental_analyzer) { - row_policy_data.init(modified_context->getRowPolicyFilter(database_name, table_name, RowPolicyFilterType::SELECT_FILTER), + row_policy_data.init( + modified_context->getRowPolicyFilter( + database_name, + table_name, + RowPolicyFilterType::SELECT_FILTER), storage, modified_context); @@ -708,7 +726,6 @@ QueryPipelineBuilderPtr ReadFromMerge::createSources( if (!plan.isInitialized()) return {}; - if (row_policy_data.needCare()) { if (auto * source_step_with_filter = dynamic_cast((plan.getRootNode()->step.get()))) @@ -742,14 +759,12 @@ QueryPipelineBuilderPtr ReadFromMerge::createSources( { size_t filters_dags_size = filter_dags.size(); for (size_t i = 0; i < filters_dags_size; ++i) - { read_from_merge_tree->addFilter(filter_dags[i], filter_nodes.nodes[i]); - } } + builder = plan.buildQueryPipeline( QueryPlanOptimizationSettings::fromContext(modified_context), BuildQueryPipelineSettings::fromContext(modified_context)); - } else if (processed_stage > storage_stage || (allow_experimental_analyzer && processed_stage != QueryProcessingStage::FetchColumns)) { @@ -835,6 +850,7 @@ QueryPipelineBuilderPtr ReadFromMerge::createSources( /// Subordinary tables could have different but convertible types, like numeric types of different width. /// We must return streams with structure equals to structure of Merge table. + /// Besides this we add FilterTransform if it is needed to follow row level policies. convertingSourceStream(header, storage_snapshot->metadata, @@ -852,7 +868,6 @@ void ReadFromMerge::RowPolicyData::init(RowPolicyFilterPtr row_policy_filter_ptr const std::shared_ptr storage, ContextPtr local_context) { - if (row_policy_filter_ptr_) { row_policy_filter_ptr = row_policy_filter_ptr_; @@ -861,19 +876,20 @@ void ReadFromMerge::RowPolicyData::init(RowPolicyFilterPtr row_policy_filter_ptr auto storage_metadata_snapshot = storage->getInMemoryMetadataPtr(); auto storage_columns = storage_metadata_snapshot->getColumns(); - auto needed_columns = storage_columns.getAllPhysical(); // header.getNamesAndTypesList() - + auto needed_columns = storage_columns.getAllPhysical(); - auto syntax_result = TreeRewriter(local_context).analyze(expr, needed_columns /* pipe_columns */); + auto syntax_result = TreeRewriter(local_context).analyze(expr, needed_columns); auto expression_analyzer = ExpressionAnalyzer{expr, syntax_result, local_context}; actions_dag = expression_analyzer.getActionsDAG(true, false); - filter_actions = std::make_shared(actions_dag, ExpressionActionsSettings::fromContext(local_context, CompileExpressions::yes)); + filter_actions = std::make_shared(actions_dag, + ExpressionActionsSettings::fromContext(local_context, CompileExpressions::yes)); filter_column_name = namesDifference(filter_actions->getSampleBlock().getNames(), filter_actions->getRequiredColumns()); } - } +// Add columns that needed to evaluate row policies +// SELECT x from t if t has row policy void ReadFromMerge::RowPolicyData::extendNames(Names & names) { assert(row_policy_filter_ptr); @@ -885,7 +901,6 @@ void ReadFromMerge::RowPolicyData::extendNames(Names & names) auto req_columns = columns_context.requiredColumns(); for (const auto & req_column : req_columns) { - LOG_TRACE(&Poco::Logger::get("ReadFromMerge::RowPolicyData::extendNames"), "req.column: {}", req_column); std::sort(names.begin(), names.end()); if (!std::binary_search(names.begin(), names.end(), req_column)) @@ -898,30 +913,22 @@ void ReadFromMerge::RowPolicyData::extendNames(Names & names) void ReadFromMerge::RowPolicyData::addStorageFilter(SourceStepWithFilter * step) { assert(row_policy_filter_ptr); - LOG_TRACE(&Poco::Logger::get("ReadFromMerge::createSources"), "filter_actions_dag: {},<> {}, <> {}", - filter_actions->getActionsDAG().dumpNames(), filter_actions->getActionsDAG().dumpDAG(), filter_actions->getSampleBlock().dumpStructure()); - step->addFilter(actions_dag, filter_column_name); + LOG_TRACE(&Poco::Logger::get("ReadFromMerge::RowPolicyData::addStorageFilter"), "filter_actions_dag: {},<> {}, <> {}", + filter_actions->getActionsDAG().dumpNames(), + filter_actions->getActionsDAG().dumpDAG(), + filter_actions->getSampleBlock().dumpStructure()); + + step->addFilter(actions_dag, filter_column_name); } void ReadFromMerge::RowPolicyData::addFilterTransform(QueryPipelineBuilder & builder) { assert(row_policy_filter_ptr); - LOG_TRACE(&Poco::Logger::get("ReadFromMerge::convertinfSourceStream"), "filter_actions_dag: {},<> {}, <> {}", - filter_actions->getActionsDAG().dumpNames(), filter_actions->getActionsDAG().dumpDAG(), filter_actions->getSampleBlock().dumpStructure()); - - - for (auto & colname : filter_actions->getSampleBlock().getNames()) - { - LOG_TRACE(&Poco::Logger::get("ReadFromMerge::convertingSourceStream"), "filter_actions->getSampleBlock().getNames(): {}", colname); - } - - for (auto & colname : filter_actions->getRequiredColumns()) - { - LOG_TRACE(&Poco::Logger::get("ReadFromMerge::convertingSourceStream"), "filter_actions->getRequiredColumns(): {}", colname); - } - - // auto filter_column_name = namesDifference(filter_actions->getSampleBlock().getNames(), filter_actions->getRequiredColumns()); + LOG_TRACE(&Poco::Logger::get("ReadFromMerge::RowPolicyData::addFilterTransform"), "filter_actions_dag: {},<> {}, <> {}", + filter_actions->getActionsDAG().dumpNames(), + filter_actions->getActionsDAG().dumpDAG(), + filter_actions->getSampleBlock().dumpStructure()); builder.addSimpleTransform([&](const Block & stream_header) { @@ -929,10 +936,10 @@ void ReadFromMerge::RowPolicyData::addFilterTransform(QueryPipelineBuilder & bui }); } +/// Find out an item that in outer_set vector, but not in inner_set vector std::string ReadFromMerge::RowPolicyData::namesDifference(Names && outer_set, Names && inner_set) { std::sort(outer_set.begin(), outer_set.end()); - std::sort(inner_set.begin(), inner_set.end()); Names result; @@ -949,6 +956,7 @@ std::string ReadFromMerge::RowPolicyData::namesDifference(Names && outer_set, Na return result.front(); } + StorageMerge::StorageListWithLocks StorageMerge::getSelectedTables( ContextPtr query_context, const ASTPtr & query /* = nullptr */, @@ -1171,8 +1179,6 @@ void ReadFromMerge::convertingSourceStream( { return std::make_shared(stream_header, actions); }); - - } bool ReadFromMerge::requestReadingInOrder(InputOrderInfoPtr order_info_) diff --git a/tests/queries/0_stateless/02763_row_policy_storage_merge.reference b/tests/queries/0_stateless/02763_row_policy_storage_merge.reference index 070520bb392a..9c91a1652c7a 100644 --- a/tests/queries/0_stateless/02763_row_policy_storage_merge.reference +++ b/tests/queries/0_stateless/02763_row_policy_storage_merge.reference @@ -135,6 +135,33 @@ SELECT y from merge(currentDatabase(), 02763_merge) 14 14 14 +02763_merge_fancycols +SELECT * +SELECT x, lc +SELECT * +1 11 111 111 42 +1 11 111 111 42 +SELECT x, lc +1 111 +1 111 +SELECT x, lc, cnst +1 111 42 +1 111 42 +SELECT x, y from merge(currentDatabase(), 02763_merge +1 11 +1 11 +1 11 +1 11 +2 12 +2 12 +3 13 +3 13 +3 13 +3 13 +4 14 +4 14 +4 14 +4 14 SETTINGS optimize_move_to_prewhere= 1 SELECT * FROM 02763_merge_log_1 3 13 @@ -250,3 +277,30 @@ SELECT y from merge(currentDatabase(), 02763_merge) 14 14 14 +02763_merge_fancycols +SELECT * +SELECT x, lc +SELECT * +1 11 111 111 42 +1 11 111 111 42 +SELECT x, lc +1 111 +1 111 +SELECT x, lc, cnst +1 111 42 +1 111 42 +SELECT x, y from merge(currentDatabase(), 02763_merge +1 11 +1 11 +1 11 +1 11 +2 12 +2 12 +3 13 +3 13 +3 13 +3 13 +4 14 +4 14 +4 14 +4 14 diff --git a/tests/queries/0_stateless/02763_row_policy_storage_merge.sql.j2 b/tests/queries/0_stateless/02763_row_policy_storage_merge.sql.j2 index 3030be2539fe..ea2cf1fef00e 100644 --- a/tests/queries/0_stateless/02763_row_policy_storage_merge.sql.j2 +++ b/tests/queries/0_stateless/02763_row_policy_storage_merge.sql.j2 @@ -98,10 +98,35 @@ SELECT *, (x=4 OR y>12) FROM merge(currentDatabase(), '02763_merge_merge') WHERE SELECT 'SELECT y from merge(currentDatabase(), 02763_merge)'; SELECT y from merge(currentDatabase(), '02763_merge') ORDER BY y SETTINGS optimize_move_to_prewhere= {{prew}}; +SELECT '02763_merge_fancycols'; +CREATE OR REPLACE TABLE 02763_merge_fancycols (x UInt8, y Nullable(UInt64), z String DEFAULT CONCAT(toString(x), toString(y)), lc LowCardinality(String) DEFAULT z, cnst UInt32 MATERIALIZED 42) ENGINE = MergeTree() ORDER BY tuple(); +INSERT INTO 02763_merge_fancycols (x, y) SELECT x, y from merge(currentDatabase(), '02763_merge'); + +CREATE ROW POLICY 02763_filter_5 ON 02763_merge_fancycols USING cnst<>42 AS permissive TO ALL; +SELECT 'SELECT *'; +SELECT * from merge(currentDatabase(), '02763_merge_fancycols') ORDER BY x; +SELECT 'SELECT x, lc'; +SELECT x, lc from merge(currentDatabase(), '02763_merge_fancycols') ORDER BY x; + +CREATE ROW POLICY 02763_filter_6 ON 02763_merge_fancycols USING lc='111' AS permissive TO ALL; +SELECT 'SELECT *'; +SELECT * from merge(currentDatabase(), '02763_merge_fancycols') ORDER BY x; +SELECT 'SELECT x, lc'; +SELECT x, lc from merge(currentDatabase(), '02763_merge_fancycols') ORDER BY x; +SELECT 'SELECT x, lc, cnst'; +SELECT x, lc, cnst from merge(currentDatabase(), '02763_merge_fancycols') ORDER BY x; +SELECT 'SELECT x, y from merge(currentDatabase(), 02763_merge'; +SELECT x, y from merge(currentDatabase(), '02763_merge') ORDER BY x; + +DROP TABLE 02763_merge_fancycols; + DROP ROW POLICY 02763_filter_1 ON 02763_merge_log_1; DROP ROW POLICY 02763_filter_2 ON 02763_merge_merge_1; DROP ROW POLICY 02763_filter_3 ON 02763_merge_log_1; DROP ROW POLICY 02763_filter_4 ON 02763_merge_merge_1; +DROP ROW POLICY 02763_filter_5 ON 02763_merge_fancycols; +DROP ROW POLICY 02763_filter_6 ON 02763_merge_fancycols; + {% endfor %} From 2f0135f8e8ddd198198cc7c7105196f46485ed8c Mon Sep 17 00:00:00 2001 From: Ilya Golshtein Date: Mon, 17 Jul 2023 12:10:27 +0000 Subject: [PATCH 086/123] merge_row_policy: cleanup, verbose short messages in 00002_log ... --- src/Storages/StorageMerge.cpp | 13 ++++++------- src/Storages/StorageMerge.h | 4 ++-- .../00002_log_and_exception_messages_formatting.sql | 3 ++- .../02763_row_policy_storage_merge.sql.j2 | 7 ++++++- 4 files changed, 16 insertions(+), 11 deletions(-) diff --git a/src/Storages/StorageMerge.cpp b/src/Storages/StorageMerge.cpp index 49cc923359d2..b94ecdf546d6 100644 --- a/src/Storages/StorageMerge.cpp +++ b/src/Storages/StorageMerge.cpp @@ -387,9 +387,8 @@ class ReadFromMerge::RowPolicyData { return static_cast(row_policy_filter_ptr); } - void init(RowPolicyFilterPtr, - const std::shared_ptr, - ContextPtr); + + void init(RowPolicyFilterPtr, std::shared_ptr, ContextPtr); /// Add columns that needed for row policies to data stream /// SELECT x from T if T has row policy y=42 @@ -660,8 +659,8 @@ SelectQueryInfo ReadFromMerge::getModifiedQueryInfo(const SelectQueryInfo & quer QueryPipelineBuilderPtr ReadFromMerge::createSources( const StorageSnapshotPtr & storage_snapshot, SelectQueryInfo & modified_query_info, - const QueryProcessingStage::Enum & processed_stage, - const UInt64 max_block_size, + QueryProcessingStage::Enum processed_stage, + UInt64 max_block_size, const Block & header, const Aliases & aliases, const StorageWithLockAndName & storage_with_lock, @@ -865,7 +864,7 @@ QueryPipelineBuilderPtr ReadFromMerge::createSources( } void ReadFromMerge::RowPolicyData::init(RowPolicyFilterPtr row_policy_filter_ptr_, - const std::shared_ptr storage, + std::shared_ptr storage, ContextPtr local_context) { if (row_policy_filter_ptr_) @@ -1133,7 +1132,7 @@ void ReadFromMerge::convertingSourceStream( const Aliases & aliases, ContextPtr local_context, QueryPipelineBuilder & builder, - const QueryProcessingStage::Enum & processed_stage, + QueryProcessingStage::Enum processed_stage, RowPolicyData & row_policy_data) { if (row_policy_data.needCare()) diff --git a/src/Storages/StorageMerge.h b/src/Storages/StorageMerge.h index 4dc7cb77c488..1a5aca24e4c1 100644 --- a/src/Storages/StorageMerge.h +++ b/src/Storages/StorageMerge.h @@ -187,7 +187,7 @@ class ReadFromMerge final : public SourceStepWithFilter QueryPipelineBuilderPtr createSources( const StorageSnapshotPtr & storage_snapshot, SelectQueryInfo & query_info, - const QueryProcessingStage::Enum & processed_stage, + QueryProcessingStage::Enum processed_stage, UInt64 max_block_size, const Block & header, const Aliases & aliases, @@ -203,7 +203,7 @@ class ReadFromMerge final : public SourceStepWithFilter const Aliases & aliases, ContextPtr context, QueryPipelineBuilder & builder, - const QueryProcessingStage::Enum & processed_stage, + QueryProcessingStage::Enum processed_stage, RowPolicyData & row_policy_data); }; diff --git a/tests/queries/0_stateless/00002_log_and_exception_messages_formatting.sql b/tests/queries/0_stateless/00002_log_and_exception_messages_formatting.sql index 53e8ad13ae6a..bd1802c452f5 100644 --- a/tests/queries/0_stateless/00002_log_and_exception_messages_formatting.sql +++ b/tests/queries/0_stateless/00002_log_and_exception_messages_formatting.sql @@ -52,7 +52,8 @@ create temporary table known_short_messages (s String) as select * from (select select 'messages shorter than 10', max2(countDistinctOrDefault(message_format_string), 1) from logs where length(message_format_string) < 10 and message_format_string not in known_short_messages; -- Same as above. Feel free to update the threshold or remove this query if really necessary -select 'messages shorter than 16', max2(countDistinctOrDefault(message_format_string), 3) from logs where length(message_format_string) < 16 and message_format_string not in known_short_messages; +-- select 'messages shorter than 16', max2(countDistinctOrDefault(message_format_string), 3) from logs where length(message_format_string) < 16 and message_format_string not in known_short_messages; +select message_format_string from logs where length(message_format_string) < 16 and message_format_string not in known_short_messages; -- Unlike above, here we look at length of the formatted message, not format string. Most short format strings are fine because they end up decorated with context from outer or inner exceptions, e.g.: -- "Expected end of line" -> "Code: 117. DB::Exception: Expected end of line: (in file/uri /var/lib/clickhouse/user_files/data_02118): (at row 1)" diff --git a/tests/queries/0_stateless/02763_row_policy_storage_merge.sql.j2 b/tests/queries/0_stateless/02763_row_policy_storage_merge.sql.j2 index ea2cf1fef00e..9b3197cc34fe 100644 --- a/tests/queries/0_stateless/02763_row_policy_storage_merge.sql.j2 +++ b/tests/queries/0_stateless/02763_row_policy_storage_merge.sql.j2 @@ -2,8 +2,13 @@ DROP TABLE IF EXISTS 02763_merge_log_1; DROP TABLE IF EXISTS 02763_merge_log_2; DROP TABLE IF EXISTS 02763_merge_merge_1; DROP TABLE IF EXISTS 02763_merge_merge_2; +DROP TABLE IF EXISTS 02763_merge_fancycols; DROP ROW POLICY IF EXISTS 02763_filter_1 ON 02763_merge_log_1; DROP ROW POLICY IF EXISTS 02763_filter_2 ON 02763_merge_merge_1; +DROP ROW POLICY IF EXISTS 02763_filter_3 ON 02763_merge_log_1; +DROP ROW POLICY IF EXISTS 02763_filter_4 ON 02763_merge_merge_1; +DROP ROW POLICY IF EXISTS 02763_filter_5 ON 02763_merge_fancycols; +DROP ROW POLICY IF EXISTS 02763_filter_6 ON 02763_merge_fancycols; CREATE TABLE 02763_merge_log_1 (x UInt8, y UInt64) ENGINE = Log; @@ -99,7 +104,7 @@ SELECT 'SELECT y from merge(currentDatabase(), 02763_merge)'; SELECT y from merge(currentDatabase(), '02763_merge') ORDER BY y SETTINGS optimize_move_to_prewhere= {{prew}}; SELECT '02763_merge_fancycols'; -CREATE OR REPLACE TABLE 02763_merge_fancycols (x UInt8, y Nullable(UInt64), z String DEFAULT CONCAT(toString(x), toString(y)), lc LowCardinality(String) DEFAULT z, cnst UInt32 MATERIALIZED 42) ENGINE = MergeTree() ORDER BY tuple(); +CREATE TABLE 02763_merge_fancycols (x UInt8, y Nullable(UInt64), z String DEFAULT CONCAT(toString(x), toString(y)), lc LowCardinality(String) DEFAULT z, cnst UInt32 MATERIALIZED 42) ENGINE = MergeTree() ORDER BY tuple(); INSERT INTO 02763_merge_fancycols (x, y) SELECT x, y from merge(currentDatabase(), '02763_merge'); CREATE ROW POLICY 02763_filter_5 ON 02763_merge_fancycols USING cnst<>42 AS permissive TO ALL; From 18ca3647696cb5909ab66661e3f8434c980238cf Mon Sep 17 00:00:00 2001 From: Ilya Golshtein Date: Wed, 19 Jul 2023 21:32:52 +0000 Subject: [PATCH 087/123] merge_row_policy: more verbosity for 00002_log_and_exception_... --- .../00002_log_and_exception_messages_formatting.sql | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/tests/queries/0_stateless/00002_log_and_exception_messages_formatting.sql b/tests/queries/0_stateless/00002_log_and_exception_messages_formatting.sql index bd1802c452f5..f7587de60d03 100644 --- a/tests/queries/0_stateless/00002_log_and_exception_messages_formatting.sql +++ b/tests/queries/0_stateless/00002_log_and_exception_messages_formatting.sql @@ -53,7 +53,8 @@ select 'messages shorter than 10', max2(countDistinctOrDefault(message_format_st -- Same as above. Feel free to update the threshold or remove this query if really necessary -- select 'messages shorter than 16', max2(countDistinctOrDefault(message_format_string), 3) from logs where length(message_format_string) < 16 and message_format_string not in known_short_messages; -select message_format_string from logs where length(message_format_string) < 16 and message_format_string not in known_short_messages; +select 'from logs', message_format_string from logs where length(message_format_string) < 16 and message_format_string not in known_short_messages; +select 'from system.text_log', message_format_string from system.text_log where length(message_format_string) < 16 and message_format_string not in known_short_messages and now() - toIntervalMinute(120) < event_time; -- Unlike above, here we look at length of the formatted message, not format string. Most short format strings are fine because they end up decorated with context from outer or inner exceptions, e.g.: -- "Expected end of line" -> "Code: 117. DB::Exception: Expected end of line: (in file/uri /var/lib/clickhouse/user_files/data_02118): (at row 1)" From 5f3a4af1dd466921fa1421ba793d98dbe4b6f519 Mon Sep 17 00:00:00 2001 From: Ilya Golshtein Date: Mon, 11 Sep 2023 15:07:10 +0000 Subject: [PATCH 088/123] merge_row_policy: revert debug, 00002_log... test is fixed in master --- .../00002_log_and_exception_messages_formatting.sql | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/tests/queries/0_stateless/00002_log_and_exception_messages_formatting.sql b/tests/queries/0_stateless/00002_log_and_exception_messages_formatting.sql index f7587de60d03..53e8ad13ae6a 100644 --- a/tests/queries/0_stateless/00002_log_and_exception_messages_formatting.sql +++ b/tests/queries/0_stateless/00002_log_and_exception_messages_formatting.sql @@ -52,9 +52,7 @@ create temporary table known_short_messages (s String) as select * from (select select 'messages shorter than 10', max2(countDistinctOrDefault(message_format_string), 1) from logs where length(message_format_string) < 10 and message_format_string not in known_short_messages; -- Same as above. Feel free to update the threshold or remove this query if really necessary --- select 'messages shorter than 16', max2(countDistinctOrDefault(message_format_string), 3) from logs where length(message_format_string) < 16 and message_format_string not in known_short_messages; -select 'from logs', message_format_string from logs where length(message_format_string) < 16 and message_format_string not in known_short_messages; -select 'from system.text_log', message_format_string from system.text_log where length(message_format_string) < 16 and message_format_string not in known_short_messages and now() - toIntervalMinute(120) < event_time; +select 'messages shorter than 16', max2(countDistinctOrDefault(message_format_string), 3) from logs where length(message_format_string) < 16 and message_format_string not in known_short_messages; -- Unlike above, here we look at length of the formatted message, not format string. Most short format strings are fine because they end up decorated with context from outer or inner exceptions, e.g.: -- "Expected end of line" -> "Code: 117. DB::Exception: Expected end of line: (in file/uri /var/lib/clickhouse/user_files/data_02118): (at row 1)" From cb44ee47b2686f360fed388ff80acd98df566a84 Mon Sep 17 00:00:00 2001 From: Ilya Golshtein Date: Wed, 4 Oct 2023 08:31:12 +0000 Subject: [PATCH 089/123] merge_row_policy: per code review --- src/Storages/StorageMerge.cpp | 26 ++++++++++++++++---------- 1 file changed, 16 insertions(+), 10 deletions(-) diff --git a/src/Storages/StorageMerge.cpp b/src/Storages/StorageMerge.cpp index b94ecdf546d6..7baeaba94389 100644 --- a/src/Storages/StorageMerge.cpp +++ b/src/Storages/StorageMerge.cpp @@ -383,7 +383,7 @@ class ReadFromMerge::RowPolicyData { public: /// Row policy requires extra filtering - bool needCare() + bool hasRowPolicy() { return static_cast(row_policy_filter_ptr); } @@ -708,7 +708,7 @@ QueryPipelineBuilderPtr ReadFromMerge::createSources( storage, modified_context); - if (row_policy_data.needCare()) + if (row_policy_data.hasRowPolicy()) { row_policy_data.extendNames(real_column_names); } @@ -725,7 +725,7 @@ QueryPipelineBuilderPtr ReadFromMerge::createSources( if (!plan.isInitialized()) return {}; - if (row_policy_data.needCare()) + if (row_policy_data.hasRowPolicy()) { if (auto * source_step_with_filter = dynamic_cast((plan.getRootNode()->step.get()))) { @@ -880,7 +880,7 @@ void ReadFromMerge::RowPolicyData::init(RowPolicyFilterPtr row_policy_filter_ptr auto syntax_result = TreeRewriter(local_context).analyze(expr, needed_columns); auto expression_analyzer = ExpressionAnalyzer{expr, syntax_result, local_context}; - actions_dag = expression_analyzer.getActionsDAG(true, false); + actions_dag = expression_analyzer.getActionsDAG(false /* add_aliases */, false /* project_result */); filter_actions = std::make_shared(actions_dag, ExpressionActionsSettings::fromContext(local_context, CompileExpressions::yes)); filter_column_name = namesDifference(filter_actions->getSampleBlock().getNames(), filter_actions->getRequiredColumns()); @@ -897,16 +897,22 @@ void ReadFromMerge::RowPolicyData::extendNames(Names & names) RequiredSourceColumnsVisitor::Data columns_context; RequiredSourceColumnsVisitor(columns_context).visit(expr); - auto req_columns = columns_context.requiredColumns(); + const auto req_columns = columns_context.requiredColumns(); + + std::sort(names.begin(), names.end()); + NameSet added_names; + for (const auto & req_column : req_columns) { - std::sort(names.begin(), names.end()); - if (!std::binary_search(names.begin(), names.end(), req_column)) { - names.push_back(req_column); + added_names.insert(req_column); } } + if (!added_names.empty()) + { + std::copy(added_names.begin(), added_names.end(), std::back_inserter(names)); + } } void ReadFromMerge::RowPolicyData::addStorageFilter(SourceStepWithFilter * step) @@ -931,7 +937,7 @@ void ReadFromMerge::RowPolicyData::addFilterTransform(QueryPipelineBuilder & bui builder.addSimpleTransform([&](const Block & stream_header) { - return std::make_shared(stream_header, filter_actions, filter_column_name, true /* remove fake column */); + return std::make_shared(stream_header, filter_actions, filter_column_name, true /* remove filter column */); }); } @@ -1135,7 +1141,7 @@ void ReadFromMerge::convertingSourceStream( QueryProcessingStage::Enum processed_stage, RowPolicyData & row_policy_data) { - if (row_policy_data.needCare()) + if (row_policy_data.hasRowPolicy()) { row_policy_data.addFilterTransform(builder); } From 15c95ffb354801ed9e7ba91123273cd077939589 Mon Sep 17 00:00:00 2001 From: Ilya Golshtein Date: Mon, 16 Oct 2023 21:22:43 +0000 Subject: [PATCH 090/123] merge_row_policy: some changes per code review --- src/Storages/StorageMerge.cpp | 76 ++++++++++++++--------------------- src/Storages/StorageMerge.h | 3 +- 2 files changed, 32 insertions(+), 47 deletions(-) diff --git a/src/Storages/StorageMerge.cpp b/src/Storages/StorageMerge.cpp index 7baeaba94389..fc63940d6929 100644 --- a/src/Storages/StorageMerge.cpp +++ b/src/Storages/StorageMerge.cpp @@ -382,13 +382,7 @@ void StorageMerge::read( class ReadFromMerge::RowPolicyData { public: - /// Row policy requires extra filtering - bool hasRowPolicy() - { - return static_cast(row_policy_filter_ptr); - } - - void init(RowPolicyFilterPtr, std::shared_ptr, ContextPtr); + RowPolicyData(RowPolicyFilterPtr, std::shared_ptr, ContextPtr); /// Add columns that needed for row policies to data stream /// SELECT x from T if T has row policy y=42 @@ -687,7 +681,7 @@ QueryPipelineBuilderPtr ReadFromMerge::createSources( storage_snapshot, modified_query_info); - RowPolicyData row_policy_data; + std::optional row_policy_data; if (processed_stage <= storage_stage || (allow_experimental_analyzer && processed_stage == QueryProcessingStage::FetchColumns)) { @@ -700,19 +694,17 @@ QueryPipelineBuilderPtr ReadFromMerge::createSources( StorageView * view = dynamic_cast(storage.get()); if (!view || allow_experimental_analyzer) { - row_policy_data.init( - modified_context->getRowPolicyFilter( + auto row_policy_filter_ptr = modified_context->getRowPolicyFilter( database_name, table_name, - RowPolicyFilterType::SELECT_FILTER), - storage, - modified_context); - - if (row_policy_data.hasRowPolicy()) + RowPolicyFilterType::SELECT_FILTER); + if (row_policy_filter_ptr) { - row_policy_data.extendNames(real_column_names); + row_policy_data.emplace(row_policy_filter_ptr, storage, modified_context); + row_policy_data->extendNames(real_column_names); } + storage->read(plan, real_column_names, storage_snapshot, @@ -725,11 +717,11 @@ QueryPipelineBuilderPtr ReadFromMerge::createSources( if (!plan.isInitialized()) return {}; - if (row_policy_data.hasRowPolicy()) + if (row_policy_data) { if (auto * source_step_with_filter = dynamic_cast((plan.getRootNode()->step.get()))) { - row_policy_data.addStorageFilter(source_step_with_filter); + row_policy_data->addStorageFilter(source_step_with_filter); } } } @@ -847,6 +839,11 @@ QueryPipelineBuilderPtr ReadFromMerge::createSources( }); } + if (row_policy_data) + { + row_policy_data->addFilterTransform(*builder); + } + /// Subordinary tables could have different but convertible types, like numeric types of different width. /// We must return streams with structure equals to structure of Merge table. /// Besides this we add FilterTransform if it is needed to follow row level policies. @@ -856,42 +853,38 @@ QueryPipelineBuilderPtr ReadFromMerge::createSources( aliases, modified_context, *builder, - processed_stage, - row_policy_data); + processed_stage); } return builder; } -void ReadFromMerge::RowPolicyData::init(RowPolicyFilterPtr row_policy_filter_ptr_, +ReadFromMerge::RowPolicyData::RowPolicyData(RowPolicyFilterPtr row_policy_filter_ptr_, std::shared_ptr storage, ContextPtr local_context) + : row_policy_filter_ptr(row_policy_filter_ptr_) { - if (row_policy_filter_ptr_) - { - row_policy_filter_ptr = row_policy_filter_ptr_; + assert(row_policy_filter_ptr_); - ASTPtr expr = row_policy_filter_ptr->expression; + ASTPtr expr = row_policy_filter_ptr->expression; - auto storage_metadata_snapshot = storage->getInMemoryMetadataPtr(); - auto storage_columns = storage_metadata_snapshot->getColumns(); - auto needed_columns = storage_columns.getAllPhysical(); + auto storage_metadata_snapshot = storage->getInMemoryMetadataPtr(); + auto storage_columns = storage_metadata_snapshot->getColumns(); + auto needed_columns = storage_columns.getAllPhysical(); - auto syntax_result = TreeRewriter(local_context).analyze(expr, needed_columns); - auto expression_analyzer = ExpressionAnalyzer{expr, syntax_result, local_context}; + auto syntax_result = TreeRewriter(local_context).analyze(expr, needed_columns); + auto expression_analyzer = ExpressionAnalyzer{expr, syntax_result, local_context}; - actions_dag = expression_analyzer.getActionsDAG(false /* add_aliases */, false /* project_result */); - filter_actions = std::make_shared(actions_dag, - ExpressionActionsSettings::fromContext(local_context, CompileExpressions::yes)); - filter_column_name = namesDifference(filter_actions->getSampleBlock().getNames(), filter_actions->getRequiredColumns()); - } + actions_dag = expression_analyzer.getActionsDAG(false /* add_aliases */, false /* project_result */); + filter_actions = std::make_shared(actions_dag, + ExpressionActionsSettings::fromContext(local_context, CompileExpressions::yes)); + filter_column_name = namesDifference(filter_actions->getSampleBlock().getNames(), filter_actions->getRequiredColumns()); } // Add columns that needed to evaluate row policies // SELECT x from t if t has row policy void ReadFromMerge::RowPolicyData::extendNames(Names & names) { - assert(row_policy_filter_ptr); ASTPtr expr = row_policy_filter_ptr->expression; RequiredSourceColumnsVisitor::Data columns_context; @@ -899,12 +892,11 @@ void ReadFromMerge::RowPolicyData::extendNames(Names & names) const auto req_columns = columns_context.requiredColumns(); - std::sort(names.begin(), names.end()); NameSet added_names; for (const auto & req_column : req_columns) { - if (!std::binary_search(names.begin(), names.end(), req_column)) + if (std::find(names.begin(), names.end(), req_column) == names.end()) { added_names.insert(req_column); } @@ -1138,14 +1130,8 @@ void ReadFromMerge::convertingSourceStream( const Aliases & aliases, ContextPtr local_context, QueryPipelineBuilder & builder, - QueryProcessingStage::Enum processed_stage, - RowPolicyData & row_policy_data) + QueryProcessingStage::Enum processed_stage) { - if (row_policy_data.hasRowPolicy()) - { - row_policy_data.addFilterTransform(builder); - } - Block before_block_header = builder.getHeader(); auto storage_sample_block = metadata_snapshot->getSampleBlock(); diff --git a/src/Storages/StorageMerge.h b/src/Storages/StorageMerge.h index 1a5aca24e4c1..04c5a70651ff 100644 --- a/src/Storages/StorageMerge.h +++ b/src/Storages/StorageMerge.h @@ -203,8 +203,7 @@ class ReadFromMerge final : public SourceStepWithFilter const Aliases & aliases, ContextPtr context, QueryPipelineBuilder & builder, - QueryProcessingStage::Enum processed_stage, - RowPolicyData & row_policy_data); + QueryProcessingStage::Enum processed_stage); }; } From 8e1f090a0517e2a3edf77ad0fecded2a675ee691 Mon Sep 17 00:00:00 2001 From: Ilya Golshtein Date: Mon, 16 Oct 2023 21:49:16 +0000 Subject: [PATCH 091/123] merge_row_policy: a comment --- src/Storages/StorageMerge.cpp | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/Storages/StorageMerge.cpp b/src/Storages/StorageMerge.cpp index fc63940d6929..dc15897332df 100644 --- a/src/Storages/StorageMerge.cpp +++ b/src/Storages/StorageMerge.cpp @@ -881,8 +881,8 @@ ReadFromMerge::RowPolicyData::RowPolicyData(RowPolicyFilterPtr row_policy_filter filter_column_name = namesDifference(filter_actions->getSampleBlock().getNames(), filter_actions->getRequiredColumns()); } -// Add columns that needed to evaluate row policies -// SELECT x from t if t has row policy +// Add columns that needed _only_ to evaluate row policies +// SELECT x from t if t has row policy that is based on y void ReadFromMerge::RowPolicyData::extendNames(Names & names) { ASTPtr expr = row_policy_filter_ptr->expression; From 75ea62c2ebf68f46821aa1d80eda17a415b37ae7 Mon Sep 17 00:00:00 2001 From: Ilya Golshtein Date: Mon, 16 Oct 2023 22:16:53 +0000 Subject: [PATCH 092/123] merge_row_policy: get rid of RequiredSourceColumnsVisitor --- src/Storages/StorageMerge.cpp | 9 +-------- 1 file changed, 1 insertion(+), 8 deletions(-) diff --git a/src/Storages/StorageMerge.cpp b/src/Storages/StorageMerge.cpp index dc15897332df..5e6106030438 100644 --- a/src/Storages/StorageMerge.cpp +++ b/src/Storages/StorageMerge.cpp @@ -885,16 +885,9 @@ ReadFromMerge::RowPolicyData::RowPolicyData(RowPolicyFilterPtr row_policy_filter // SELECT x from t if t has row policy that is based on y void ReadFromMerge::RowPolicyData::extendNames(Names & names) { - ASTPtr expr = row_policy_filter_ptr->expression; - - RequiredSourceColumnsVisitor::Data columns_context; - RequiredSourceColumnsVisitor(columns_context).visit(expr); - - const auto req_columns = columns_context.requiredColumns(); - NameSet added_names; - for (const auto & req_column : req_columns) + for (const auto & req_column : filter_actions->getRequiredColumns()) { if (std::find(names.begin(), names.end(), req_column) == names.end()) { From 23e8c4b6c1d05ae72a936ab404f1c54960cb5262 Mon Sep 17 00:00:00 2001 From: Ilya Golshtein Date: Tue, 24 Oct 2023 13:09:57 +0000 Subject: [PATCH 093/123] merge_row_policy: cleanup --- src/Storages/StorageMerge.cpp | 88 +++++++++++++---------------------- 1 file changed, 32 insertions(+), 56 deletions(-) diff --git a/src/Storages/StorageMerge.cpp b/src/Storages/StorageMerge.cpp index 5e6106030438..f22511060df7 100644 --- a/src/Storages/StorageMerge.cpp +++ b/src/Storages/StorageMerge.cpp @@ -16,7 +16,6 @@ #include #include #include -#include #include #include #include @@ -46,7 +45,6 @@ #include #include -#include namespace { @@ -384,23 +382,23 @@ class ReadFromMerge::RowPolicyData public: RowPolicyData(RowPolicyFilterPtr, std::shared_ptr, ContextPtr); - /// Add columns that needed for row policies to data stream - /// SELECT x from T if T has row policy y=42 - /// required y in data pipeline + /// Add to data stream columns that are needed only for row policies + /// SELECT x from T if T has row policy y=42 + /// required y in data pipeline void extendNames(Names &); /// Use storage facilities to filter data - /// does not guarantee accuracy, but reduce number of rows + /// optimization + /// does not guarantee accuracy, but reduces number of rows void addStorageFilter(SourceStepWithFilter *); - /// Create explicit filter transform to stop + /// Create explicit filter transform to exclude /// rows that are not conform to row level policy void addFilterTransform(QueryPipelineBuilder &); private: - static std::string namesDifference(Names && outer_set, Names && inner_set); RowPolicyFilterPtr row_policy_filter_ptr; - std::string filter_column_name; // complex filer, may contain logic operations + std::string filter_column_name; // complex filter, may contain logic operations ActionsDAGPtr actions_dag; ExpressionActionsPtr filter_actions; }; @@ -704,7 +702,6 @@ QueryPipelineBuilderPtr ReadFromMerge::createSources( row_policy_data->extendNames(real_column_names); } - storage->read(plan, real_column_names, storage_snapshot, @@ -713,17 +710,6 @@ QueryPipelineBuilderPtr ReadFromMerge::createSources( processed_stage, max_block_size, UInt32(streams_num)); - - if (!plan.isInitialized()) - return {}; - - if (row_policy_data) - { - if (auto * source_step_with_filter = dynamic_cast((plan.getRootNode()->step.get()))) - { - row_policy_data->addStorageFilter(source_step_with_filter); - } - } } else { @@ -741,9 +727,17 @@ QueryPipelineBuilderPtr ReadFromMerge::createSources( view->getInMemoryMetadataPtr(), SelectQueryOptions(processed_stage)); interpreter.buildQueryPlan(plan); + } + + if (!plan.isInitialized()) + return {}; - if (!plan.isInitialized()) - return {}; + if (row_policy_data) + { + if (auto * source_step_with_filter = dynamic_cast((plan.getRootNode()->step.get()))) + { + row_policy_data->addStorageFilter(source_step_with_filter); + } } if (auto * read_from_merge_tree = typeid_cast(plan.getRootNode()->step.get())) @@ -846,14 +840,7 @@ QueryPipelineBuilderPtr ReadFromMerge::createSources( /// Subordinary tables could have different but convertible types, like numeric types of different width. /// We must return streams with structure equals to structure of Merge table. - /// Besides this we add FilterTransform if it is needed to follow row level policies. - - convertingSourceStream(header, - storage_snapshot->metadata, - aliases, - modified_context, - *builder, - processed_stage); + convertingSourceStream(header, storage_snapshot->metadata, aliases, modified_context, *builder, processed_stage); } return builder; @@ -878,18 +865,28 @@ ReadFromMerge::RowPolicyData::RowPolicyData(RowPolicyFilterPtr row_policy_filter actions_dag = expression_analyzer.getActionsDAG(false /* add_aliases */, false /* project_result */); filter_actions = std::make_shared(actions_dag, ExpressionActionsSettings::fromContext(local_context, CompileExpressions::yes)); - filter_column_name = namesDifference(filter_actions->getSampleBlock().getNames(), filter_actions->getRequiredColumns()); + const auto & required_columns = filter_actions->getRequiredColumnsWithTypes(); + const auto & sample_block_columns = filter_actions->getSampleBlock().getNamesAndTypesList(); + + NamesAndTypesList added, deleted; + sample_block_columns.getDifference(required_columns, added, deleted); + if (!deleted.empty() || added.size() != 1) + { + throw Exception(ErrorCodes::LOGICAL_ERROR, + "Cannot determine row level filter"); + } + + filter_column_name = added.getNames().front(); } -// Add columns that needed _only_ to evaluate row policies -// SELECT x from t if t has row policy that is based on y void ReadFromMerge::RowPolicyData::extendNames(Names & names) { + std::sort(names.begin(), names.end()); NameSet added_names; for (const auto & req_column : filter_actions->getRequiredColumns()) { - if (std::find(names.begin(), names.end(), req_column) == names.end()) + if (!std::binary_search(names.begin(), names.end(), req_column)) { added_names.insert(req_column); } @@ -926,27 +923,6 @@ void ReadFromMerge::RowPolicyData::addFilterTransform(QueryPipelineBuilder & bui }); } -/// Find out an item that in outer_set vector, but not in inner_set vector -std::string ReadFromMerge::RowPolicyData::namesDifference(Names && outer_set, Names && inner_set) -{ - std::sort(outer_set.begin(), outer_set.end()); - std::sort(inner_set.begin(), inner_set.end()); - - Names result; - - std::set_difference(outer_set.begin(), outer_set.end(), - inner_set.begin(), inner_set.end(), std::inserter(result, result.begin())); - - if (result.size() != 1) - { - throw Exception(ErrorCodes::LOGICAL_ERROR, - "Cannot determine row level filter"); - } - - return result.front(); -} - - StorageMerge::StorageListWithLocks StorageMerge::getSelectedTables( ContextPtr query_context, const ASTPtr & query /* = nullptr */, From b3d60be9debf99d612573d931f9100b8c9031075 Mon Sep 17 00:00:00 2001 From: Ilya Golshtein Date: Mon, 6 Nov 2023 15:18:25 +0300 Subject: [PATCH 094/123] merge_row_policy: aliases slightly work --- src/Storages/StorageMerge.cpp | 108 +++++++++++++----- src/Storages/StorageMerge.h | 2 + src/TableFunctions/TableFunctionMerge.cpp | 2 +- .../02763_row_policy_storage_merge.sql.j2 | 32 ++++++ 4 files changed, 117 insertions(+), 27 deletions(-) diff --git a/src/Storages/StorageMerge.cpp b/src/Storages/StorageMerge.cpp index f22511060df7..c1eca6829bd8 100644 --- a/src/Storages/StorageMerge.cpp +++ b/src/Storages/StorageMerge.cpp @@ -403,6 +403,8 @@ class ReadFromMerge::RowPolicyData ExpressionActionsPtr filter_actions; }; +// using RowPolicyDataPtr = std::unique_ptr; + ReadFromMerge::ReadFromMerge( Block common_header_, StorageListWithLocks selected_tables_, @@ -492,6 +494,22 @@ void ReadFromMerge::initializePipeline(QueryPipelineBuilder & pipeline, const Bu if (sampling_requested && !storage->supportsSampling()) throw Exception(ErrorCodes::SAMPLING_NOT_SUPPORTED, "Illegal SAMPLE: table doesn't support sampling"); + const auto & [database_name, _, _b, table_name] = table; + + std::unique_ptr row_policy_data_ptr; + + auto row_policy_filter_ptr = context->getRowPolicyFilter( + database_name, + table_name, + RowPolicyFilterType::SELECT_FILTER); + if (row_policy_filter_ptr) + { + row_policy_data_ptr = std::make_unique(row_policy_filter_ptr, storage, context); + row_policy_data_ptr->extendNames(column_names); + } + + + Aliases aliases; auto storage_metadata_snapshot = storage->getInMemoryMetadataPtr(); auto nested_storage_snaphsot = storage->getStorageSnapshot(storage_metadata_snapshot, context); @@ -529,6 +547,11 @@ void ReadFromMerge::initializePipeline(QueryPipelineBuilder & pipeline, const Bu auto type = sample_block.getByName(column).type; aliases.push_back({ .name = column, .type = type, .expression = column_expr->clone() }); + + LOG_TRACE(&Poco::Logger::get("ReadFromMerge::initializePipeline"), + "adding new alias name {}, expression {}", + column, column_expr->formatForLogging()); + } else column_expr = std::make_shared(column); @@ -542,6 +565,12 @@ void ReadFromMerge::initializePipeline(QueryPipelineBuilder & pipeline, const Bu auto alias_actions = ExpressionAnalyzer(required_columns_expr_list, syntax_result, context).getActionsDAG(true); column_names_as_aliases = alias_actions->getRequiredColumns().getNames(); + LOG_TRACE(&Poco::Logger::get("ReadFromMerge::initializePipeline"), + "alias_actions->getRequiredColumns: {}", alias_actions->getRequiredColumns().toString()); + + // if (row_policy_data_ptr) + // row_policy_data_ptr->extendNames(column_names_as_aliases); + if (column_names_as_aliases.empty()) column_names_as_aliases.push_back(ExpressionActions::getSmallestColumn(storage_metadata_snapshot->getColumns().getAllPhysical()).name); } @@ -556,6 +585,7 @@ void ReadFromMerge::initializePipeline(QueryPipelineBuilder & pipeline, const Bu aliases, table, column_names_as_aliases.empty() ? column_names : column_names_as_aliases, + std::move(row_policy_data_ptr), context, current_streams); @@ -657,6 +687,7 @@ QueryPipelineBuilderPtr ReadFromMerge::createSources( const Aliases & aliases, const StorageWithLockAndName & storage_with_lock, Names real_column_names, + std::unique_ptr row_policy_data_ptr, ContextMutablePtr modified_context, size_t streams_num, bool concat_streams) @@ -679,7 +710,7 @@ QueryPipelineBuilderPtr ReadFromMerge::createSources( storage_snapshot, modified_query_info); - std::optional row_policy_data; + // std::optional row_policy_data; if (processed_stage <= storage_stage || (allow_experimental_analyzer && processed_stage == QueryProcessingStage::FetchColumns)) { @@ -692,15 +723,15 @@ QueryPipelineBuilderPtr ReadFromMerge::createSources( StorageView * view = dynamic_cast(storage.get()); if (!view || allow_experimental_analyzer) { - auto row_policy_filter_ptr = modified_context->getRowPolicyFilter( - database_name, - table_name, - RowPolicyFilterType::SELECT_FILTER); - if (row_policy_filter_ptr) - { - row_policy_data.emplace(row_policy_filter_ptr, storage, modified_context); - row_policy_data->extendNames(real_column_names); - } + // auto row_policy_filter_ptr = modified_context->getRowPolicyFilter( + // database_name, + // table_name, + // RowPolicyFilterType::SELECT_FILTER); + // if (row_policy_data_ptr) + // { + // row_policy_data.emplace(row_policy_filter_ptr, storage, modified_context); + // row_policy_data_ptr->extendNames(real_column_names); + // } storage->read(plan, real_column_names, @@ -732,11 +763,11 @@ QueryPipelineBuilderPtr ReadFromMerge::createSources( if (!plan.isInitialized()) return {}; - if (row_policy_data) + if (row_policy_data_ptr) { if (auto * source_step_with_filter = dynamic_cast((plan.getRootNode()->step.get()))) { - row_policy_data->addStorageFilter(source_step_with_filter); + row_policy_data_ptr->addStorageFilter(source_step_with_filter); } } @@ -833,14 +864,15 @@ QueryPipelineBuilderPtr ReadFromMerge::createSources( }); } - if (row_policy_data) - { - row_policy_data->addFilterTransform(*builder); - } + // if (row_policy_data_ptr) + // { + // row_policy_data_ptr->addFilterTransform(*builder); + // } /// Subordinary tables could have different but convertible types, like numeric types of different width. /// We must return streams with structure equals to structure of Merge table. - convertingSourceStream(header, storage_snapshot->metadata, aliases, modified_context, *builder, processed_stage); + convertingSourceStream(header, storage_snapshot->metadata, aliases, std::move(row_policy_data_ptr), modified_context, *builder, processed_stage); + } return builder; @@ -851,15 +883,16 @@ ReadFromMerge::RowPolicyData::RowPolicyData(RowPolicyFilterPtr row_policy_filter ContextPtr local_context) : row_policy_filter_ptr(row_policy_filter_ptr_) { - assert(row_policy_filter_ptr_); - - ASTPtr expr = row_policy_filter_ptr->expression; - auto storage_metadata_snapshot = storage->getInMemoryMetadataPtr(); auto storage_columns = storage_metadata_snapshot->getColumns(); - auto needed_columns = storage_columns.getAllPhysical(); + auto needed_columns = storage_columns.getAll/*Physical*/(); + + ASTPtr expr = row_policy_filter_ptr->expression; - auto syntax_result = TreeRewriter(local_context).analyze(expr, needed_columns); + auto syntax_result = TreeRewriter(local_context).analyze(expr, + needed_columns /*, + storage, + storage->getStorageSnapshot(storage_metadata_snapshot, local_context)*/); auto expression_analyzer = ExpressionAnalyzer{expr, syntax_result, local_context}; actions_dag = expression_analyzer.getActionsDAG(false /* add_aliases */, false /* project_result */); @@ -894,12 +927,18 @@ void ReadFromMerge::RowPolicyData::extendNames(Names & names) if (!added_names.empty()) { std::copy(added_names.begin(), added_names.end(), std::back_inserter(names)); + LOG_TRACE(&Poco::Logger::get("ReadFromMerge::RowPolicyData::extendNames"), + "{} names added", added_names.size()); + // for (const auto & added_name : added_names) + // { + // LOG_TRACE(&Poco::Logger::get("ReadFromMerge::RowPolicyData::extendNames"), + // " added name {}", added_name); + // } } } void ReadFromMerge::RowPolicyData::addStorageFilter(SourceStepWithFilter * step) { - assert(row_policy_filter_ptr); LOG_TRACE(&Poco::Logger::get("ReadFromMerge::RowPolicyData::addStorageFilter"), "filter_actions_dag: {},<> {}, <> {}", filter_actions->getActionsDAG().dumpNames(), filter_actions->getActionsDAG().dumpDAG(), @@ -910,8 +949,6 @@ void ReadFromMerge::RowPolicyData::addStorageFilter(SourceStepWithFilter * step) void ReadFromMerge::RowPolicyData::addFilterTransform(QueryPipelineBuilder & builder) { - assert(row_policy_filter_ptr); - LOG_TRACE(&Poco::Logger::get("ReadFromMerge::RowPolicyData::addFilterTransform"), "filter_actions_dag: {},<> {}, <> {}", filter_actions->getActionsDAG().dumpNames(), filter_actions->getActionsDAG().dumpDAG(), @@ -919,6 +956,10 @@ void ReadFromMerge::RowPolicyData::addFilterTransform(QueryPipelineBuilder & bui builder.addSimpleTransform([&](const Block & stream_header) { + LOG_TRACE(&Poco::Logger::get("ReadFromMerge::RowPolicyData::addFilterTransform"), + "stream_header.dumpNames {}", stream_header.dumpNames()); + + return std::make_shared(stream_header, filter_actions, filter_column_name, true /* remove filter column */); }); } @@ -1097,6 +1138,7 @@ void ReadFromMerge::convertingSourceStream( const Block & header, const StorageMetadataPtr & metadata_snapshot, const Aliases & aliases, + std::unique_ptr row_policy_data_ptr, ContextPtr local_context, QueryPipelineBuilder & builder, QueryProcessingStage::Enum processed_stage) @@ -1123,11 +1165,24 @@ void ReadFromMerge::convertingSourceStream( }); } + + ActionsDAG::MatchColumnsMode convert_actions_match_columns_mode = ActionsDAG::MatchColumnsMode::Name; if (local_context->getSettingsRef().allow_experimental_analyzer && processed_stage != QueryProcessingStage::FetchColumns) convert_actions_match_columns_mode = ActionsDAG::MatchColumnsMode::Position; + LOG_TRACE(&Poco::Logger::get("ReadFromMerge::convertingSourceStream"), + "builder.getHeader(): {}, header.getColumnsWithTypeAndName: {}", + builder.getHeader().dumpStructure(), + header.dumpStructure()); + + + if (row_policy_data_ptr) + { + row_policy_data_ptr->addFilterTransform(builder); + } + auto convert_actions_dag = ActionsDAG::makeConvertingActions(builder.getHeader().getColumnsWithTypeAndName(), header.getColumnsWithTypeAndName(), convert_actions_match_columns_mode); @@ -1139,6 +1194,7 @@ void ReadFromMerge::convertingSourceStream( { return std::make_shared(stream_header, actions); }); + } bool ReadFromMerge::requestReadingInOrder(InputOrderInfoPtr order_info_) diff --git a/src/Storages/StorageMerge.h b/src/Storages/StorageMerge.h index 04c5a70651ff..80c5790d3197 100644 --- a/src/Storages/StorageMerge.h +++ b/src/Storages/StorageMerge.h @@ -193,6 +193,7 @@ class ReadFromMerge final : public SourceStepWithFilter const Aliases & aliases, const StorageWithLockAndName & storage_with_lock, Names real_column_names, + std::unique_ptr row_policy_data_ptr, ContextMutablePtr modified_context, size_t streams_num, bool concat_streams = false); @@ -201,6 +202,7 @@ class ReadFromMerge final : public SourceStepWithFilter const Block & header, const StorageMetadataPtr & metadata_snapshot, const Aliases & aliases, + std::unique_ptr row_policy_data_ptr, ContextPtr context, QueryPipelineBuilder & builder, QueryProcessingStage::Enum processed_stage); diff --git a/src/TableFunctions/TableFunctionMerge.cpp b/src/TableFunctions/TableFunctionMerge.cpp index 599953a1add7..5231c651f173 100644 --- a/src/TableFunctions/TableFunctionMerge.cpp +++ b/src/TableFunctions/TableFunctionMerge.cpp @@ -126,7 +126,7 @@ ColumnsDescription TableFunctionMerge::getActualTableStructure(ContextPtr contex { auto storage = DatabaseCatalog::instance().tryGetTable(StorageID{db_with_tables.first, table}, context); if (storage) - return ColumnsDescription{storage->getInMemoryMetadataPtr()->getColumns().getAllPhysical()}; + return ColumnsDescription{storage->getInMemoryMetadataPtr()->getColumns().getAllPhysical()}; // !!! } } diff --git a/tests/queries/0_stateless/02763_row_policy_storage_merge.sql.j2 b/tests/queries/0_stateless/02763_row_policy_storage_merge.sql.j2 index 9b3197cc34fe..0bc1292d4a5f 100644 --- a/tests/queries/0_stateless/02763_row_policy_storage_merge.sql.j2 +++ b/tests/queries/0_stateless/02763_row_policy_storage_merge.sql.j2 @@ -3,6 +3,7 @@ DROP TABLE IF EXISTS 02763_merge_log_2; DROP TABLE IF EXISTS 02763_merge_merge_1; DROP TABLE IF EXISTS 02763_merge_merge_2; DROP TABLE IF EXISTS 02763_merge_fancycols; +DROP TABLE IF EXISTS 02763_merge_aliases; DROP ROW POLICY IF EXISTS 02763_filter_1 ON 02763_merge_log_1; DROP ROW POLICY IF EXISTS 02763_filter_2 ON 02763_merge_merge_1; DROP ROW POLICY IF EXISTS 02763_filter_3 ON 02763_merge_log_1; @@ -17,6 +18,8 @@ CREATE TABLE 02763_merge_log_2 (x UInt8, y UInt64) ENGINE = Log; CREATE TABLE 02763_merge_merge_1 (x UInt8, y UInt64) ENGINE = MergeTree ORDER BY x; CREATE TABLE 02763_merge_merge_2 (x UInt8, y UInt64) ENGINE = MergeTree ORDER BY x; +CREATE TABLE 02763_engine_merge_12 (x UInt8, y UInt64) ENGINE = Merge(currentDatabase(), '02763_merge_merge'); + INSERT INTO 02763_merge_log_1 VALUES (1, 11), (2, 12), (3, 13), (4, 14); INSERT INTO 02763_merge_log_2 VALUES (1, 11), (2, 12), (3, 13), (4, 14); INSERT INTO 02763_merge_merge_1 VALUES (1, 11), (2, 12), (3, 13), (4, 14); @@ -56,6 +59,10 @@ SELECT 'SELECT * FROM merge(currentDatabase(), 02763_merge_merge) WHERE x>2'; SELECT * FROM merge(currentDatabase(), '02763_merge_merge') WHERE x>2 ORDER BY x SETTINGS optimize_move_to_prewhere= {{prew}}; +SELECT 'SELECT * FROM engine_merge_12 WHERE x>2'; +SELECT * FROM 02763_engine_merge_12 WHERE x>2 ORDER BY x SETTINGS optimize_move_to_prewhere= {{prew}}; + + SELECT 'SELECT * FROM merge(currentDatabase(), 02763_merge)'; SELECT * FROM merge(currentDatabase(), '02763_merge') ORDER BY x SETTINGS optimize_move_to_prewhere= {{prew}}; SELECT 'SELECT * FROM merge(currentDatabase(), 02763_merge) WHERE x>2'; @@ -125,6 +132,29 @@ SELECT x, y from merge(currentDatabase(), '02763_merge') ORDER BY x; DROP TABLE 02763_merge_fancycols; +SELECT '02763_merge_aliases'; +CREATE TABLE 02763_alias (x UInt8, y UInt64, z UInt64 ALIAS plus(x,y)) ENGINE = MergeTree ORDER BY x; +INSERT INTO 02763_alias VALUES (1, 11), (2, 12), (3, 13), (4, 14); + +CREATE ROW POLICY 02763_filter_7 ON 02763_alias USING z>15 AS permissive TO ALL; + +-- SELECT 'SELECT * FROM 02763_alias ORDER BY x'; +-- SELECT x, y, z FROM 02763_alias ORDER BY x; + +CREATE TABLE 02763_a_merge (x UInt8, y UInt64, z UInt64) ENGINE = Merge(currentDatabase(), '02763_alias'); + + +-- SELECT 'SELECT * FROM merge(currentDatabase(), 02763_merge_alias) ORDER BY x'; +-- SELECT * FROM merge(currentDatabase(), '02763_merge_alias') ORDER BY x; + +SELECT 'x, y, z FROM 02763_a_merge ORDER BY x'; +SELECT x, y, z FROM 02763_a_merge ORDER BY x; +-- SELECT 'SELECT x, y, z FROM merge(currentDatabase(), 02763_merge_alias) ORDER BY x'; +-- SELECT x, y, z FROM merge(currentDatabase(), '02763_merge_alias') ORDER BY x; + +DROP TABLE 02763_alias; +DROP TABLE 02763_a_merge; + DROP ROW POLICY 02763_filter_1 ON 02763_merge_log_1; DROP ROW POLICY 02763_filter_2 ON 02763_merge_merge_1; @@ -134,4 +164,6 @@ DROP ROW POLICY 02763_filter_4 ON 02763_merge_merge_1; DROP ROW POLICY 02763_filter_5 ON 02763_merge_fancycols; DROP ROW POLICY 02763_filter_6 ON 02763_merge_fancycols; +DROP ROW POLICY 02763_filter_7 ON 02763_alias; + {% endfor %} From 697db7c2ec43bca52a976a182857036b981620a6 Mon Sep 17 00:00:00 2001 From: Ilya Golshtein Date: Wed, 8 Nov 2023 18:55:19 +0300 Subject: [PATCH 095/123] merge_row_policy: processAliases introduced --- src/Storages/StorageMerge.cpp | 279 ++++++++++++++++------ src/Storages/StorageMerge.h | 16 +- src/TableFunctions/TableFunctionMerge.cpp | 2 +- 3 files changed, 213 insertions(+), 84 deletions(-) diff --git a/src/Storages/StorageMerge.cpp b/src/Storages/StorageMerge.cpp index c1eca6829bd8..d94601906774 100644 --- a/src/Storages/StorageMerge.cpp +++ b/src/Storages/StorageMerge.cpp @@ -385,7 +385,7 @@ class ReadFromMerge::RowPolicyData /// Add to data stream columns that are needed only for row policies /// SELECT x from T if T has row policy y=42 /// required y in data pipeline - void extendNames(Names &); + void extendNames(Names &, bool alias_allowed = true); /// Use storage facilities to filter data /// optimization @@ -401,6 +401,7 @@ class ReadFromMerge::RowPolicyData std::string filter_column_name; // complex filter, may contain logic operations ActionsDAGPtr actions_dag; ExpressionActionsPtr filter_actions; + StorageMetadataPtr storage_metadata_snapshot; }; // using RowPolicyDataPtr = std::unique_ptr; @@ -472,7 +473,7 @@ void ReadFromMerge::initializePipeline(QueryPipelineBuilder & pipeline, const Bu query_info.input_order_info = input_sorting_info; } - auto sample_block = merge_storage_snapshot->getMetadataForQuery()->getSampleBlock(); + // auto sample_block = merge_storage_snapshot->getMetadataForQuery()->getSampleBlock(); std::vector> pipelines; QueryPlanResourceHolder resources; @@ -494,98 +495,100 @@ void ReadFromMerge::initializePipeline(QueryPipelineBuilder & pipeline, const Bu if (sampling_requested && !storage->supportsSampling()) throw Exception(ErrorCodes::SAMPLING_NOT_SUPPORTED, "Illegal SAMPLE: table doesn't support sampling"); - const auto & [database_name, _, _b, table_name] = table; + // const auto & [database_name, _, _b, table_name] = table; - std::unique_ptr row_policy_data_ptr; + // std::unique_ptr row_policy_data_ptr; - auto row_policy_filter_ptr = context->getRowPolicyFilter( - database_name, - table_name, - RowPolicyFilterType::SELECT_FILTER); - if (row_policy_filter_ptr) - { - row_policy_data_ptr = std::make_unique(row_policy_filter_ptr, storage, context); - row_policy_data_ptr->extendNames(column_names); - } + // auto row_policy_filter_ptr = context->getRowPolicyFilter( + // database_name, + // table_name, + // RowPolicyFilterType::SELECT_FILTER); + // if (row_policy_filter_ptr) + // { + // row_policy_data_ptr = std::make_unique(row_policy_filter_ptr, storage, context); + // row_policy_data_ptr->extendNames(column_names); + // } - Aliases aliases; - auto storage_metadata_snapshot = storage->getInMemoryMetadataPtr(); - auto nested_storage_snaphsot = storage->getStorageSnapshot(storage_metadata_snapshot, context); + // Aliases aliases; + // auto storage_metadata_snapshot = storage->getInMemoryMetadataPtr(); + // auto nested_storage_snaphsot = storage->getStorageSnapshot(storage_metadata_snapshot, context); - auto modified_query_info = getModifiedQueryInfo(query_info, context, table, nested_storage_snaphsot); - Names column_names_as_aliases; + // auto modified_query_info = getModifiedQueryInfo(query_info, context, table, nested_storage_snaphsot); + // Names column_names_as_aliases; - if (!context->getSettingsRef().allow_experimental_analyzer) - { - auto storage_columns = storage_metadata_snapshot->getColumns(); - auto syntax_result = TreeRewriter(context).analyzeSelect( - modified_query_info.query, TreeRewriterResult({}, storage, nested_storage_snaphsot)); + // if (!context->getSettingsRef().allow_experimental_analyzer) + // { + // auto storage_columns = storage_metadata_snapshot->getColumns(); + // auto syntax_result = TreeRewriter(context).analyzeSelect( + // modified_query_info.query, TreeRewriterResult({}, storage, nested_storage_snaphsot)); - bool with_aliases = common_processed_stage == QueryProcessingStage::FetchColumns && !storage_columns.getAliases().empty(); - if (with_aliases) - { - ASTPtr required_columns_expr_list = std::make_shared(); - ASTPtr column_expr; + // bool with_aliases = common_processed_stage == QueryProcessingStage::FetchColumns && !storage_columns.getAliases().empty(); + // if (with_aliases) + // { + // ASTPtr required_columns_expr_list = std::make_shared(); + // ASTPtr column_expr; - for (const auto & column : column_names) - { - const auto column_default = storage_columns.getDefault(column); - bool is_alias = column_default && column_default->kind == ColumnDefaultKind::Alias; + // for (const auto & column : column_names) + // { + // const auto column_default = storage_columns.getDefault(column); + // bool is_alias = column_default && column_default->kind == ColumnDefaultKind::Alias; - if (is_alias) - { - column_expr = column_default->expression->clone(); - replaceAliasColumnsInQuery(column_expr, storage_metadata_snapshot->getColumns(), - syntax_result->array_join_result_to_source, context); + // if (is_alias) + // { + // column_expr = column_default->expression->clone(); + // replaceAliasColumnsInQuery(column_expr, storage_metadata_snapshot->getColumns(), + // syntax_result->array_join_result_to_source, context); - auto column_description = storage_columns.get(column); - column_expr = addTypeConversionToAST(std::move(column_expr), column_description.type->getName(), - storage_metadata_snapshot->getColumns().getAll(), context); - column_expr = setAlias(column_expr, column); + // const auto & column_description = storage_columns.get(column); + // column_expr = addTypeConversionToAST(std::move(column_expr), column_description.type->getName(), + // storage_metadata_snapshot->getColumns().getAll(), context); + // column_expr = setAlias(column_expr, column); - auto type = sample_block.getByName(column).type; - aliases.push_back({ .name = column, .type = type, .expression = column_expr->clone() }); + // auto type = sample_block.getByName(column).type; + // aliases.push_back({ .name = column, .type = type, .expression = column_expr->clone() }); - LOG_TRACE(&Poco::Logger::get("ReadFromMerge::initializePipeline"), - "adding new alias name {}, expression {}", - column, column_expr->formatForLogging()); + // LOG_TRACE(&Poco::Logger::get("ReadFromMerge::initializePipeline"), + // "adding new alias name {}, expression {}", + // column, column_expr->formatForLogging()); - } - else - column_expr = std::make_shared(column); + // } + // else + // column_expr = std::make_shared(column); - required_columns_expr_list->children.emplace_back(std::move(column_expr)); - } + // required_columns_expr_list->children.emplace_back(std::move(column_expr)); + // } - syntax_result = TreeRewriter(context).analyze( - required_columns_expr_list, storage_columns.getAllPhysical(), storage, storage->getStorageSnapshot(storage_metadata_snapshot, context)); + // syntax_result = TreeRewriter(context).analyze( + // required_columns_expr_list, storage_columns.getAllPhysical(), storage, storage->getStorageSnapshot(storage_metadata_snapshot, context)); - auto alias_actions = ExpressionAnalyzer(required_columns_expr_list, syntax_result, context).getActionsDAG(true); + // auto alias_actions = ExpressionAnalyzer(required_columns_expr_list, syntax_result, context).getActionsDAG(true); - column_names_as_aliases = alias_actions->getRequiredColumns().getNames(); - LOG_TRACE(&Poco::Logger::get("ReadFromMerge::initializePipeline"), - "alias_actions->getRequiredColumns: {}", alias_actions->getRequiredColumns().toString()); + // column_names_as_aliases = alias_actions->getRequiredColumns().getNames(); + // LOG_TRACE(&Poco::Logger::get("ReadFromMerge::initializePipeline"), + // "alias_actions->getRequiredColumns: {}", alias_actions->getRequiredColumns().toString()); - // if (row_policy_data_ptr) - // row_policy_data_ptr->extendNames(column_names_as_aliases); + // // if (row_policy_data_ptr) + // // row_policy_data_ptr->extendNames(column_names_as_aliases, false /* alias_allowed */); - if (column_names_as_aliases.empty()) - column_names_as_aliases.push_back(ExpressionActions::getSmallestColumn(storage_metadata_snapshot->getColumns().getAllPhysical()).name); - } - } + // if (column_names_as_aliases.empty()) + // column_names_as_aliases.push_back(ExpressionActions::getSmallestColumn(storage_metadata_snapshot->getColumns().getAllPhysical()).name); + // } + // } auto source_pipeline = createSources( - nested_storage_snaphsot, - modified_query_info, + // nested_storage_snaphsot, + // modified_query_info, common_processed_stage, required_max_block_size, common_header, - aliases, + // aliases, table, - column_names_as_aliases.empty() ? column_names : column_names_as_aliases, - std::move(row_policy_data_ptr), + // column_names_as_aliases.empty() ? column_names : column_names_as_aliases, + column_names, + merge_storage_snapshot->getMetadataForQuery()->getSampleBlock(), + // std::move(row_policy_data_ptr), context, current_streams); @@ -678,23 +681,128 @@ SelectQueryInfo ReadFromMerge::getModifiedQueryInfo(const SelectQueryInfo & quer return modified_query_info; } +void ReadFromMerge::processAliases( + Names & real_column_names, + const StorageWithLockAndName & storage_with_lock, + Aliases & aliases, + const Block & sample_block, + ContextMutablePtr modified_context) +{ + auto storage = std::get<1>(storage_with_lock); + auto storage_metadata_snapshot = storage->getInMemoryMetadataPtr(); + auto nested_storage_snaphsot = storage->getStorageSnapshot(storage_metadata_snapshot, modified_context); + + auto modified_query_info = getModifiedQueryInfo(query_info, context, storage_with_lock, nested_storage_snaphsot); + Names column_names_as_aliases; + + if (!context->getSettingsRef().allow_experimental_analyzer) + { + auto storage_columns = storage_metadata_snapshot->getColumns(); + auto syntax_result = TreeRewriter(context).analyzeSelect( + modified_query_info.query, TreeRewriterResult({}, storage, nested_storage_snaphsot)); + + bool with_aliases = common_processed_stage == QueryProcessingStage::FetchColumns && !storage_columns.getAliases().empty(); + if (with_aliases) + { + ASTPtr required_columns_expr_list = std::make_shared(); + ASTPtr column_expr; + + for (const auto & column : real_column_names) + { + const auto column_default = storage_columns.getDefault(column); + bool is_alias = column_default && column_default->kind == ColumnDefaultKind::Alias; + + if (is_alias) + { + column_expr = column_default->expression->clone(); + replaceAliasColumnsInQuery(column_expr, storage_metadata_snapshot->getColumns(), + syntax_result->array_join_result_to_source, context); + + const auto & column_description = storage_columns.get(column); + column_expr = addTypeConversionToAST(std::move(column_expr), column_description.type->getName(), + storage_metadata_snapshot->getColumns().getAll(), context); + column_expr = setAlias(column_expr, column); + + auto type = sample_block.getByName(column).type; + aliases.push_back({ .name = column, .type = type, .expression = column_expr->clone() }); + + LOG_TRACE(&Poco::Logger::get("ReadFromMerge::initializePipeline"), + "adding new alias name {}, expression {}", + column, column_expr->formatForLogging()); + + } + else + column_expr = std::make_shared(column); + + required_columns_expr_list->children.emplace_back(std::move(column_expr)); + } + + syntax_result = TreeRewriter(context).analyze( + required_columns_expr_list, storage_columns.getAllPhysical(), storage, storage->getStorageSnapshot(storage_metadata_snapshot, context)); + + auto alias_actions = ExpressionAnalyzer(required_columns_expr_list, syntax_result, context).getActionsDAG(true); + + column_names_as_aliases = alias_actions->getRequiredColumns().getNames(); + LOG_TRACE(&Poco::Logger::get("ReadFromMerge::processAliases"), + "alias_actions->getRequiredColumns: {}", alias_actions->getRequiredColumns().toString()); + + // if (row_policy_data_ptr) + // row_policy_data_ptr->extendNames(column_names_as_aliases, false /* alias_allowed */); + + if (column_names_as_aliases.empty()) + column_names_as_aliases.push_back(ExpressionActions::getSmallestColumn(storage_metadata_snapshot->getColumns().getAllPhysical()).name); + } + } + if (!column_names_as_aliases.empty()) + { + LOG_TRACE(&Poco::Logger::get("ReadFromMerge::processAliases"), + "substitute real_column_names by column_names_as_aliases"); + real_column_names = column_names_as_aliases; + } +} + + QueryPipelineBuilderPtr ReadFromMerge::createSources( - const StorageSnapshotPtr & storage_snapshot, - SelectQueryInfo & modified_query_info, + // const StorageSnapshotPtr & storage_snapshot, + // SelectQueryInfo & modified_query_info, QueryProcessingStage::Enum processed_stage, UInt64 max_block_size, const Block & header, - const Aliases & aliases, const StorageWithLockAndName & storage_with_lock, Names real_column_names, - std::unique_ptr row_policy_data_ptr, + const Block & sample_block, + // std::unique_ptr row_policy_data_ptr, ContextMutablePtr modified_context, size_t streams_num, bool concat_streams) { const auto & [database_name, storage, _, table_name] = storage_with_lock; + auto storage_metadata_snapshot = storage->getInMemoryMetadataPtr(); + auto storage_snapshot = storage->getStorageSnapshot(storage_metadata_snapshot, context); + auto modified_query_info = getModifiedQueryInfo(query_info, context, storage_with_lock, storage_snapshot); + auto & modified_select = modified_query_info.query->as(); + std::unique_ptr row_policy_data_ptr; + + auto row_policy_filter_ptr = context->getRowPolicyFilter( + database_name, + table_name, + RowPolicyFilterType::SELECT_FILTER); + if (row_policy_filter_ptr) + { + row_policy_data_ptr = std::make_unique(row_policy_filter_ptr, storage, context); + row_policy_data_ptr->extendNames(real_column_names); + } + + + Aliases aliases; + processAliases(real_column_names, storage_with_lock, aliases, sample_block, modified_context); + // if (row_policy_data_ptr) + // { + // row_policy_data_ptr->extendNames(real_column_names); + // } + QueryPipelineBuilderPtr builder; if (!InterpreterSelectQuery::isQueryWithFinal(modified_query_info) && storage->needRewriteQueryWithFinal(real_column_names)) { @@ -883,7 +991,10 @@ ReadFromMerge::RowPolicyData::RowPolicyData(RowPolicyFilterPtr row_policy_filter ContextPtr local_context) : row_policy_filter_ptr(row_policy_filter_ptr_) { - auto storage_metadata_snapshot = storage->getInMemoryMetadataPtr(); + LOG_TRACE(&Poco::Logger::get("ReadFromMerge::RowPolicyData ctor"), + "storage {}", storage->getName()); + + storage_metadata_snapshot = storage->getInMemoryMetadataPtr(); auto storage_columns = storage_metadata_snapshot->getColumns(); auto needed_columns = storage_columns.getAll/*Physical*/(); @@ -912,7 +1023,7 @@ ReadFromMerge::RowPolicyData::RowPolicyData(RowPolicyFilterPtr row_policy_filter filter_column_name = added.getNames().front(); } -void ReadFromMerge::RowPolicyData::extendNames(Names & names) +void ReadFromMerge::RowPolicyData::extendNames(Names & names, bool alias_allowed) { std::sort(names.begin(), names.end()); NameSet added_names; @@ -921,6 +1032,16 @@ void ReadFromMerge::RowPolicyData::extendNames(Names & names) { if (!std::binary_search(names.begin(), names.end(), req_column)) { + if (!alias_allowed) + { + auto storage_columns = storage_metadata_snapshot->getColumns(); + const auto column_default = storage_columns.getDefault(req_column); + bool is_alias = column_default && column_default->kind == ColumnDefaultKind::Alias; + if (is_alias) + { + continue; + } + } added_names.insert(req_column); } } @@ -929,11 +1050,11 @@ void ReadFromMerge::RowPolicyData::extendNames(Names & names) std::copy(added_names.begin(), added_names.end(), std::back_inserter(names)); LOG_TRACE(&Poco::Logger::get("ReadFromMerge::RowPolicyData::extendNames"), "{} names added", added_names.size()); - // for (const auto & added_name : added_names) - // { - // LOG_TRACE(&Poco::Logger::get("ReadFromMerge::RowPolicyData::extendNames"), - // " added name {}", added_name); - // } + for (const auto & added_name : added_names) + { + LOG_TRACE(&Poco::Logger::get("ReadFromMerge::RowPolicyData::extendNames"), + " added name {}", added_name); + } } } diff --git a/src/Storages/StorageMerge.h b/src/Storages/StorageMerge.h index 80c5790d3197..350d871359a9 100644 --- a/src/Storages/StorageMerge.h +++ b/src/Storages/StorageMerge.h @@ -184,16 +184,24 @@ class ReadFromMerge final : public SourceStepWithFilter const StorageWithLockAndName & storage_with_lock_and_name, const StorageSnapshotPtr & storage_snapshot); + void processAliases( + Names & real_column_names, + const StorageWithLockAndName & storage_with_lock, + Aliases & aliases, + const Block & sample_block, + ContextMutablePtr modified_context); + QueryPipelineBuilderPtr createSources( - const StorageSnapshotPtr & storage_snapshot, - SelectQueryInfo & query_info, + // const StorageSnapshotPtr & storage_snapshot, + // SelectQueryInfo & query_info, QueryProcessingStage::Enum processed_stage, UInt64 max_block_size, const Block & header, - const Aliases & aliases, + // const Aliases & aliases, const StorageWithLockAndName & storage_with_lock, Names real_column_names, - std::unique_ptr row_policy_data_ptr, + const Block & sample_block, + // std::unique_ptr row_policy_data_ptr, ContextMutablePtr modified_context, size_t streams_num, bool concat_streams = false); diff --git a/src/TableFunctions/TableFunctionMerge.cpp b/src/TableFunctions/TableFunctionMerge.cpp index 5231c651f173..d04575041442 100644 --- a/src/TableFunctions/TableFunctionMerge.cpp +++ b/src/TableFunctions/TableFunctionMerge.cpp @@ -126,7 +126,7 @@ ColumnsDescription TableFunctionMerge::getActualTableStructure(ContextPtr contex { auto storage = DatabaseCatalog::instance().tryGetTable(StorageID{db_with_tables.first, table}, context); if (storage) - return ColumnsDescription{storage->getInMemoryMetadataPtr()->getColumns().getAllPhysical()}; // !!! + return ColumnsDescription{storage->getInMemoryMetadataPtr()->getColumns().getAll/* Physical*/()}; // !!! } } From c513c2e54b93dfac798c76d8f2be2318cf13d682 Mon Sep 17 00:00:00 2001 From: Ilya Golshtein Date: Wed, 8 Nov 2023 23:07:08 +0300 Subject: [PATCH 096/123] merge_row_policy: cleanup, traces removed --- src/Storages/StorageMerge.cpp | 191 ++---------------- src/Storages/StorageMerge.h | 2 +- src/TableFunctions/TableFunctionMerge.cpp | 2 +- .../02763_row_policy_storage_merge.reference | 58 ++++++ .../02763_row_policy_storage_merge.sql.j2 | 38 ++-- 5 files changed, 100 insertions(+), 191 deletions(-) diff --git a/src/Storages/StorageMerge.cpp b/src/Storages/StorageMerge.cpp index d94601906774..5412a87fa01e 100644 --- a/src/Storages/StorageMerge.cpp +++ b/src/Storages/StorageMerge.cpp @@ -385,7 +385,7 @@ class ReadFromMerge::RowPolicyData /// Add to data stream columns that are needed only for row policies /// SELECT x from T if T has row policy y=42 /// required y in data pipeline - void extendNames(Names &, bool alias_allowed = true); + void extendNames(Names &); /// Use storage facilities to filter data /// optimization @@ -495,100 +495,13 @@ void ReadFromMerge::initializePipeline(QueryPipelineBuilder & pipeline, const Bu if (sampling_requested && !storage->supportsSampling()) throw Exception(ErrorCodes::SAMPLING_NOT_SUPPORTED, "Illegal SAMPLE: table doesn't support sampling"); - // const auto & [database_name, _, _b, table_name] = table; - - // std::unique_ptr row_policy_data_ptr; - - // auto row_policy_filter_ptr = context->getRowPolicyFilter( - // database_name, - // table_name, - // RowPolicyFilterType::SELECT_FILTER); - // if (row_policy_filter_ptr) - // { - // row_policy_data_ptr = std::make_unique(row_policy_filter_ptr, storage, context); - // row_policy_data_ptr->extendNames(column_names); - // } - - - - // Aliases aliases; - // auto storage_metadata_snapshot = storage->getInMemoryMetadataPtr(); - // auto nested_storage_snaphsot = storage->getStorageSnapshot(storage_metadata_snapshot, context); - - // auto modified_query_info = getModifiedQueryInfo(query_info, context, table, nested_storage_snaphsot); - // Names column_names_as_aliases; - - // if (!context->getSettingsRef().allow_experimental_analyzer) - // { - // auto storage_columns = storage_metadata_snapshot->getColumns(); - // auto syntax_result = TreeRewriter(context).analyzeSelect( - // modified_query_info.query, TreeRewriterResult({}, storage, nested_storage_snaphsot)); - - // bool with_aliases = common_processed_stage == QueryProcessingStage::FetchColumns && !storage_columns.getAliases().empty(); - // if (with_aliases) - // { - // ASTPtr required_columns_expr_list = std::make_shared(); - // ASTPtr column_expr; - - // for (const auto & column : column_names) - // { - // const auto column_default = storage_columns.getDefault(column); - // bool is_alias = column_default && column_default->kind == ColumnDefaultKind::Alias; - - // if (is_alias) - // { - // column_expr = column_default->expression->clone(); - // replaceAliasColumnsInQuery(column_expr, storage_metadata_snapshot->getColumns(), - // syntax_result->array_join_result_to_source, context); - - // const auto & column_description = storage_columns.get(column); - // column_expr = addTypeConversionToAST(std::move(column_expr), column_description.type->getName(), - // storage_metadata_snapshot->getColumns().getAll(), context); - // column_expr = setAlias(column_expr, column); - - // auto type = sample_block.getByName(column).type; - // aliases.push_back({ .name = column, .type = type, .expression = column_expr->clone() }); - - // LOG_TRACE(&Poco::Logger::get("ReadFromMerge::initializePipeline"), - // "adding new alias name {}, expression {}", - // column, column_expr->formatForLogging()); - - // } - // else - // column_expr = std::make_shared(column); - - // required_columns_expr_list->children.emplace_back(std::move(column_expr)); - // } - - // syntax_result = TreeRewriter(context).analyze( - // required_columns_expr_list, storage_columns.getAllPhysical(), storage, storage->getStorageSnapshot(storage_metadata_snapshot, context)); - - // auto alias_actions = ExpressionAnalyzer(required_columns_expr_list, syntax_result, context).getActionsDAG(true); - - // column_names_as_aliases = alias_actions->getRequiredColumns().getNames(); - // LOG_TRACE(&Poco::Logger::get("ReadFromMerge::initializePipeline"), - // "alias_actions->getRequiredColumns: {}", alias_actions->getRequiredColumns().toString()); - - // // if (row_policy_data_ptr) - // // row_policy_data_ptr->extendNames(column_names_as_aliases, false /* alias_allowed */); - - // if (column_names_as_aliases.empty()) - // column_names_as_aliases.push_back(ExpressionActions::getSmallestColumn(storage_metadata_snapshot->getColumns().getAllPhysical()).name); - // } - // } - auto source_pipeline = createSources( - // nested_storage_snaphsot, - // modified_query_info, common_processed_stage, required_max_block_size, common_header, - // aliases, table, - // column_names_as_aliases.empty() ? column_names : column_names_as_aliases, column_names, merge_storage_snapshot->getMetadataForQuery()->getSampleBlock(), - // std::move(row_policy_data_ptr), context, current_streams); @@ -723,13 +636,9 @@ void ReadFromMerge::processAliases( storage_metadata_snapshot->getColumns().getAll(), context); column_expr = setAlias(column_expr, column); - auto type = sample_block.getByName(column).type; - aliases.push_back({ .name = column, .type = type, .expression = column_expr->clone() }); - - LOG_TRACE(&Poco::Logger::get("ReadFromMerge::initializePipeline"), - "adding new alias name {}, expression {}", - column, column_expr->formatForLogging()); + auto type = sample_block.has(column) ? sample_block.getByName(column).type : column_description.type; + aliases.push_back({ .name = column, .type = type, .expression = column_expr->clone() }); } else column_expr = std::make_shared(column); @@ -743,11 +652,6 @@ void ReadFromMerge::processAliases( auto alias_actions = ExpressionAnalyzer(required_columns_expr_list, syntax_result, context).getActionsDAG(true); column_names_as_aliases = alias_actions->getRequiredColumns().getNames(); - LOG_TRACE(&Poco::Logger::get("ReadFromMerge::processAliases"), - "alias_actions->getRequiredColumns: {}", alias_actions->getRequiredColumns().toString()); - - // if (row_policy_data_ptr) - // row_policy_data_ptr->extendNames(column_names_as_aliases, false /* alias_allowed */); if (column_names_as_aliases.empty()) column_names_as_aliases.push_back(ExpressionActions::getSmallestColumn(storage_metadata_snapshot->getColumns().getAllPhysical()).name); @@ -755,23 +659,18 @@ void ReadFromMerge::processAliases( } if (!column_names_as_aliases.empty()) { - LOG_TRACE(&Poco::Logger::get("ReadFromMerge::processAliases"), - "substitute real_column_names by column_names_as_aliases"); real_column_names = column_names_as_aliases; } } QueryPipelineBuilderPtr ReadFromMerge::createSources( - // const StorageSnapshotPtr & storage_snapshot, - // SelectQueryInfo & modified_query_info, QueryProcessingStage::Enum processed_stage, UInt64 max_block_size, const Block & header, const StorageWithLockAndName & storage_with_lock, Names real_column_names, const Block & sample_block, - // std::unique_ptr row_policy_data_ptr, ContextMutablePtr modified_context, size_t streams_num, bool concat_streams) @@ -795,13 +694,8 @@ QueryPipelineBuilderPtr ReadFromMerge::createSources( row_policy_data_ptr->extendNames(real_column_names); } - Aliases aliases; processAliases(real_column_names, storage_with_lock, aliases, sample_block, modified_context); - // if (row_policy_data_ptr) - // { - // row_policy_data_ptr->extendNames(real_column_names); - // } QueryPipelineBuilderPtr builder; if (!InterpreterSelectQuery::isQueryWithFinal(modified_query_info) && storage->needRewriteQueryWithFinal(real_column_names)) @@ -818,8 +712,6 @@ QueryPipelineBuilderPtr ReadFromMerge::createSources( storage_snapshot, modified_query_info); - // std::optional row_policy_data; - if (processed_stage <= storage_stage || (allow_experimental_analyzer && processed_stage == QueryProcessingStage::FetchColumns)) { /// If there are only virtual columns in query, you must request at least one other column. @@ -831,16 +723,6 @@ QueryPipelineBuilderPtr ReadFromMerge::createSources( StorageView * view = dynamic_cast(storage.get()); if (!view || allow_experimental_analyzer) { - // auto row_policy_filter_ptr = modified_context->getRowPolicyFilter( - // database_name, - // table_name, - // RowPolicyFilterType::SELECT_FILTER); - // if (row_policy_data_ptr) - // { - // row_policy_data.emplace(row_policy_filter_ptr, storage, modified_context); - // row_policy_data_ptr->extendNames(real_column_names); - // } - storage->read(plan, real_column_names, storage_snapshot, @@ -972,15 +854,14 @@ QueryPipelineBuilderPtr ReadFromMerge::createSources( }); } - // if (row_policy_data_ptr) - // { - // row_policy_data_ptr->addFilterTransform(*builder); - // } - /// Subordinary tables could have different but convertible types, like numeric types of different width. /// We must return streams with structure equals to structure of Merge table. - convertingSourceStream(header, storage_snapshot->metadata, aliases, std::move(row_policy_data_ptr), modified_context, *builder, processed_stage); - + convertAndFilterSourceStream(header, + storage_snapshot->metadata, + aliases, std::move(row_policy_data_ptr), + modified_context, + *builder, + processed_stage); } return builder; @@ -991,9 +872,6 @@ ReadFromMerge::RowPolicyData::RowPolicyData(RowPolicyFilterPtr row_policy_filter ContextPtr local_context) : row_policy_filter_ptr(row_policy_filter_ptr_) { - LOG_TRACE(&Poco::Logger::get("ReadFromMerge::RowPolicyData ctor"), - "storage {}", storage->getName()); - storage_metadata_snapshot = storage->getInMemoryMetadataPtr(); auto storage_columns = storage_metadata_snapshot->getColumns(); auto needed_columns = storage_columns.getAll/*Physical*/(); @@ -1017,70 +895,41 @@ ReadFromMerge::RowPolicyData::RowPolicyData(RowPolicyFilterPtr row_policy_filter if (!deleted.empty() || added.size() != 1) { throw Exception(ErrorCodes::LOGICAL_ERROR, - "Cannot determine row level filter"); + "Cannot determine row level filter; {} columns deleted, {} columns added", + deleted.size(), added.size()); } filter_column_name = added.getNames().front(); } -void ReadFromMerge::RowPolicyData::extendNames(Names & names, bool alias_allowed) +void ReadFromMerge::RowPolicyData::extendNames(Names & names) { - std::sort(names.begin(), names.end()); + boost::container::flat_set names_set(names.begin(), names.end()); NameSet added_names; for (const auto & req_column : filter_actions->getRequiredColumns()) { - if (!std::binary_search(names.begin(), names.end(), req_column)) + if (!names_set.contains(req_column)) { - if (!alias_allowed) - { - auto storage_columns = storage_metadata_snapshot->getColumns(); - const auto column_default = storage_columns.getDefault(req_column); - bool is_alias = column_default && column_default->kind == ColumnDefaultKind::Alias; - if (is_alias) - { - continue; - } - } - added_names.insert(req_column); + added_names.emplace(req_column); } } + if (!added_names.empty()) { std::copy(added_names.begin(), added_names.end(), std::back_inserter(names)); - LOG_TRACE(&Poco::Logger::get("ReadFromMerge::RowPolicyData::extendNames"), - "{} names added", added_names.size()); - for (const auto & added_name : added_names) - { - LOG_TRACE(&Poco::Logger::get("ReadFromMerge::RowPolicyData::extendNames"), - " added name {}", added_name); - } } } void ReadFromMerge::RowPolicyData::addStorageFilter(SourceStepWithFilter * step) { - LOG_TRACE(&Poco::Logger::get("ReadFromMerge::RowPolicyData::addStorageFilter"), "filter_actions_dag: {},<> {}, <> {}", - filter_actions->getActionsDAG().dumpNames(), - filter_actions->getActionsDAG().dumpDAG(), - filter_actions->getSampleBlock().dumpStructure()); - step->addFilter(actions_dag, filter_column_name); } void ReadFromMerge::RowPolicyData::addFilterTransform(QueryPipelineBuilder & builder) { - LOG_TRACE(&Poco::Logger::get("ReadFromMerge::RowPolicyData::addFilterTransform"), "filter_actions_dag: {},<> {}, <> {}", - filter_actions->getActionsDAG().dumpNames(), - filter_actions->getActionsDAG().dumpDAG(), - filter_actions->getSampleBlock().dumpStructure()); - builder.addSimpleTransform([&](const Block & stream_header) { - LOG_TRACE(&Poco::Logger::get("ReadFromMerge::RowPolicyData::addFilterTransform"), - "stream_header.dumpNames {}", stream_header.dumpNames()); - - return std::make_shared(stream_header, filter_actions, filter_column_name, true /* remove filter column */); }); } @@ -1255,7 +1104,7 @@ void StorageMerge::alter( setInMemoryMetadata(storage_metadata); } -void ReadFromMerge::convertingSourceStream( +void ReadFromMerge::convertAndFilterSourceStream( const Block & header, const StorageMetadataPtr & metadata_snapshot, const Aliases & aliases, @@ -1293,12 +1142,6 @@ void ReadFromMerge::convertingSourceStream( if (local_context->getSettingsRef().allow_experimental_analyzer && processed_stage != QueryProcessingStage::FetchColumns) convert_actions_match_columns_mode = ActionsDAG::MatchColumnsMode::Position; - LOG_TRACE(&Poco::Logger::get("ReadFromMerge::convertingSourceStream"), - "builder.getHeader(): {}, header.getColumnsWithTypeAndName: {}", - builder.getHeader().dumpStructure(), - header.dumpStructure()); - - if (row_policy_data_ptr) { row_policy_data_ptr->addFilterTransform(builder); diff --git a/src/Storages/StorageMerge.h b/src/Storages/StorageMerge.h index 350d871359a9..f7c31ab3ef81 100644 --- a/src/Storages/StorageMerge.h +++ b/src/Storages/StorageMerge.h @@ -206,7 +206,7 @@ class ReadFromMerge final : public SourceStepWithFilter size_t streams_num, bool concat_streams = false); - static void convertingSourceStream( + static void convertAndFilterSourceStream( const Block & header, const StorageMetadataPtr & metadata_snapshot, const Aliases & aliases, diff --git a/src/TableFunctions/TableFunctionMerge.cpp b/src/TableFunctions/TableFunctionMerge.cpp index d04575041442..599953a1add7 100644 --- a/src/TableFunctions/TableFunctionMerge.cpp +++ b/src/TableFunctions/TableFunctionMerge.cpp @@ -126,7 +126,7 @@ ColumnsDescription TableFunctionMerge::getActualTableStructure(ContextPtr contex { auto storage = DatabaseCatalog::instance().tryGetTable(StorageID{db_with_tables.first, table}, context); if (storage) - return ColumnsDescription{storage->getInMemoryMetadataPtr()->getColumns().getAll/* Physical*/()}; // !!! + return ColumnsDescription{storage->getInMemoryMetadataPtr()->getColumns().getAllPhysical()}; } } diff --git a/tests/queries/0_stateless/02763_row_policy_storage_merge.reference b/tests/queries/0_stateless/02763_row_policy_storage_merge.reference index 9c91a1652c7a..0b7664deb7d7 100644 --- a/tests/queries/0_stateless/02763_row_policy_storage_merge.reference +++ b/tests/queries/0_stateless/02763_row_policy_storage_merge.reference @@ -49,6 +49,10 @@ SELECT * FROM merge(currentDatabase(), 02763_merge_merge) WHERE x>2 3 13 4 14 4 14 +SELECT * FROM engine_merge_12 WHERE x>2 +3 13 +4 14 +4 14 SELECT * FROM merge(currentDatabase(), 02763_merge) 1 11 1 11 @@ -162,6 +166,31 @@ SELECT x, y from merge(currentDatabase(), 02763_merge 4 14 4 14 4 14 +02763_merge_aliases +x, y, z FROM 02763_a_merge +3 13 16 +4 14 18 +* FROM 02763_a_merge +3 13 16 +4 14 18 +x, y FROM 02763_a_merge +3 13 +4 14 +SELECT x, y FROM merge(currentDatabase(), 02763_alias) +3 13 +4 14 +SELECT x, y FROM merge(currentDatabase(), 02763_alias) +2 12 +3 13 +4 14 +SELECT x FROM merge(currentDatabase(), 02763_alias) +12 +13 +14 +SELECT y FROM merge(currentDatabase(), 02763_alias) +2 +3 +4 SETTINGS optimize_move_to_prewhere= 1 SELECT * FROM 02763_merge_log_1 3 13 @@ -191,6 +220,10 @@ SELECT * FROM merge(currentDatabase(), 02763_merge_merge) WHERE x>2 3 13 4 14 4 14 +SELECT * FROM engine_merge_12 WHERE x>2 +3 13 +4 14 +4 14 SELECT * FROM merge(currentDatabase(), 02763_merge) 1 11 1 11 @@ -304,3 +337,28 @@ SELECT x, y from merge(currentDatabase(), 02763_merge 4 14 4 14 4 14 +02763_merge_aliases +x, y, z FROM 02763_a_merge +3 13 16 +4 14 18 +* FROM 02763_a_merge +3 13 16 +4 14 18 +x, y FROM 02763_a_merge +3 13 +4 14 +SELECT x, y FROM merge(currentDatabase(), 02763_alias) +3 13 +4 14 +SELECT x, y FROM merge(currentDatabase(), 02763_alias) +2 12 +3 13 +4 14 +SELECT x FROM merge(currentDatabase(), 02763_alias) +12 +13 +14 +SELECT y FROM merge(currentDatabase(), 02763_alias) +2 +3 +4 diff --git a/tests/queries/0_stateless/02763_row_policy_storage_merge.sql.j2 b/tests/queries/0_stateless/02763_row_policy_storage_merge.sql.j2 index 0bc1292d4a5f..a22a9c5b6414 100644 --- a/tests/queries/0_stateless/02763_row_policy_storage_merge.sql.j2 +++ b/tests/queries/0_stateless/02763_row_policy_storage_merge.sql.j2 @@ -116,19 +116,19 @@ INSERT INTO 02763_merge_fancycols (x, y) SELECT x, y from merge(currentDatabase CREATE ROW POLICY 02763_filter_5 ON 02763_merge_fancycols USING cnst<>42 AS permissive TO ALL; SELECT 'SELECT *'; -SELECT * from merge(currentDatabase(), '02763_merge_fancycols') ORDER BY x; +SELECT * from merge(currentDatabase(), '02763_merge_fancycols') ORDER BY x SETTINGS optimize_move_to_prewhere= {{prew}}; SELECT 'SELECT x, lc'; -SELECT x, lc from merge(currentDatabase(), '02763_merge_fancycols') ORDER BY x; +SELECT x, lc from merge(currentDatabase(), '02763_merge_fancycols') ORDER BY x SETTINGS optimize_move_to_prewhere= {{prew}}; CREATE ROW POLICY 02763_filter_6 ON 02763_merge_fancycols USING lc='111' AS permissive TO ALL; SELECT 'SELECT *'; -SELECT * from merge(currentDatabase(), '02763_merge_fancycols') ORDER BY x; +SELECT * from merge(currentDatabase(), '02763_merge_fancycols') ORDER BY x SETTINGS optimize_move_to_prewhere= {{prew}}; SELECT 'SELECT x, lc'; -SELECT x, lc from merge(currentDatabase(), '02763_merge_fancycols') ORDER BY x; +SELECT x, lc from merge(currentDatabase(), '02763_merge_fancycols') ORDER BY x SETTINGS optimize_move_to_prewhere= {{prew}}; SELECT 'SELECT x, lc, cnst'; -SELECT x, lc, cnst from merge(currentDatabase(), '02763_merge_fancycols') ORDER BY x; +SELECT x, lc, cnst from merge(currentDatabase(), '02763_merge_fancycols') ORDER BY x SETTINGS optimize_move_to_prewhere= {{prew}}; SELECT 'SELECT x, y from merge(currentDatabase(), 02763_merge'; -SELECT x, y from merge(currentDatabase(), '02763_merge') ORDER BY x; +SELECT x, y from merge(currentDatabase(), '02763_merge') ORDER BY x SETTINGS optimize_move_to_prewhere= {{prew}}; DROP TABLE 02763_merge_fancycols; @@ -138,19 +138,26 @@ INSERT INTO 02763_alias VALUES (1, 11), (2, 12), (3, 13), (4, 14); CREATE ROW POLICY 02763_filter_7 ON 02763_alias USING z>15 AS permissive TO ALL; --- SELECT 'SELECT * FROM 02763_alias ORDER BY x'; --- SELECT x, y, z FROM 02763_alias ORDER BY x; - CREATE TABLE 02763_a_merge (x UInt8, y UInt64, z UInt64) ENGINE = Merge(currentDatabase(), '02763_alias'); --- SELECT 'SELECT * FROM merge(currentDatabase(), 02763_merge_alias) ORDER BY x'; --- SELECT * FROM merge(currentDatabase(), '02763_merge_alias') ORDER BY x; +SELECT 'x, y, z FROM 02763_a_merge'; +SELECT x, y, z FROM 02763_a_merge ORDER BY x SETTINGS optimize_move_to_prewhere= {{prew}}; +SELECT '* FROM 02763_a_merge'; +SELECT * FROM 02763_a_merge ORDER BY x SETTINGS optimize_move_to_prewhere= {{prew}}; +SELECT 'x, y FROM 02763_a_merge'; +SELECT x, y FROM 02763_a_merge ORDER BY x SETTINGS optimize_move_to_prewhere= {{prew}}; +SELECT 'SELECT x, y FROM merge(currentDatabase(), 02763_alias)'; +SELECT x, y FROM merge(currentDatabase(), '02763_alias') ORDER BY x SETTINGS optimize_move_to_prewhere= {{prew}}; + +CREATE ROW POLICY 02763_filter_8 ON 02763_alias USING y>11 AS permissive TO ALL; -SELECT 'x, y, z FROM 02763_a_merge ORDER BY x'; -SELECT x, y, z FROM 02763_a_merge ORDER BY x; --- SELECT 'SELECT x, y, z FROM merge(currentDatabase(), 02763_merge_alias) ORDER BY x'; --- SELECT x, y, z FROM merge(currentDatabase(), '02763_merge_alias') ORDER BY x; +SELECT 'SELECT x, y FROM merge(currentDatabase(), 02763_alias)'; +SELECT x, y FROM merge(currentDatabase(), '02763_alias') ORDER BY x SETTINGS optimize_move_to_prewhere= {{prew}}; +SELECT 'SELECT x FROM merge(currentDatabase(), 02763_alias)'; +SELECT y FROM merge(currentDatabase(), '02763_alias') ORDER BY x SETTINGS optimize_move_to_prewhere= {{prew}}; +SELECT 'SELECT y FROM merge(currentDatabase(), 02763_alias)'; +SELECT x FROM merge(currentDatabase(), '02763_alias') ORDER BY x SETTINGS optimize_move_to_prewhere= {{prew}}; DROP TABLE 02763_alias; DROP TABLE 02763_a_merge; @@ -165,5 +172,6 @@ DROP ROW POLICY 02763_filter_5 ON 02763_merge_fancycols; DROP ROW POLICY 02763_filter_6 ON 02763_merge_fancycols; DROP ROW POLICY 02763_filter_7 ON 02763_alias; +DROP ROW POLICY 02763_filter_8 ON 02763_alias; {% endfor %} From 988f344369ea4d45857c86ec690f7eb4d1cfbfc6 Mon Sep 17 00:00:00 2001 From: Ilya Golshtein Date: Thu, 9 Nov 2023 00:51:57 +0300 Subject: [PATCH 097/123] merge_row_policy: style fix, cleanup --- src/Storages/StorageMerge.cpp | 9 ++------- src/Storages/StorageMerge.h | 4 ---- 2 files changed, 2 insertions(+), 11 deletions(-) diff --git a/src/Storages/StorageMerge.cpp b/src/Storages/StorageMerge.cpp index 5412a87fa01e..5d5c4b716ecc 100644 --- a/src/Storages/StorageMerge.cpp +++ b/src/Storages/StorageMerge.cpp @@ -376,7 +376,8 @@ void StorageMerge::read( /// A transient object of this helper class is created /// when processing a Merge table data source (subordinary table) -/// to guarantee that row policies are applied +/// that has row policies +/// to guarantee that these row policies are applied class ReadFromMerge::RowPolicyData { public: @@ -404,7 +405,6 @@ class ReadFromMerge::RowPolicyData StorageMetadataPtr storage_metadata_snapshot; }; -// using RowPolicyDataPtr = std::unique_ptr; ReadFromMerge::ReadFromMerge( Block common_header_, @@ -473,8 +473,6 @@ void ReadFromMerge::initializePipeline(QueryPipelineBuilder & pipeline, const Bu query_info.input_order_info = input_sorting_info; } - // auto sample_block = merge_storage_snapshot->getMetadataForQuery()->getSampleBlock(); - std::vector> pipelines; QueryPlanResourceHolder resources; @@ -1135,8 +1133,6 @@ void ReadFromMerge::convertAndFilterSourceStream( }); } - - ActionsDAG::MatchColumnsMode convert_actions_match_columns_mode = ActionsDAG::MatchColumnsMode::Name; if (local_context->getSettingsRef().allow_experimental_analyzer && processed_stage != QueryProcessingStage::FetchColumns) @@ -1158,7 +1154,6 @@ void ReadFromMerge::convertAndFilterSourceStream( { return std::make_shared(stream_header, actions); }); - } bool ReadFromMerge::requestReadingInOrder(InputOrderInfoPtr order_info_) diff --git a/src/Storages/StorageMerge.h b/src/Storages/StorageMerge.h index f7c31ab3ef81..71680b900725 100644 --- a/src/Storages/StorageMerge.h +++ b/src/Storages/StorageMerge.h @@ -192,16 +192,12 @@ class ReadFromMerge final : public SourceStepWithFilter ContextMutablePtr modified_context); QueryPipelineBuilderPtr createSources( - // const StorageSnapshotPtr & storage_snapshot, - // SelectQueryInfo & query_info, QueryProcessingStage::Enum processed_stage, UInt64 max_block_size, const Block & header, - // const Aliases & aliases, const StorageWithLockAndName & storage_with_lock, Names real_column_names, const Block & sample_block, - // std::unique_ptr row_policy_data_ptr, ContextMutablePtr modified_context, size_t streams_num, bool concat_streams = false); From 05c1ff6b01a93be007af8a0c133bba4e677c7ea5 Mon Sep 17 00:00:00 2001 From: Ilya Golshtein Date: Tue, 14 Nov 2023 12:22:32 +0000 Subject: [PATCH 098/123] merge_row_policy: some comments --- src/Storages/StorageMerge.cpp | 2 ++ src/Storages/StorageMerge.h | 9 ++++++--- 2 files changed, 8 insertions(+), 3 deletions(-) diff --git a/src/Storages/StorageMerge.cpp b/src/Storages/StorageMerge.cpp index 5d5c4b716ecc..1eb93f13e9b8 100644 --- a/src/Storages/StorageMerge.cpp +++ b/src/Storages/StorageMerge.cpp @@ -634,6 +634,8 @@ void ReadFromMerge::processAliases( storage_metadata_snapshot->getColumns().getAll(), context); column_expr = setAlias(column_expr, column); + /// use storage type for transient columns that are not represented in result + /// e.g. for columns that needed to evaluate row policy auto type = sample_block.has(column) ? sample_block.getByName(column).type : column_description.type; aliases.push_back({ .name = column, .type = type, .expression = column_expr->clone() }); diff --git a/src/Storages/StorageMerge.h b/src/Storages/StorageMerge.h index 71680b900725..d6b4c5d8ad2f 100644 --- a/src/Storages/StorageMerge.h +++ b/src/Storages/StorageMerge.h @@ -170,9 +170,9 @@ class ReadFromMerge final : public SourceStepWithFilter struct AliasData { - String name; - DataTypePtr type; - ASTPtr expression; + String name; /// "size" in "size String Alias formatReadableSize(size_bytes)" + DataTypePtr type; /// String in "size String Alias formatReadableSize(size_bytes)", or something different came from query + ASTPtr expression; /// formatReadableSize(size_bytes) in "size String Alias formatReadableSize(size_bytes)" }; using Aliases = std::vector; @@ -184,6 +184,9 @@ class ReadFromMerge final : public SourceStepWithFilter const StorageWithLockAndName & storage_with_lock_and_name, const StorageSnapshotPtr & storage_snapshot); + /// Populates AliasData structures for further processing + /// using types from result query if possible + /// and removes alias columns from real_column_names void processAliases( Names & real_column_names, const StorageWithLockAndName & storage_with_lock, From d9495d4f27ab35b8d9adb2e1f8407cac6ae9e078 Mon Sep 17 00:00:00 2001 From: Ilya Golshtein Date: Mon, 20 Nov 2023 10:41:07 +0000 Subject: [PATCH 099/123] merge_row_policy: alias test separated out and disabled if analyzer --- .../02763_row_policy_storage_merge.reference | 50 ------------------- .../02763_row_policy_storage_merge.sql.j2 | 34 ------------- ...3_row_policy_storage_merge_alias.reference | 49 ++++++++++++++++++ ...2763_row_policy_storage_merge_alias.sql.j2 | 41 +++++++++++++++ 4 files changed, 90 insertions(+), 84 deletions(-) create mode 100644 tests/queries/0_stateless/02763_row_policy_storage_merge_alias.reference create mode 100644 tests/queries/0_stateless/02763_row_policy_storage_merge_alias.sql.j2 diff --git a/tests/queries/0_stateless/02763_row_policy_storage_merge.reference b/tests/queries/0_stateless/02763_row_policy_storage_merge.reference index 0b7664deb7d7..9fa5612e7cd4 100644 --- a/tests/queries/0_stateless/02763_row_policy_storage_merge.reference +++ b/tests/queries/0_stateless/02763_row_policy_storage_merge.reference @@ -166,31 +166,6 @@ SELECT x, y from merge(currentDatabase(), 02763_merge 4 14 4 14 4 14 -02763_merge_aliases -x, y, z FROM 02763_a_merge -3 13 16 -4 14 18 -* FROM 02763_a_merge -3 13 16 -4 14 18 -x, y FROM 02763_a_merge -3 13 -4 14 -SELECT x, y FROM merge(currentDatabase(), 02763_alias) -3 13 -4 14 -SELECT x, y FROM merge(currentDatabase(), 02763_alias) -2 12 -3 13 -4 14 -SELECT x FROM merge(currentDatabase(), 02763_alias) -12 -13 -14 -SELECT y FROM merge(currentDatabase(), 02763_alias) -2 -3 -4 SETTINGS optimize_move_to_prewhere= 1 SELECT * FROM 02763_merge_log_1 3 13 @@ -337,28 +312,3 @@ SELECT x, y from merge(currentDatabase(), 02763_merge 4 14 4 14 4 14 -02763_merge_aliases -x, y, z FROM 02763_a_merge -3 13 16 -4 14 18 -* FROM 02763_a_merge -3 13 16 -4 14 18 -x, y FROM 02763_a_merge -3 13 -4 14 -SELECT x, y FROM merge(currentDatabase(), 02763_alias) -3 13 -4 14 -SELECT x, y FROM merge(currentDatabase(), 02763_alias) -2 12 -3 13 -4 14 -SELECT x FROM merge(currentDatabase(), 02763_alias) -12 -13 -14 -SELECT y FROM merge(currentDatabase(), 02763_alias) -2 -3 -4 diff --git a/tests/queries/0_stateless/02763_row_policy_storage_merge.sql.j2 b/tests/queries/0_stateless/02763_row_policy_storage_merge.sql.j2 index a22a9c5b6414..0263e1a974f0 100644 --- a/tests/queries/0_stateless/02763_row_policy_storage_merge.sql.j2 +++ b/tests/queries/0_stateless/02763_row_policy_storage_merge.sql.j2 @@ -3,7 +3,6 @@ DROP TABLE IF EXISTS 02763_merge_log_2; DROP TABLE IF EXISTS 02763_merge_merge_1; DROP TABLE IF EXISTS 02763_merge_merge_2; DROP TABLE IF EXISTS 02763_merge_fancycols; -DROP TABLE IF EXISTS 02763_merge_aliases; DROP ROW POLICY IF EXISTS 02763_filter_1 ON 02763_merge_log_1; DROP ROW POLICY IF EXISTS 02763_filter_2 ON 02763_merge_merge_1; DROP ROW POLICY IF EXISTS 02763_filter_3 ON 02763_merge_log_1; @@ -132,36 +131,6 @@ SELECT x, y from merge(currentDatabase(), '02763_merge') ORDER BY x SETTINGS opt DROP TABLE 02763_merge_fancycols; -SELECT '02763_merge_aliases'; -CREATE TABLE 02763_alias (x UInt8, y UInt64, z UInt64 ALIAS plus(x,y)) ENGINE = MergeTree ORDER BY x; -INSERT INTO 02763_alias VALUES (1, 11), (2, 12), (3, 13), (4, 14); - -CREATE ROW POLICY 02763_filter_7 ON 02763_alias USING z>15 AS permissive TO ALL; - -CREATE TABLE 02763_a_merge (x UInt8, y UInt64, z UInt64) ENGINE = Merge(currentDatabase(), '02763_alias'); - - -SELECT 'x, y, z FROM 02763_a_merge'; -SELECT x, y, z FROM 02763_a_merge ORDER BY x SETTINGS optimize_move_to_prewhere= {{prew}}; -SELECT '* FROM 02763_a_merge'; -SELECT * FROM 02763_a_merge ORDER BY x SETTINGS optimize_move_to_prewhere= {{prew}}; -SELECT 'x, y FROM 02763_a_merge'; -SELECT x, y FROM 02763_a_merge ORDER BY x SETTINGS optimize_move_to_prewhere= {{prew}}; -SELECT 'SELECT x, y FROM merge(currentDatabase(), 02763_alias)'; -SELECT x, y FROM merge(currentDatabase(), '02763_alias') ORDER BY x SETTINGS optimize_move_to_prewhere= {{prew}}; - -CREATE ROW POLICY 02763_filter_8 ON 02763_alias USING y>11 AS permissive TO ALL; - -SELECT 'SELECT x, y FROM merge(currentDatabase(), 02763_alias)'; -SELECT x, y FROM merge(currentDatabase(), '02763_alias') ORDER BY x SETTINGS optimize_move_to_prewhere= {{prew}}; -SELECT 'SELECT x FROM merge(currentDatabase(), 02763_alias)'; -SELECT y FROM merge(currentDatabase(), '02763_alias') ORDER BY x SETTINGS optimize_move_to_prewhere= {{prew}}; -SELECT 'SELECT y FROM merge(currentDatabase(), 02763_alias)'; -SELECT x FROM merge(currentDatabase(), '02763_alias') ORDER BY x SETTINGS optimize_move_to_prewhere= {{prew}}; - -DROP TABLE 02763_alias; -DROP TABLE 02763_a_merge; - DROP ROW POLICY 02763_filter_1 ON 02763_merge_log_1; DROP ROW POLICY 02763_filter_2 ON 02763_merge_merge_1; @@ -171,7 +140,4 @@ DROP ROW POLICY 02763_filter_4 ON 02763_merge_merge_1; DROP ROW POLICY 02763_filter_5 ON 02763_merge_fancycols; DROP ROW POLICY 02763_filter_6 ON 02763_merge_fancycols; -DROP ROW POLICY 02763_filter_7 ON 02763_alias; -DROP ROW POLICY 02763_filter_8 ON 02763_alias; - {% endfor %} diff --git a/tests/queries/0_stateless/02763_row_policy_storage_merge_alias.reference b/tests/queries/0_stateless/02763_row_policy_storage_merge_alias.reference new file mode 100644 index 000000000000..56bfdbe0b18c --- /dev/null +++ b/tests/queries/0_stateless/02763_row_policy_storage_merge_alias.reference @@ -0,0 +1,49 @@ +02763_merge_aliases +x, y, z FROM 02763_a_merge +3 13 16 +4 14 18 +* FROM 02763_a_merge +3 13 16 +4 14 18 +x, y FROM 02763_a_merge +3 13 +4 14 +SELECT x, y FROM merge(currentDatabase(), 02763_alias) +3 13 +4 14 +SELECT x, y FROM merge(currentDatabase(), 02763_alias) +2 12 +3 13 +4 14 +SELECT x FROM merge(currentDatabase(), 02763_alias) +12 +13 +14 +SELECT y FROM merge(currentDatabase(), 02763_alias) +2 +3 +4 +x, y, z FROM 02763_a_merge +3 13 16 +4 14 18 +* FROM 02763_a_merge +3 13 16 +4 14 18 +x, y FROM 02763_a_merge +3 13 +4 14 +SELECT x, y FROM merge(currentDatabase(), 02763_alias) +3 13 +4 14 +SELECT x, y FROM merge(currentDatabase(), 02763_alias) +2 12 +3 13 +4 14 +SELECT x FROM merge(currentDatabase(), 02763_alias) +12 +13 +14 +SELECT y FROM merge(currentDatabase(), 02763_alias) +2 +3 +4 diff --git a/tests/queries/0_stateless/02763_row_policy_storage_merge_alias.sql.j2 b/tests/queries/0_stateless/02763_row_policy_storage_merge_alias.sql.j2 new file mode 100644 index 000000000000..bdd456951ddd --- /dev/null +++ b/tests/queries/0_stateless/02763_row_policy_storage_merge_alias.sql.j2 @@ -0,0 +1,41 @@ +DROP TABLE IF EXISTS 02763_alias; +DROP TABLE IF EXISTS 02763_a_merge; + + +SELECT '02763_merge_aliases'; +CREATE TABLE 02763_alias (x UInt8, y UInt64, z UInt64 ALIAS plus(x,y)) ENGINE = MergeTree ORDER BY x; +INSERT INTO 02763_alias VALUES (1, 11), (2, 12), (3, 13), (4, 14); + +CREATE ROW POLICY 02763_filter_7 ON 02763_alias USING z>15 AS permissive TO ALL; + +CREATE TABLE 02763_a_merge (x UInt8, y UInt64, z UInt64) ENGINE = Merge(currentDatabase(), '02763_alias'); + +{% for prew in [0 , 1] -%} + + + +SELECT 'x, y, z FROM 02763_a_merge'; +SELECT x, y, z FROM 02763_a_merge ORDER BY x SETTINGS optimize_move_to_prewhere= {{prew}}; +SELECT '* FROM 02763_a_merge'; +SELECT * FROM 02763_a_merge ORDER BY x SETTINGS optimize_move_to_prewhere= {{prew}}; +SELECT 'x, y FROM 02763_a_merge'; +SELECT x, y FROM 02763_a_merge ORDER BY x SETTINGS optimize_move_to_prewhere= {{prew}}; +SELECT 'SELECT x, y FROM merge(currentDatabase(), 02763_alias)'; +SELECT x, y FROM merge(currentDatabase(), '02763_alias') ORDER BY x SETTINGS optimize_move_to_prewhere= {{prew}}; + +CREATE ROW POLICY 02763_filter_8 ON 02763_alias USING y>11 AS permissive TO ALL; + +SELECT 'SELECT x, y FROM merge(currentDatabase(), 02763_alias)'; +SELECT x, y FROM merge(currentDatabase(), '02763_alias') ORDER BY x SETTINGS optimize_move_to_prewhere= {{prew}}; +SELECT 'SELECT x FROM merge(currentDatabase(), 02763_alias)'; +SELECT y FROM merge(currentDatabase(), '02763_alias') ORDER BY x SETTINGS optimize_move_to_prewhere= {{prew}}; +SELECT 'SELECT y FROM merge(currentDatabase(), 02763_alias)'; +SELECT x FROM merge(currentDatabase(), '02763_alias') ORDER BY x SETTINGS optimize_move_to_prewhere= {{prew}}; + +DROP ROW POLICY 02763_filter_8 ON 02763_alias; +{% endfor %} + +DROP TABLE 02763_alias; +DROP TABLE 02763_a_merge; + +DROP ROW POLICY 02763_filter_7 ON 02763_alias; From 5a4ebd430f613430672d08c12a393b6e43b4dc80 Mon Sep 17 00:00:00 2001 From: MyroTk <44327070+MyroTk@users.noreply.github.com> Date: Wed, 11 Sep 2024 23:07:51 -0700 Subject: [PATCH 100/123] Update docker server tag --- tests/ci/docker_server.py | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/tests/ci/docker_server.py b/tests/ci/docker_server.py index da15ad2bf4fd..0cec4a8968b8 100644 --- a/tests/ci/docker_server.py +++ b/tests/ci/docker_server.py @@ -36,8 +36,7 @@ TEMP_PATH = p.join(RUNNER_TEMP, "docker_images_check") BUCKETS = { "amd64": "package_release", - # NOTE(vnemkov): arm64 is temporary not supported - # "arm64": "package_aarch64" + "arm64": "package_aarch64" } git = Git(ignore_no_tags=True) @@ -59,7 +58,7 @@ def parse_args() -> argparse.Namespace: parser.add_argument( "--version", type=version_arg, - default=get_version_from_repo(git=git).string, + default=get_version_from_repo(git=git), help="a version to build, automatically got from version_helper, accepts either " "tag ('refs/tags/' is removed automatically) or a normal 22.2.2.2 format", ) @@ -338,7 +337,7 @@ def main(): args.bucket_prefix = ( f"{S3_DOWNLOAD}/{S3_BUILDS_BUCKET}/{release_or_pr}/{pr_info.sha}" ) - tags.append(f"{pr_info.number}-{args.version}") + tags.append(f"{pr_info.number}-{args.version.string}") if args.push: subprocess.check_output( # pylint: disable=unexpected-keyword-arg From 35cfb06e870c8ea797538c368e3e80311ee2c56f Mon Sep 17 00:00:00 2001 From: MyroTk <44327070+MyroTk@users.noreply.github.com> Date: Wed, 11 Sep 2024 23:21:33 -0700 Subject: [PATCH 101/123] Update release_branches.yml --- .github/workflows/release_branches.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/release_branches.yml b/.github/workflows/release_branches.yml index e35e168a7c35..1883f6842523 100644 --- a/.github/workflows/release_branches.yml +++ b/.github/workflows/release_branches.yml @@ -704,7 +704,7 @@ jobs: - name: Set envs run: | cat >> "$GITHUB_ENV" << 'EOF' - CHECK_NAME=SignAarch64 + CHECK_NAME=Sign aarch64 TEMP_PATH=${{runner.temp}}/signed REPORTS_PATH=${{runner.temp}}/reports_dir EOF From 5659b84345e9aea99a4e5f6d8211dd5e2ef8bce8 Mon Sep 17 00:00:00 2001 From: Vasily Nemkov Date: Thu, 12 Sep 2024 11:29:39 +0000 Subject: [PATCH 102/123] Attempt to fix test_distributed_directory_monitor_split_batch_on_failure/test.py::test_distributed_directory_monitor_split_batch_on_failure_OFF --- .../test.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/integration/test_distributed_directory_monitor_split_batch_on_failure/test.py b/tests/integration/test_distributed_directory_monitor_split_batch_on_failure/test.py index e66631460f7a..7c337f5f0d3b 100644 --- a/tests/integration/test_distributed_directory_monitor_split_batch_on_failure/test.py +++ b/tests/integration/test_distributed_directory_monitor_split_batch_on_failure/test.py @@ -76,7 +76,7 @@ def test_distributed_directory_monitor_split_batch_on_failure_OFF(started_cluste if setting == "monitor_batch_inserts" and setting_value == 1: with pytest.raises( QueryRuntimeException, - match=r"DB::Exception: Received from.*Memory limit \(for query\) exceeded: .*while pushing to view default\.mv", + match=r"DB::Exception: Received from.*Memory limit \(for query\) exceeded: .*While sending a batch\.mv", ): node2.query("system flush distributed dist") assert int(node2.query("select count() from dist_data")) == 0 From e955ad1b453771adbb425acf1544fcec26488ea4 Mon Sep 17 00:00:00 2001 From: MyroTk <44327070+MyroTk@users.noreply.github.com> Date: Thu, 12 Sep 2024 05:11:45 -0700 Subject: [PATCH 103/123] Update release_branches.yml --- .github/workflows/release_branches.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/release_branches.yml b/.github/workflows/release_branches.yml index 1883f6842523..27d987c3257a 100644 --- a/.github/workflows/release_branches.yml +++ b/.github/workflows/release_branches.yml @@ -687,7 +687,7 @@ jobs: - name: Upload signed hashes uses: actions/upload-artifact@v4 with: - name: signed-hashes + name: signed-release-hashes path: ${{ env.TEMP_PATH }}/*.gpg - name: Cleanup if: always() @@ -729,7 +729,7 @@ jobs: - name: Upload signed hashes uses: actions/upload-artifact@v4 with: - name: signed-hashes + name: signed-aarch64-hashes path: ${{ env.TEMP_PATH }}/*.gpg - name: Cleanup if: always() From 1226632c7297838757300c5a4870079ab07bb162 Mon Sep 17 00:00:00 2001 From: Vasily Nemkov Date: Thu, 12 Sep 2024 17:01:16 +0200 Subject: [PATCH 104/123] Update test.py --- .../test.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/integration/test_distributed_directory_monitor_split_batch_on_failure/test.py b/tests/integration/test_distributed_directory_monitor_split_batch_on_failure/test.py index 7c337f5f0d3b..f2e0cb9b95b6 100644 --- a/tests/integration/test_distributed_directory_monitor_split_batch_on_failure/test.py +++ b/tests/integration/test_distributed_directory_monitor_split_batch_on_failure/test.py @@ -76,7 +76,7 @@ def test_distributed_directory_monitor_split_batch_on_failure_OFF(started_cluste if setting == "monitor_batch_inserts" and setting_value == 1: with pytest.raises( QueryRuntimeException, - match=r"DB::Exception: Received from.*Memory limit \(for query\) exceeded: .*While sending a batch\.mv", + match=r"DB::Exception: Received from.*Memory limit \(for query\) exceeded: .*While sending a batch", ): node2.query("system flush distributed dist") assert int(node2.query("select count() from dist_data")) == 0 From 5fd626168226b0c00e9f889f433fe3d5322b71a1 Mon Sep 17 00:00:00 2001 From: Alexander Gololobov Date: Thu, 13 Jun 2024 13:37:36 +0000 Subject: [PATCH 105/123] Merge pull request #65051 from ClickHouse/fix_capture_this_in_context_access Capture weak_ptr of ContextAccess for safety --- src/Access/ContextAccess.cpp | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/src/Access/ContextAccess.cpp b/src/Access/ContextAccess.cpp index c3aeec15cdd6..df496c025d2f 100644 --- a/src/Access/ContextAccess.cpp +++ b/src/Access/ContextAccess.cpp @@ -306,10 +306,13 @@ void ContextAccess::setUser(const UserPtr & user_) const subscription_for_roles_changes.reset(); enabled_roles = access_control->getEnabledRoles(current_roles, current_roles_with_admin_option); - subscription_for_roles_changes = enabled_roles->subscribeForChanges([this](const std::shared_ptr & roles_info_) + subscription_for_roles_changes = enabled_roles->subscribeForChanges([weak_ptr = weak_from_this()](const std::shared_ptr & roles_info_) { - std::lock_guard lock{mutex}; - setRolesInfo(roles_info_); + auto ptr = weak_ptr.lock(); + if (!ptr) + return; + std::lock_guard lock{ptr->mutex}; + ptr->setRolesInfo(roles_info_); }); setRolesInfo(enabled_roles->getRolesInfo()); From 8140e24eb13657dda813bb14aa4bbc3673c73e92 Mon Sep 17 00:00:00 2001 From: Vasily Nemkov Date: Thu, 12 Sep 2024 20:02:51 +0200 Subject: [PATCH 106/123] Update test.py --- .../test.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/tests/integration/test_distributed_directory_monitor_split_batch_on_failure/test.py b/tests/integration/test_distributed_directory_monitor_split_batch_on_failure/test.py index f2e0cb9b95b6..a9893b1f7688 100644 --- a/tests/integration/test_distributed_directory_monitor_split_batch_on_failure/test.py +++ b/tests/integration/test_distributed_directory_monitor_split_batch_on_failure/test.py @@ -76,7 +76,8 @@ def test_distributed_directory_monitor_split_batch_on_failure_OFF(started_cluste if setting == "monitor_batch_inserts" and setting_value == 1: with pytest.raises( QueryRuntimeException, - match=r"DB::Exception: Received from.*Memory limit \(for query\) exceeded: .*While sending a batch", + # NOTE(vnemkov): memory limit may hit in different stages of query, just make sure that it was reached an query cancelled. + match=r"DB::Exception: Received from.*Memory limit \(for query\) exceeded:", ): node2.query("system flush distributed dist") assert int(node2.query("select count() from dist_data")) == 0 From 034c9f62db9685d95e6516a0882e376c52955cf6 Mon Sep 17 00:00:00 2001 From: shiyer7474 Date: Fri, 13 Sep 2024 15:37:07 +0000 Subject: [PATCH 107/123] Backport #63685 fix SIGSEGV due to profiler --- contrib/libunwind | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/contrib/libunwind b/contrib/libunwind index 30cc1d3fd365..ba533a7246a2 160000 --- a/contrib/libunwind +++ b/contrib/libunwind @@ -1 +1 @@ -Subproject commit 30cc1d3fd3655a5cfa0ab112fe320fb9fc0a8344 +Subproject commit ba533a7246a2686b0552061809612f503804d26b From 793aa27f50e10041ebc70a1d88eb8b2b0d11019f Mon Sep 17 00:00:00 2001 From: MyroTk <44327070+MyroTk@users.noreply.github.com> Date: Tue, 17 Sep 2024 07:57:31 -0700 Subject: [PATCH 108/123] Update release_branches.yml --- .github/workflows/release_branches.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/release_branches.yml b/.github/workflows/release_branches.yml index 27d987c3257a..53bbe25d3e13 100644 --- a/.github/workflows/release_branches.yml +++ b/.github/workflows/release_branches.yml @@ -641,7 +641,7 @@ jobs: secrets: inherit with: runner_type: altinity-on-demand, altinity-type-cpx51, altinity-image-x86-system-ubuntu-22.04, altinity-setup-regression - commit: 17a81c07fc1f41fbee651e0ef0ca4b44e537e5b1 + commit: 66fff15e3afa278543a6b6ffab9be576e054a921 arch: release build_sha: ${{ github.event_name == 'pull_request' && github.event.pull_request.head.sha || github.sha }} @@ -651,7 +651,7 @@ jobs: secrets: inherit with: runner_type: altinity-on-demand, altinity-type-cax41, altinity-image-arm-system-ubuntu-22.04, altinity-setup-regression - commit: 17a81c07fc1f41fbee651e0ef0ca4b44e537e5b1 + commit: 66fff15e3afa278543a6b6ffab9be576e054a921 arch: aarch64 build_sha: ${{ github.event_name == 'pull_request' && github.event.pull_request.head.sha || github.sha }} From d4cb826d73093085a9edae458dc9d26e5a3aef86 Mon Sep 17 00:00:00 2001 From: MyroTk <44327070+MyroTk@users.noreply.github.com> Date: Wed, 18 Sep 2024 03:34:29 -0700 Subject: [PATCH 109/123] Upgrade regression runners to ubuntu:24.04 --- .github/workflows/release_branches.yml | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/.github/workflows/release_branches.yml b/.github/workflows/release_branches.yml index 53bbe25d3e13..8a529b826278 100644 --- a/.github/workflows/release_branches.yml +++ b/.github/workflows/release_branches.yml @@ -26,8 +26,7 @@ on: # yamllint disable-line rule:truthy push: branches: - 'releases/23.8**' - schedule: - - cron: "0 0 * * 6" + workflow_dispatch: jobs: DockerHubPushAarch64: @@ -640,7 +639,7 @@ jobs: uses: ./.github/workflows/regression.yml secrets: inherit with: - runner_type: altinity-on-demand, altinity-type-cpx51, altinity-image-x86-system-ubuntu-22.04, altinity-setup-regression + runner_type: altinity-on-demand, altinity-type-cpx51, altinity-image-x86-app-docker-ce, altinity-setup-regression commit: 66fff15e3afa278543a6b6ffab9be576e054a921 arch: release build_sha: ${{ github.event_name == 'pull_request' && github.event.pull_request.head.sha || github.sha }} @@ -650,7 +649,7 @@ jobs: uses: ./.github/workflows/regression.yml secrets: inherit with: - runner_type: altinity-on-demand, altinity-type-cax41, altinity-image-arm-system-ubuntu-22.04, altinity-setup-regression + runner_type: altinity-on-demand, altinity-type-cax41, altinity-image-arm-app-docker-ce, altinity-setup-regression commit: 66fff15e3afa278543a6b6ffab9be576e054a921 arch: aarch64 build_sha: ${{ github.event_name == 'pull_request' && github.event.pull_request.head.sha || github.sha }} From 5645cbe052c11442bcb0a1ab8156656db356c2fd Mon Sep 17 00:00:00 2001 From: "Mikhail f. Shiryaev" Date: Fri, 26 Apr 2024 08:10:04 +0000 Subject: [PATCH 110/123] Merge pull request #62886 from ilejn/kerberized_hadoop_dockerfile Build kererberized_hadoop image by downloading commons-daemon via https --- docker/test/integration/kerberized_hadoop/Dockerfile | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/docker/test/integration/kerberized_hadoop/Dockerfile b/docker/test/integration/kerberized_hadoop/Dockerfile index f3a063a6ac97..b1ea38484916 100644 --- a/docker/test/integration/kerberized_hadoop/Dockerfile +++ b/docker/test/integration/kerberized_hadoop/Dockerfile @@ -14,14 +14,14 @@ RUN curl -o krb5-libs-1.10.3-65.el6.x86_64.rpm ftp://ftp.icm.edu.pl/vol/rzm6/lin rpm -Uvh libkadm5-1.10.3-65.el6.x86_64.rpm libss-1.41.12-24.el6.x86_64.rpm krb5-libs-1.10.3-65.el6.x86_64.rpm krb5-workstation-1.10.3-65.el6.x86_64.rpm libcom_err-1.41.12-24.el6.x86_64.rpm && \ rm -fr *.rpm +ADD https://archive.apache.org/dist/commons/daemon/source/commons-daemon-1.0.15-src.tar.gz /tmp/commons-daemon-1.0.15-src.tar.gz + RUN cd /tmp && \ - curl -o wget.rpm ftp://ftp.pbone.net/mirror/vault.centos.org/6.9/os/x86_64/Packages/wget-1.12-10.el6.x86_64.rpm && \ - rpm -i wget.rpm && \ - rm -fr *.rpm && \ - wget --no-check-certificate https://archive.apache.org/dist/commons/daemon/source/commons-daemon-1.0.15-src.tar.gz && \ tar xzf commons-daemon-1.0.15-src.tar.gz && \ cd commons-daemon-1.0.15-src/src/native/unix && \ ./configure && \ make && \ cp ./jsvc /usr/local/hadoop-2.7.0/sbin && \ - [ -e /usr/local/hadoop ] || ln -s ./hadoop-2.7.0 /usr/local/hadoop + cd /tmp && \ + rm -rf commons-daemon-1.0.15-src* && \ + { [ -e /usr/local/hadoop ] || ln -s ./hadoop-2.7.0 /usr/local/hadoop; } From 860881772ea6b06d1b4a07f64e5b2d57500df1b6 Mon Sep 17 00:00:00 2001 From: MyroTk <44327070+MyroTk@users.noreply.github.com> Date: Mon, 23 Sep 2024 09:07:14 -0700 Subject: [PATCH 111/123] Check for push event. --- tests/ci/docker_images_check.py | 27 ++++++++++++++++----------- 1 file changed, 16 insertions(+), 11 deletions(-) diff --git a/tests/ci/docker_images_check.py b/tests/ci/docker_images_check.py index b7a80f86e917..e1e7709ca447 100644 --- a/tests/ci/docker_images_check.py +++ b/tests/ci/docker_images_check.py @@ -110,19 +110,24 @@ def get_changed_docker_images( changed_images.append(DockerImage(dockerfile_dir, name, only_amd64)) break - # Rebuild all on opened PR or during release - if pr_info.event['action'] in ['opened', 'reopened', 'published', 'prereleased']: + # Rebuild all images on push or release + if pr_info.number == 0: changed_images = all_images - # Check that image for the PR exists - elif pr_info.event['action'] == 'synchronize': - unchanged_images = [ - image for image in all_images if image not in changed_images - ] - logging.info(f"Unchanged images: {unchanged_images}") - for image in unchanged_images: - if subprocess.run(f"docker manifest inspect {image.repo}:{pr_info.number}", shell=True).returncode != 0: - changed_images.append(image) + else: + # Rebuild all on opened PR + if pr_info.event['action'] in ['opened', 'reopened']: + changed_images = all_images + + # Check that image for the PR exists + elif pr_info.event['action'] == 'synchronize': + unchanged_images = [ + image for image in all_images if image not in changed_images + ] + logging.info(f"Unchanged images: {unchanged_images}") + for image in unchanged_images: + if subprocess.run(f"docker manifest inspect {image.repo}:{pr_info.number}", shell=True).returncode != 0: + changed_images.append(image) # The order is important: dependents should go later than bases, so that # they are built with updated base versions. From 65aa954adef8c20c7341602163d4fd10cf53f363 Mon Sep 17 00:00:00 2001 From: MyroTk Date: Tue, 1 Oct 2024 10:27:59 -0700 Subject: [PATCH 112/123] add support for maintenance runs --- .github/workflows/regression.yml | 3 +++ .github/workflows/release_branches.yml | 4 ++-- tests/ci/build_check.py | 4 ++++ 3 files changed, 9 insertions(+), 2 deletions(-) diff --git a/.github/workflows/regression.yml b/.github/workflows/regression.yml index aa4b2f68904e..66d89ddede43 100644 --- a/.github/workflows/regression.yml +++ b/.github/workflows/regression.yml @@ -93,6 +93,9 @@ env: AWS_DEFAULT_REGION: ${{ secrets.AWS_REPORT_REGION }} DOCKER_USERNAME: ${{ secrets.DOCKER_USERNAME }} DOCKER_PASSWORD: ${{ secrets.DOCKER_PASSWORD }} + CHECKS_DATABASE_HOST: ${{ secrets.CHECKS_DATABASE_HOST }} + CHECKS_DATABASE_USER: ${{ secrets.CHECKS_DATABASE_USER }} + CHECKS_DATABASE_PASSWORD: ${{ secrets.CHECKS_DATABASE_PASSWORD }} args: --test-to-end --no-colors --local diff --git a/.github/workflows/release_branches.yml b/.github/workflows/release_branches.yml index 8a529b826278..a8113d891bd8 100644 --- a/.github/workflows/release_branches.yml +++ b/.github/workflows/release_branches.yml @@ -640,7 +640,7 @@ jobs: secrets: inherit with: runner_type: altinity-on-demand, altinity-type-cpx51, altinity-image-x86-app-docker-ce, altinity-setup-regression - commit: 66fff15e3afa278543a6b6ffab9be576e054a921 + commit: 91b2de501b77cd90ae9345abb7b56e5fa8c1a921 arch: release build_sha: ${{ github.event_name == 'pull_request' && github.event.pull_request.head.sha || github.sha }} @@ -650,7 +650,7 @@ jobs: secrets: inherit with: runner_type: altinity-on-demand, altinity-type-cax41, altinity-image-arm-app-docker-ce, altinity-setup-regression - commit: 66fff15e3afa278543a6b6ffab9be576e054a921 + commit: 91b2de501b77cd90ae9345abb7b56e5fa8c1a921 arch: aarch64 build_sha: ${{ github.event_name == 'pull_request' && github.event.pull_request.head.sha || github.sha }} diff --git a/tests/ci/build_check.py b/tests/ci/build_check.py index df72d19693ab..a0f7a5fc24f7 100644 --- a/tests/ci/build_check.py +++ b/tests/ci/build_check.py @@ -13,6 +13,7 @@ from docker_pull_helper import get_image_with_version from env_helper import ( GITHUB_JOB_API_URL, + GITHUB_RUN_ID, IMAGES_PATH, REPO_COPY, S3_ACCESS_KEY_ID, @@ -184,6 +185,9 @@ def get_release_or_pr(pr_info: PRInfo, version: ClickHouseVersion) -> Tuple[str, # It should be fixed in performance-comparison image eventually # For performance tests we always set PRs prefix performance_pr = "PRs/0" + if "commits" not in pr_info.event and "pull_request" not in pr_info.event: + # for dispatch maintenance run we use sha and run id + return f"maintenance/{pr_info.base_ref}/{GITHUB_RUN_ID}", performance_pr if "release" in pr_info.labels or "release-lts" in pr_info.labels: # for release pull requests we use branch names prefixes, not pr numbers return pr_info.head_ref, performance_pr From 8516d66015258518d7ac51cade1946573d811caa Mon Sep 17 00:00:00 2001 From: MyroTk <44327070+MyroTk@users.noreply.github.com> Date: Wed, 2 Oct 2024 08:33:57 -0700 Subject: [PATCH 113/123] Update docker_images_check.py --- tests/ci/docker_images_check.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/ci/docker_images_check.py b/tests/ci/docker_images_check.py index e1e7709ca447..df64c7a6094c 100644 --- a/tests/ci/docker_images_check.py +++ b/tests/ci/docker_images_check.py @@ -111,7 +111,7 @@ def get_changed_docker_images( break # Rebuild all images on push or release - if pr_info.number == 0: + if pr_info.number == 0 or pr_info.number == f"{pr_info.version.major}.{pr_info.version.minor}.{pr_info.version.patch}": changed_images = all_images else: From 07d98304736fbfbb90e74cc8af04fe6816c0b8fb Mon Sep 17 00:00:00 2001 From: MyroTk <44327070+MyroTk@users.noreply.github.com> Date: Wed, 2 Oct 2024 13:21:51 -0700 Subject: [PATCH 114/123] Update Dockerfile --- docker/test/util/Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docker/test/util/Dockerfile b/docker/test/util/Dockerfile index f2041fe445c3..deff1273ac7c 100644 --- a/docker/test/util/Dockerfile +++ b/docker/test/util/Dockerfile @@ -12,7 +12,7 @@ RUN apt-get update \ && apt-get install \ apt-transport-https='2.4.*' \ apt-utils='2.4.*' \ - ca-certificates='20230311ubuntu0.22.04.*' \ + ca-certificates='20240203~22.04*' \ curl='7.81.*' \ dnsutils='1:9.18.*' \ gnupg='2.2.*' \ From 7082e220a6741ccd51871bc0b6f912ff83cd888c Mon Sep 17 00:00:00 2001 From: Alexey Milovidov Date: Sat, 11 Nov 2023 02:29:04 +0100 Subject: [PATCH 115/123] Merge pull request #56502 from amosbird/fix-56481 Fix two cases of projection analysis. --- src/Core/Settings.h | 1 + .../optimizeUseAggregateProjection.cpp | 103 ++++++++++++------ .../optimizeUseNormalProjection.cpp | 40 ++++++- .../Optimizations/projectionsCommon.cpp | 17 ++- .../Optimizations/projectionsCommon.h | 3 +- .../QueryPlan/ReadFromMergeTree.cpp | 21 +++- src/Processors/QueryPlan/ReadFromMergeTree.h | 10 +- src/Storages/MergeTree/MergeTreeData.cpp | 20 +++- src/Storages/MergeTree/MergeTreeData.h | 6 +- .../MergeTree/MergeTreeDataSelectExecutor.cpp | 10 +- .../MergeTree/MergeTreeDataSelectExecutor.h | 8 +- src/Storages/MergeTree/PartitionPruner.cpp | 2 +- src/Storages/MergeTree/PartitionPruner.h | 5 +- src/Storages/StorageMergeTree.cpp | 6 +- ...jection_analysis_reuse_partition.reference | 1 + ...710_projection_analysis_reuse_partition.sh | 16 +++ ...rojection_with_alter_conversions.reference | 1 + ...1710_projection_with_alter_conversions.sql | 15 +++ .../queries/0_stateless/01710_projections.sql | 2 +- 19 files changed, 207 insertions(+), 80 deletions(-) create mode 100644 tests/queries/0_stateless/01710_projection_analysis_reuse_partition.reference create mode 100755 tests/queries/0_stateless/01710_projection_analysis_reuse_partition.sh create mode 100644 tests/queries/0_stateless/01710_projection_with_alter_conversions.reference create mode 100644 tests/queries/0_stateless/01710_projection_with_alter_conversions.sql diff --git a/src/Core/Settings.h b/src/Core/Settings.h index b3fe90b86446..eab06e06db0f 100644 --- a/src/Core/Settings.h +++ b/src/Core/Settings.h @@ -594,6 +594,7 @@ class IColumn; M(Bool, optimize_use_projections, true, "Automatically choose projections to perform SELECT query", 0) ALIAS(allow_experimental_projection_optimization) \ M(Bool, optimize_use_implicit_projections, true, "Automatically choose implicit projections to perform SELECT query", 0) \ M(Bool, force_optimize_projection, false, "If projection optimization is enabled, SELECT queries need to use projection", 0) \ + M(String, preferred_optimize_projection_name, "", "If it is set to a non-empty string, ClickHouse tries to apply specified projection", 0) \ M(Bool, async_socket_for_remote, true, "Asynchronously read from socket executing remote query", 0) \ M(Bool, async_query_sending_for_remote, true, "Asynchronously create connections and send query to shards in remote query", 0) \ M(Bool, insert_null_as_default, true, "Insert DEFAULT values instead of NULL in INSERT SELECT (UNION ALL)", 0) \ diff --git a/src/Processors/QueryPlan/Optimizations/optimizeUseAggregateProjection.cpp b/src/Processors/QueryPlan/Optimizations/optimizeUseAggregateProjection.cpp index 0599a0fa369d..f4e2a6b7dbd1 100644 --- a/src/Processors/QueryPlan/Optimizations/optimizeUseAggregateProjection.cpp +++ b/src/Processors/QueryPlan/Optimizations/optimizeUseAggregateProjection.cpp @@ -411,7 +411,6 @@ struct MinMaxProjectionCandidate { AggregateProjectionCandidate candidate; Block block; - MergeTreeData::DataPartsVector normal_parts; }; struct AggregateProjectionCandidates @@ -476,7 +475,6 @@ AggregateProjectionCandidates getAggregateProjectionCandidates( { // LOG_TRACE(&Poco::Logger::get("optimizeUseProjections"), "Projection analyzed DAG {}", proj_dag->dumpDAG()); AggregateProjectionCandidate candidate{.info = std::move(info), .dag = std::move(proj_dag)}; - MergeTreeData::DataPartsVector minmax_projection_normal_parts; // LOG_TRACE(&Poco::Logger::get("optimizeUseProjections"), "Projection sample block {}", sample_block.dumpStructure()); auto block = reading.getMergeTreeData().getMinMaxCountProjectionBlock( @@ -485,13 +483,13 @@ AggregateProjectionCandidates getAggregateProjectionCandidates( dag.filter_node != nullptr, query_info, parts, - minmax_projection_normal_parts, + nullptr, max_added_blocks.get(), context); // LOG_TRACE(&Poco::Logger::get("optimizeUseProjections"), "Projection sample block 2 {}", block.dumpStructure()); - // minmax_count_projection cannot be used used when there is no data to process, because + // minmax_count_projection cannot be used when there is no data to process, because // it will produce incorrect result during constant aggregation. // See https://github.com/ClickHouse/ClickHouse/issues/36728 if (block) @@ -499,7 +497,6 @@ AggregateProjectionCandidates getAggregateProjectionCandidates( MinMaxProjectionCandidate minmax; minmax.candidate = std::move(candidate); minmax.block = std::move(block); - minmax.normal_parts = std::move(minmax_projection_normal_parts); minmax.candidate.projection = projection; candidates.minmax_projection.emplace(std::move(minmax)); } @@ -508,6 +505,18 @@ AggregateProjectionCandidates getAggregateProjectionCandidates( if (!candidates.minmax_projection) { + auto it = std::find_if(agg_projections.begin(), agg_projections.end(), [&](const auto * projection) + { + return projection->name == context->getSettings().preferred_optimize_projection_name.value; + }); + + if (it != agg_projections.end()) + { + const ProjectionDescription * preferred_projection = *it; + agg_projections.clear(); + agg_projections.push_back(preferred_projection); + } + candidates.real.reserve(agg_projections.size()); for (const auto * projection : agg_projections) { @@ -569,49 +578,75 @@ bool optimizeUseAggregateProjections(QueryPlan::Node & node, QueryPlan::Nodes & auto candidates = getAggregateProjectionCandidates(node, *aggregating, *reading, max_added_blocks, allow_implicit_projections); - AggregateProjectionCandidate * best_candidate = nullptr; - if (candidates.minmax_projection) - best_candidate = &candidates.minmax_projection->candidate; - else if (candidates.real.empty()) - return false; - const auto & parts = reading->getParts(); + const auto & alter_conversions = reading->getAlterConvertionsForParts(); const auto & query_info = reading->getQueryInfo(); const auto metadata = reading->getStorageMetadata(); ContextPtr context = reading->getContext(); MergeTreeDataSelectExecutor reader(reading->getMergeTreeData()); + AggregateProjectionCandidate * best_candidate = nullptr; - auto ordinary_reading_select_result = reading->selectRangesToRead(parts, /* alter_conversions = */ {}); - size_t ordinary_reading_marks = ordinary_reading_select_result->marks(); - - /// Selecting best candidate. - for (auto & candidate : candidates.real) + if (candidates.minmax_projection) { - auto required_column_names = candidate.dag->getRequiredColumnsNames(); - ActionDAGNodes added_filter_nodes; - if (candidates.has_filter) - added_filter_nodes.nodes.push_back(candidate.dag->getOutputs().front()); + best_candidate = &candidates.minmax_projection->candidate; + } + else if (!candidates.real.empty()) + { + auto ordinary_reading_select_result = reading->selectRangesToRead(parts, alter_conversions); + size_t ordinary_reading_marks = ordinary_reading_select_result->marks(); + + /// Nothing to read. Ignore projections. + if (ordinary_reading_marks == 0) + { + reading->setAnalyzedResult(std::move(ordinary_reading_select_result)); + return false; + } - bool analyzed = analyzeProjectionCandidate( - candidate, *reading, reader, required_column_names, parts, - metadata, query_info, context, max_added_blocks, added_filter_nodes); + const auto & parts_with_ranges = ordinary_reading_select_result->partsWithRanges(); - if (!analyzed) - continue; + /// Selecting best candidate. + for (auto & candidate : candidates.real) + { + auto required_column_names = candidate.dag->getRequiredColumnsNames(); + ActionDAGNodes added_filter_nodes; + if (candidates.has_filter) + added_filter_nodes.nodes.push_back(candidate.dag->getOutputs().front()); + + bool analyzed = analyzeProjectionCandidate( + candidate, + *reading, + reader, + required_column_names, + parts_with_ranges, + metadata, + query_info, + context, + max_added_blocks, + added_filter_nodes); - if (candidate.sum_marks > ordinary_reading_marks) - continue; + if (!analyzed) + continue; - if (best_candidate == nullptr || best_candidate->sum_marks > candidate.sum_marks) - best_candidate = &candidate; - } + if (candidate.sum_marks > ordinary_reading_marks) + continue; + + if (best_candidate == nullptr || best_candidate->sum_marks > candidate.sum_marks) + best_candidate = &candidate; + } - if (!best_candidate) + if (!best_candidate) + { + reading->setAnalyzedResult(std::move(ordinary_reading_select_result)); + return false; + } + } + else { - reading->setAnalyzedResult(std::move(ordinary_reading_select_result)); return false; } + chassert(best_candidate != nullptr); + QueryPlanStepPtr projection_reading; bool has_ordinary_parts; @@ -632,9 +667,7 @@ bool optimizeUseAggregateProjections(QueryPlan::Node & node, QueryPlan::Nodes & .storage_id = reading->getMergeTreeData().getStorageID(), .projection_name = candidates.minmax_projection->candidate.projection->name, }); - has_ordinary_parts = !candidates.minmax_projection->normal_parts.empty(); - if (has_ordinary_parts) - reading->resetParts(std::move(candidates.minmax_projection->normal_parts)); + has_ordinary_parts = false; } else { diff --git a/src/Processors/QueryPlan/Optimizations/optimizeUseNormalProjection.cpp b/src/Processors/QueryPlan/Optimizations/optimizeUseNormalProjection.cpp index fbe02265dcc6..d4acf36e0012 100644 --- a/src/Processors/QueryPlan/Optimizations/optimizeUseNormalProjection.cpp +++ b/src/Processors/QueryPlan/Optimizations/optimizeUseNormalProjection.cpp @@ -8,7 +8,7 @@ #include #include #include -#include +#include namespace DB::QueryPlanOptimizations { @@ -107,6 +107,19 @@ bool optimizeUseNormalProjections(Stack & stack, QueryPlan::Nodes & nodes) if (normal_projections.empty()) return false; + ContextPtr context = reading->getContext(); + auto it = std::find_if(normal_projections.begin(), normal_projections.end(), [&](const auto * projection) + { + return projection->name == context->getSettings().preferred_optimize_projection_name.value; + }); + + if (it != normal_projections.end()) + { + const ProjectionDescription * preferred_projection = *it; + normal_projections.clear(); + normal_projections.push_back(preferred_projection); + } + QueryDAG query; { auto & child = iter->node->children[iter->next_child - 1]; @@ -122,13 +135,22 @@ bool optimizeUseNormalProjections(Stack & stack, QueryPlan::Nodes & nodes) const Names & required_columns = reading->getRealColumnNames(); const auto & parts = reading->getParts(); + const auto & alter_conversions = reading->getAlterConvertionsForParts(); const auto & query_info = reading->getQueryInfo(); - ContextPtr context = reading->getContext(); MergeTreeDataSelectExecutor reader(reading->getMergeTreeData()); - auto ordinary_reading_select_result = reading->selectRangesToRead(parts, /* alter_conversions = */ {}); + auto ordinary_reading_select_result = reading->selectRangesToRead(parts, alter_conversions); size_t ordinary_reading_marks = ordinary_reading_select_result->marks(); + /// Nothing to read. Ignore projections. + if (ordinary_reading_marks == 0) + { + reading->setAnalyzedResult(std::move(ordinary_reading_select_result)); + return false; + } + + const auto & parts_with_ranges = ordinary_reading_select_result->partsWithRanges(); + std::shared_ptr max_added_blocks = getMaxAddedBlocks(reading); for (const auto * projection : normal_projections) @@ -144,8 +166,16 @@ bool optimizeUseNormalProjections(Stack & stack, QueryPlan::Nodes & nodes) added_filter_nodes.nodes.push_back(query.filter_node); bool analyzed = analyzeProjectionCandidate( - candidate, *reading, reader, required_columns, parts, - metadata, query_info, context, max_added_blocks, added_filter_nodes); + candidate, + *reading, + reader, + required_columns, + parts_with_ranges, + metadata, + query_info, + context, + max_added_blocks, + added_filter_nodes); if (!analyzed) continue; diff --git a/src/Processors/QueryPlan/Optimizations/projectionsCommon.cpp b/src/Processors/QueryPlan/Optimizations/projectionsCommon.cpp index 7ddda29cad43..c3b3449857b0 100644 --- a/src/Processors/QueryPlan/Optimizations/projectionsCommon.cpp +++ b/src/Processors/QueryPlan/Optimizations/projectionsCommon.cpp @@ -210,7 +210,7 @@ bool analyzeProjectionCandidate( const ReadFromMergeTree & reading, const MergeTreeDataSelectExecutor & reader, const Names & required_column_names, - const MergeTreeData::DataPartsVector & parts, + const RangesInDataParts & parts_with_ranges, const StorageMetadataPtr & metadata, const SelectQueryInfo & query_info, const ContextPtr & context, @@ -219,14 +219,20 @@ bool analyzeProjectionCandidate( { MergeTreeData::DataPartsVector projection_parts; MergeTreeData::DataPartsVector normal_parts; - for (const auto & part : parts) + std::vector alter_conversions; + for (const auto & part_with_ranges : parts_with_ranges) { - const auto & created_projections = part->getProjectionParts(); + const auto & created_projections = part_with_ranges.data_part->getProjectionParts(); auto it = created_projections.find(candidate.projection->name); if (it != created_projections.end()) + { projection_parts.push_back(it->second); + } else - normal_parts.push_back(part); + { + normal_parts.push_back(part_with_ranges.data_part); + alter_conversions.push_back(part_with_ranges.alter_conversions); + } } if (projection_parts.empty()) @@ -252,7 +258,8 @@ bool analyzeProjectionCandidate( if (!normal_parts.empty()) { - auto normal_result_ptr = reading.selectRangesToRead(std::move(normal_parts), /* alter_conversions = */ {}); + /// TODO: We can reuse existing analysis_result by filtering out projection parts + auto normal_result_ptr = reading.selectRangesToRead(std::move(normal_parts), std::move(alter_conversions)); if (normal_result_ptr->error()) return false; diff --git a/src/Processors/QueryPlan/Optimizations/projectionsCommon.h b/src/Processors/QueryPlan/Optimizations/projectionsCommon.h index 35daccad1154..055ca5d40848 100644 --- a/src/Processors/QueryPlan/Optimizations/projectionsCommon.h +++ b/src/Processors/QueryPlan/Optimizations/projectionsCommon.h @@ -19,6 +19,7 @@ using MergeTreeDataSelectAnalysisResultPtr = std::shared_ptr; using DataPartsVector = std::vector; +struct RangesInDataParts; struct StorageInMemoryMetadata; using StorageMetadataPtr = std::shared_ptr; @@ -71,7 +72,7 @@ bool analyzeProjectionCandidate( const ReadFromMergeTree & reading, const MergeTreeDataSelectExecutor & reader, const Names & required_column_names, - const DataPartsVector & parts, + const RangesInDataParts & parts_with_ranges, const StorageMetadataPtr & metadata, const SelectQueryInfo & query_info, const ContextPtr & context, diff --git a/src/Processors/QueryPlan/ReadFromMergeTree.cpp b/src/Processors/QueryPlan/ReadFromMergeTree.cpp index 0b5eb94dbaca..bc43961edf5b 100644 --- a/src/Processors/QueryPlan/ReadFromMergeTree.cpp +++ b/src/Processors/QueryPlan/ReadFromMergeTree.cpp @@ -2157,10 +2157,23 @@ size_t MergeTreeDataSelectAnalysisResult::marks() const if (std::holds_alternative(result)) std::rethrow_exception(std::get(result)); - const auto & index_stats = std::get(result).index_stats; - if (index_stats.empty()) - return 0; - return index_stats.back().num_granules_after; + return std::get(result).selected_marks; +} + +UInt64 MergeTreeDataSelectAnalysisResult::rows() const +{ + if (std::holds_alternative(result)) + std::rethrow_exception(std::get(result)); + + return std::get(result).selected_rows; +} + +const RangesInDataParts & MergeTreeDataSelectAnalysisResult::partsWithRanges() const +{ + if (std::holds_alternative(result)) + std::rethrow_exception(std::get(result)); + + return std::get(result).parts_with_ranges; } } diff --git a/src/Processors/QueryPlan/ReadFromMergeTree.h b/src/Processors/QueryPlan/ReadFromMergeTree.h index cb2a3a8ddf9e..e3406addf6c3 100644 --- a/src/Processors/QueryPlan/ReadFromMergeTree.h +++ b/src/Processors/QueryPlan/ReadFromMergeTree.h @@ -214,13 +214,9 @@ class ReadFromMergeTree final : public SourceStepWithFilter bool hasAnalyzedResult() const { return analyzed_result_ptr != nullptr; } void setAnalyzedResult(MergeTreeDataSelectAnalysisResultPtr analyzed_result_ptr_) { analyzed_result_ptr = std::move(analyzed_result_ptr_); } - void resetParts(MergeTreeData::DataPartsVector parts) - { - prepared_parts = std::move(parts); - alter_conversions_for_parts = {}; - } - const MergeTreeData::DataPartsVector & getParts() const { return prepared_parts; } + const std::vector & getAlterConvertionsForParts() const { return alter_conversions_for_parts; } + const MergeTreeData & getMergeTreeData() const { return data; } size_t getMaxBlockSize() const { return max_block_size; } size_t getNumStreams() const { return requested_num_streams; } @@ -330,6 +326,8 @@ struct MergeTreeDataSelectAnalysisResult bool error() const; size_t marks() const; + UInt64 rows() const; + const RangesInDataParts & partsWithRanges() const; }; } diff --git a/src/Storages/MergeTree/MergeTreeData.cpp b/src/Storages/MergeTree/MergeTreeData.cpp index 6bb5231f998d..145875938167 100644 --- a/src/Storages/MergeTree/MergeTreeData.cpp +++ b/src/Storages/MergeTree/MergeTreeData.cpp @@ -6596,7 +6596,7 @@ Block MergeTreeData::getMinMaxCountProjectionBlock( bool has_filter, const SelectQueryInfo & query_info, const DataPartsVector & parts, - DataPartsVector & normal_parts, + DataPartsVector * normal_parts, const PartitionIdToMaxBlock * max_block_numbers_to_read, ContextPtr query_context) const { @@ -6721,10 +6721,22 @@ Block MergeTreeData::getMinMaxCountProjectionBlock( continue; } + /// It's extremely rare that some parts have final marks while others don't. To make it + /// straightforward, disable minmax_count projection when `max(pk)' encounters any part with + /// no final mark. if (need_primary_key_max_column && !part->index_granularity.hasFinalMark()) { - normal_parts.push_back(part); - continue; + if (normal_parts) + { + // 23.8 behaviour + normal_parts->push_back(part); + continue; + } + else + { + // 23.12 behaviour + return {}; + } } real_parts.push_back(part); @@ -7161,7 +7173,7 @@ std::optional MergeTreeData::getQueryProcessingStageWithAgg !query_info.filter_asts.empty() || analysis_result.prewhere_info || analysis_result.before_where, query_info, parts, - normal_parts, + &normal_parts, max_added_blocks.get(), query_context); diff --git a/src/Storages/MergeTree/MergeTreeData.h b/src/Storages/MergeTree/MergeTreeData.h index 22d7a070ad94..1aeacf7c873c 100644 --- a/src/Storages/MergeTree/MergeTreeData.h +++ b/src/Storages/MergeTree/MergeTreeData.h @@ -396,17 +396,13 @@ class MergeTreeData : public IStorage, public WithMutableContext /// query_info - used to filter unneeded parts /// /// parts - part set to filter - /// - /// normal_parts - collects parts that don't have all the needed values to form the block. - /// Specifically, this is when a part doesn't contain a final mark and the related max value is - /// required. Block getMinMaxCountProjectionBlock( const StorageMetadataPtr & metadata_snapshot, const Names & required_columns, bool has_filter, const SelectQueryInfo & query_info, const DataPartsVector & parts, - DataPartsVector & normal_parts, + DataPartsVector * normal_parts, const PartitionIdToMaxBlock * max_block_numbers_to_read, ContextPtr query_context) const; diff --git a/src/Storages/MergeTree/MergeTreeDataSelectExecutor.cpp b/src/Storages/MergeTree/MergeTreeDataSelectExecutor.cpp index afb73ff38592..5b844d80211d 100644 --- a/src/Storages/MergeTree/MergeTreeDataSelectExecutor.cpp +++ b/src/Storages/MergeTree/MergeTreeDataSelectExecutor.cpp @@ -797,8 +797,8 @@ std::optional> MergeTreeDataSelectExecutor::filterPar } void MergeTreeDataSelectExecutor::filterPartsByPartition( - std::optional & partition_pruner, - std::optional & minmax_idx_condition, + const std::optional & partition_pruner, + const std::optional & minmax_idx_condition, MergeTreeData::DataPartsVector & parts, std::vector & alter_conversions, const std::optional> & part_values, @@ -1253,6 +1253,8 @@ MergeTreeDataSelectAnalysisResultPtr MergeTreeDataSelectExecutor::estimateNumMar selectColumnNames(column_names_to_return, data, real_column_names, virt_column_names, sample_factor_column_queried); std::optional indexes; + /// NOTE: We don't need alter_conversions because the returned analysis_result is only used for: + /// 1. estimate the number of rows to read; 2. projection reading, which doesn't have alter_conversions. return ReadFromMergeTree::selectRangesToRead( std::move(parts), /*alter_conversions=*/ {}, @@ -1785,7 +1787,7 @@ void MergeTreeDataSelectExecutor::selectPartsToRead( const std::optional> & part_values, const std::optional & minmax_idx_condition, const DataTypes & minmax_columns_types, - std::optional & partition_pruner, + const std::optional & partition_pruner, const PartitionIdToMaxBlock * max_block_numbers_to_read, PartFilterCounters & counters) { @@ -1847,7 +1849,7 @@ void MergeTreeDataSelectExecutor::selectPartsToReadWithUUIDFilter( MergeTreeData::PinnedPartUUIDsPtr pinned_part_uuids, const std::optional & minmax_idx_condition, const DataTypes & minmax_columns_types, - std::optional & partition_pruner, + const std::optional & partition_pruner, const PartitionIdToMaxBlock * max_block_numbers_to_read, ContextPtr query_context, PartFilterCounters & counters, diff --git a/src/Storages/MergeTree/MergeTreeDataSelectExecutor.h b/src/Storages/MergeTree/MergeTreeDataSelectExecutor.h index 74d8d8e3c8f7..05d11a858b33 100644 --- a/src/Storages/MergeTree/MergeTreeDataSelectExecutor.h +++ b/src/Storages/MergeTree/MergeTreeDataSelectExecutor.h @@ -126,7 +126,7 @@ class MergeTreeDataSelectExecutor const std::optional> & part_values, const std::optional & minmax_idx_condition, const DataTypes & minmax_columns_types, - std::optional & partition_pruner, + const std::optional & partition_pruner, const PartitionIdToMaxBlock * max_block_numbers_to_read, PartFilterCounters & counters); @@ -138,7 +138,7 @@ class MergeTreeDataSelectExecutor MergeTreeData::PinnedPartUUIDsPtr pinned_part_uuids, const std::optional & minmax_idx_condition, const DataTypes & minmax_columns_types, - std::optional & partition_pruner, + const std::optional & partition_pruner, const PartitionIdToMaxBlock * max_block_numbers_to_read, ContextPtr query_context, PartFilterCounters & counters, @@ -172,8 +172,8 @@ class MergeTreeDataSelectExecutor /// Filter parts using minmax index and partition key. static void filterPartsByPartition( - std::optional & partition_pruner, - std::optional & minmax_idx_condition, + const std::optional & partition_pruner, + const std::optional & minmax_idx_condition, MergeTreeData::DataPartsVector & parts, std::vector & alter_conversions, const std::optional> & part_values, diff --git a/src/Storages/MergeTree/PartitionPruner.cpp b/src/Storages/MergeTree/PartitionPruner.cpp index 97bb9f3b4d43..a5df08e3df96 100644 --- a/src/Storages/MergeTree/PartitionPruner.cpp +++ b/src/Storages/MergeTree/PartitionPruner.cpp @@ -31,7 +31,7 @@ PartitionPruner::PartitionPruner(const StorageMetadataPtr & metadata, ActionsDAG { } -bool PartitionPruner::canBePruned(const IMergeTreeDataPart & part) +bool PartitionPruner::canBePruned(const IMergeTreeDataPart & part) const { if (part.isEmpty()) return true; diff --git a/src/Storages/MergeTree/PartitionPruner.h b/src/Storages/MergeTree/PartitionPruner.h index 7f1b74795c4c..e8a740b15245 100644 --- a/src/Storages/MergeTree/PartitionPruner.h +++ b/src/Storages/MergeTree/PartitionPruner.h @@ -16,14 +16,15 @@ class PartitionPruner PartitionPruner(const StorageMetadataPtr & metadata, const SelectQueryInfo & query_info, ContextPtr context, bool strict); PartitionPruner(const StorageMetadataPtr & metadata, ActionsDAGPtr filter_actions_dag, ContextPtr context, bool strict); - bool canBePruned(const IMergeTreeDataPart & part); + bool canBePruned(const IMergeTreeDataPart & part) const; bool isUseless() const { return useless; } const KeyCondition & getKeyCondition() const { return partition_condition; } private: - std::unordered_map partition_filter_map; + /// Cache already analyzed partitions. + mutable std::unordered_map partition_filter_map; /// partition_key is adjusted here (with substitution from modulo to moduloLegacy). KeyDescription partition_key; diff --git a/src/Storages/StorageMergeTree.cpp b/src/Storages/StorageMergeTree.cpp index 850f469b03b1..4f49a6651133 100644 --- a/src/Storages/StorageMergeTree.cpp +++ b/src/Storages/StorageMergeTree.cpp @@ -345,6 +345,8 @@ void StorageMergeTree::alter( prev_mutation = it->first; } + /// Always wait previous mutations synchronously, because alters + /// should be executed in sequential order. if (prev_mutation != 0) { LOG_DEBUG(log, "Cannot change metadata with barrier alter query, will wait for mutation {}", prev_mutation); @@ -372,9 +374,7 @@ void StorageMergeTree::alter( resetObjectColumnsFromActiveParts(parts_lock); } - /// Always execute required mutations synchronously, because alters - /// should be executed in sequential order. - if (!maybe_mutation_commands.empty()) + if (!maybe_mutation_commands.empty() && local_context->getSettingsRef().alter_sync > 0) waitForMutation(mutation_version, false); } diff --git a/tests/queries/0_stateless/01710_projection_analysis_reuse_partition.reference b/tests/queries/0_stateless/01710_projection_analysis_reuse_partition.reference new file mode 100644 index 000000000000..47b07da250f1 --- /dev/null +++ b/tests/queries/0_stateless/01710_projection_analysis_reuse_partition.reference @@ -0,0 +1 @@ +Selected 2/2 parts by partition key, 1 parts by primary key, 1/2 marks by primary key, 1 marks to read from 1 ranges diff --git a/tests/queries/0_stateless/01710_projection_analysis_reuse_partition.sh b/tests/queries/0_stateless/01710_projection_analysis_reuse_partition.sh new file mode 100755 index 000000000000..ba8b3818ba38 --- /dev/null +++ b/tests/queries/0_stateless/01710_projection_analysis_reuse_partition.sh @@ -0,0 +1,16 @@ +#!/usr/bin/env bash + +CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) +# shellcheck source=../shell_config.sh +. "$CURDIR"/../shell_config.sh + +${CLICKHOUSE_CLIENT} -q "drop table if exists t" +${CLICKHOUSE_CLIENT} -q "create table t(s LowCardinality(String), e DateTime64(3), projection p1 (select * order by s, e)) engine MergeTree partition by toYYYYMM(e) order by tuple() settings index_granularity = 8192, index_granularity_bytes = '100M'" +${CLICKHOUSE_CLIENT} -q "insert into t select 'AAP', toDateTime('2023-07-01') + 360 * number from numbers(50000)" +${CLICKHOUSE_CLIENT} -q "insert into t select 'AAPL', toDateTime('2023-07-01') + 360 * number from numbers(50000)" + +CLICKHOUSE_CLIENT_DEBUG_LOG=$(echo ${CLICKHOUSE_CLIENT} | sed 's/'"--send_logs_level=${CLICKHOUSE_CLIENT_SERVER_LOGS_LEVEL}"'/--send_logs_level=debug/g') + +${CLICKHOUSE_CLIENT_DEBUG_LOG} -q "select count() from t where e >= '2023-11-08 00:00:00.000' and e < '2023-11-09 00:00:00.000' and s in ('AAPL') format Null" 2>&1 | grep -oh "Selected .* parts by partition key, *. parts by primary key, .* marks by primary key, .* marks to read from .* ranges.*$" + +${CLICKHOUSE_CLIENT} -q "drop table t" diff --git a/tests/queries/0_stateless/01710_projection_with_alter_conversions.reference b/tests/queries/0_stateless/01710_projection_with_alter_conversions.reference new file mode 100644 index 000000000000..9874d6464ab7 --- /dev/null +++ b/tests/queries/0_stateless/01710_projection_with_alter_conversions.reference @@ -0,0 +1 @@ +1 2 diff --git a/tests/queries/0_stateless/01710_projection_with_alter_conversions.sql b/tests/queries/0_stateless/01710_projection_with_alter_conversions.sql new file mode 100644 index 000000000000..649a07b9b5f1 --- /dev/null +++ b/tests/queries/0_stateless/01710_projection_with_alter_conversions.sql @@ -0,0 +1,15 @@ +drop table if exists t; + +create table t (i int, j int, projection p (select i order by i)) engine MergeTree order by tuple(); + +insert into t values (1, 2); + +system stop merges t; + +set alter_sync = 0; + +alter table t rename column j to k; + +select * from t; + +drop table t; diff --git a/tests/queries/0_stateless/01710_projections.sql b/tests/queries/0_stateless/01710_projections.sql index a96339e30fac..7c45792847e7 100644 --- a/tests/queries/0_stateless/01710_projections.sql +++ b/tests/queries/0_stateless/01710_projections.sql @@ -1,6 +1,6 @@ drop table if exists projection_test; -create table projection_test (`sum(block_count)` UInt64, domain_alias UInt64 alias length(domain), datetime DateTime, domain LowCardinality(String), x_id String, y_id String, block_count Int64, retry_count Int64, duration Int64, kbytes Int64, buffer_time Int64, first_time Int64, total_bytes Nullable(UInt64), valid_bytes Nullable(UInt64), completed_bytes Nullable(UInt64), fixed_bytes Nullable(UInt64), force_bytes Nullable(UInt64), projection p (select toStartOfMinute(datetime) dt_m, countIf(first_time = 0) / count(), avg((kbytes * 8) / duration), count(), sum(block_count) / sum(duration), avg(block_count / duration), sum(buffer_time) / sum(duration), avg(buffer_time / duration), sum(valid_bytes) / sum(total_bytes), sum(completed_bytes) / sum(total_bytes), sum(fixed_bytes) / sum(total_bytes), sum(force_bytes) / sum(total_bytes), sum(valid_bytes) / sum(total_bytes), sum(retry_count) / sum(duration), avg(retry_count / duration), countIf(block_count > 0) / count(), countIf(first_time = 0) / count(), uniqHLL12(x_id), uniqHLL12(y_id) group by dt_m, domain)) engine MergeTree partition by toDate(datetime) order by (toStartOfTenMinutes(datetime), domain) settings index_granularity_bytes = 10000000; +create table projection_test (`sum(block_count)` UInt64, domain_alias UInt64 alias length(domain), datetime DateTime, domain LowCardinality(String), x_id String, y_id String, block_count Int64, retry_count Int64, duration Int64, kbytes Int64, buffer_time Int64, first_time Int64, total_bytes Nullable(UInt64), valid_bytes Nullable(UInt64), completed_bytes Nullable(UInt64), fixed_bytes Nullable(UInt64), force_bytes Nullable(UInt64), projection p (select toStartOfMinute(datetime) dt_m, countIf(first_time = 0) / count(), avg((kbytes * 8) / duration), count(), sum(block_count) / sum(duration), avg(block_count / duration), sum(buffer_time) / sum(duration), avg(buffer_time / duration), sum(valid_bytes) / sum(total_bytes), sum(completed_bytes) / sum(total_bytes), sum(fixed_bytes) / sum(total_bytes), sum(force_bytes) / sum(total_bytes), sum(valid_bytes) / sum(total_bytes), sum(retry_count) / sum(duration), avg(retry_count / duration), countIf(block_count > 0) / count(), countIf(first_time = 0) / count(), uniqHLL12(x_id), uniqHLL12(y_id) group by dt_m, domain)) engine MergeTree partition by toDate(datetime) order by toStartOfTenMinutes(datetime) settings index_granularity_bytes = 10000000; insert into projection_test with rowNumberInAllBlocks() as id select 1, toDateTime('2020-10-24 00:00:00') + (id / 20), toString(id % 100), * from generateRandom('x_id String, y_id String, block_count Int64, retry_count Int64, duration Int64, kbytes Int64, buffer_time Int64, first_time Int64, total_bytes Nullable(UInt64), valid_bytes Nullable(UInt64), completed_bytes Nullable(UInt64), fixed_bytes Nullable(UInt64), force_bytes Nullable(UInt64)', 10, 10, 1) limit 1000 settings max_threads = 1; From eb5814ec4bd9b38402d02c0d414e78c06ac81a40 Mon Sep 17 00:00:00 2001 From: Nikolai Kochetov Date: Thu, 11 Jan 2024 14:17:48 +0100 Subject: [PATCH 116/123] Merge pull request #58638 from amosbird/fix-58620 Fix broken partition key analysis when doing projection optimization --- .../optimizeUseAggregateProjection.cpp | 1 - .../Optimizations/optimizeUseNormalProjection.cpp | 1 - .../QueryPlan/Optimizations/projectionsCommon.cpp | 2 -- .../QueryPlan/Optimizations/projectionsCommon.h | 1 - src/Processors/QueryPlan/ReadFromMergeTree.cpp | 7 +------ src/Processors/QueryPlan/ReadFromMergeTree.h | 2 -- src/Storages/MergeTree/MergeTreeData.cpp | 5 ----- .../MergeTree/MergeTreeDataSelectExecutor.cpp | 3 +-- .../MergeTree/MergeTreeDataSelectExecutor.h | 1 - .../01710_projection_fix_crash.reference | 0 .../0_stateless/01710_projection_fix_crash.sql | 15 +++++++++++++++ ...65_projection_with_partition_pruning.reference | 1 + .../02965_projection_with_partition_pruning.sql | 9 +++++++++ 13 files changed, 27 insertions(+), 21 deletions(-) create mode 100644 tests/queries/0_stateless/01710_projection_fix_crash.reference create mode 100644 tests/queries/0_stateless/01710_projection_fix_crash.sql create mode 100644 tests/queries/0_stateless/02965_projection_with_partition_pruning.reference create mode 100644 tests/queries/0_stateless/02965_projection_with_partition_pruning.sql diff --git a/src/Processors/QueryPlan/Optimizations/optimizeUseAggregateProjection.cpp b/src/Processors/QueryPlan/Optimizations/optimizeUseAggregateProjection.cpp index f4e2a6b7dbd1..3040f408cb56 100644 --- a/src/Processors/QueryPlan/Optimizations/optimizeUseAggregateProjection.cpp +++ b/src/Processors/QueryPlan/Optimizations/optimizeUseAggregateProjection.cpp @@ -618,7 +618,6 @@ bool optimizeUseAggregateProjections(QueryPlan::Node & node, QueryPlan::Nodes & reader, required_column_names, parts_with_ranges, - metadata, query_info, context, max_added_blocks, diff --git a/src/Processors/QueryPlan/Optimizations/optimizeUseNormalProjection.cpp b/src/Processors/QueryPlan/Optimizations/optimizeUseNormalProjection.cpp index d4acf36e0012..8cb64ddf9e39 100644 --- a/src/Processors/QueryPlan/Optimizations/optimizeUseNormalProjection.cpp +++ b/src/Processors/QueryPlan/Optimizations/optimizeUseNormalProjection.cpp @@ -171,7 +171,6 @@ bool optimizeUseNormalProjections(Stack & stack, QueryPlan::Nodes & nodes) reader, required_columns, parts_with_ranges, - metadata, query_info, context, max_added_blocks, diff --git a/src/Processors/QueryPlan/Optimizations/projectionsCommon.cpp b/src/Processors/QueryPlan/Optimizations/projectionsCommon.cpp index c3b3449857b0..9bc18ee38ba4 100644 --- a/src/Processors/QueryPlan/Optimizations/projectionsCommon.cpp +++ b/src/Processors/QueryPlan/Optimizations/projectionsCommon.cpp @@ -211,7 +211,6 @@ bool analyzeProjectionCandidate( const MergeTreeDataSelectExecutor & reader, const Names & required_column_names, const RangesInDataParts & parts_with_ranges, - const StorageMetadataPtr & metadata, const SelectQueryInfo & query_info, const ContextPtr & context, const std::shared_ptr & max_added_blocks, @@ -242,7 +241,6 @@ bool analyzeProjectionCandidate( std::move(projection_parts), nullptr, required_column_names, - metadata, candidate.projection->metadata, query_info, /// How it is actually used? I hope that for index we need only added_filter_nodes added_filter_nodes, diff --git a/src/Processors/QueryPlan/Optimizations/projectionsCommon.h b/src/Processors/QueryPlan/Optimizations/projectionsCommon.h index 055ca5d40848..22606ef8a299 100644 --- a/src/Processors/QueryPlan/Optimizations/projectionsCommon.h +++ b/src/Processors/QueryPlan/Optimizations/projectionsCommon.h @@ -73,7 +73,6 @@ bool analyzeProjectionCandidate( const MergeTreeDataSelectExecutor & reader, const Names & required_column_names, const RangesInDataParts & parts_with_ranges, - const StorageMetadataPtr & metadata, const SelectQueryInfo & query_info, const ContextPtr & context, const std::shared_ptr & max_added_blocks, diff --git a/src/Processors/QueryPlan/ReadFromMergeTree.cpp b/src/Processors/QueryPlan/ReadFromMergeTree.cpp index bc43961edf5b..566c577d687f 100644 --- a/src/Processors/QueryPlan/ReadFromMergeTree.cpp +++ b/src/Processors/QueryPlan/ReadFromMergeTree.cpp @@ -1168,7 +1168,6 @@ MergeTreeDataSelectAnalysisResultPtr ReadFromMergeTree::selectRangesToRead( std::move(alter_conversions), prewhere_info, filter_nodes, - storage_snapshot->metadata, metadata_for_reading, query_info, context, @@ -1354,7 +1353,6 @@ MergeTreeDataSelectAnalysisResultPtr ReadFromMergeTree::selectRangesToRead( std::vector alter_conversions, const PrewhereInfoPtr & prewhere_info, const ActionDAGNodes & added_filter_nodes, - const StorageMetadataPtr & metadata_snapshot_base, const StorageMetadataPtr & metadata_snapshot, const SelectQueryInfo & query_info, ContextPtr context, @@ -1375,7 +1373,6 @@ MergeTreeDataSelectAnalysisResultPtr ReadFromMergeTree::selectRangesToRead( return selectRangesToReadImpl( std::move(parts), std::move(alter_conversions), - metadata_snapshot_base, metadata_snapshot, updated_query_info_with_filter_dag, context, @@ -1391,7 +1388,6 @@ MergeTreeDataSelectAnalysisResultPtr ReadFromMergeTree::selectRangesToRead( return selectRangesToReadImpl( std::move(parts), std::move(alter_conversions), - metadata_snapshot_base, metadata_snapshot, query_info, context, @@ -1407,7 +1403,6 @@ MergeTreeDataSelectAnalysisResultPtr ReadFromMergeTree::selectRangesToRead( MergeTreeDataSelectAnalysisResultPtr ReadFromMergeTree::selectRangesToReadImpl( MergeTreeData::DataPartsVector parts, std::vector alter_conversions, - const StorageMetadataPtr & metadata_snapshot_base, const StorageMetadataPtr & metadata_snapshot, const SelectQueryInfo & query_info, ContextPtr context, @@ -1468,7 +1463,7 @@ MergeTreeDataSelectAnalysisResultPtr ReadFromMergeTree::selectRangesToReadImpl( parts, alter_conversions, part_values, - metadata_snapshot_base, + metadata_snapshot, data, context, max_block_numbers_to_read.get(), diff --git a/src/Processors/QueryPlan/ReadFromMergeTree.h b/src/Processors/QueryPlan/ReadFromMergeTree.h index e3406addf6c3..b4ae10ff0c7e 100644 --- a/src/Processors/QueryPlan/ReadFromMergeTree.h +++ b/src/Processors/QueryPlan/ReadFromMergeTree.h @@ -178,7 +178,6 @@ class ReadFromMergeTree final : public SourceStepWithFilter std::vector alter_conversions, const PrewhereInfoPtr & prewhere_info, const ActionDAGNodes & added_filter_nodes, - const StorageMetadataPtr & metadata_snapshot_base, const StorageMetadataPtr & metadata_snapshot, const SelectQueryInfo & query_info, ContextPtr context, @@ -228,7 +227,6 @@ class ReadFromMergeTree final : public SourceStepWithFilter static MergeTreeDataSelectAnalysisResultPtr selectRangesToReadImpl( MergeTreeData::DataPartsVector parts, std::vector alter_conversions, - const StorageMetadataPtr & metadata_snapshot_base, const StorageMetadataPtr & metadata_snapshot, const SelectQueryInfo & query_info, ContextPtr context, diff --git a/src/Storages/MergeTree/MergeTreeData.cpp b/src/Storages/MergeTree/MergeTreeData.cpp index 145875938167..99f3aed0b933 100644 --- a/src/Storages/MergeTree/MergeTreeData.cpp +++ b/src/Storages/MergeTree/MergeTreeData.cpp @@ -6536,7 +6536,6 @@ static void selectBestProjection( projection_parts, candidate.prewhere_info, candidate.required_columns, - storage_snapshot->metadata, candidate.desc->metadata, query_info, added_filter_nodes, @@ -6561,7 +6560,6 @@ static void selectBestProjection( query_info.prewhere_info, required_columns, storage_snapshot->metadata, - storage_snapshot->metadata, query_info, // TODO syntax_analysis_result set in index added_filter_nodes, query_context, @@ -7213,7 +7211,6 @@ std::optional MergeTreeData::getQueryProcessingStageWithAgg query_info.prewhere_info, analysis_result.required_columns, metadata_snapshot, - metadata_snapshot, query_info, added_filter_nodes, query_context, @@ -7246,7 +7243,6 @@ std::optional MergeTreeData::getQueryProcessingStageWithAgg query_info.prewhere_info, analysis_result.required_columns, metadata_snapshot, - metadata_snapshot, query_info, added_filter_nodes, query_context, @@ -7386,7 +7382,6 @@ bool MergeTreeData::canUseParallelReplicasBasedOnPKAnalysis( query_info.prewhere_info, storage_snapshot->getMetadataForQuery()->getColumns().getAll().getNames(), storage_snapshot->metadata, - storage_snapshot->metadata, query_info, /*added_filter_nodes*/ActionDAGNodes{}, query_context, diff --git a/src/Storages/MergeTree/MergeTreeDataSelectExecutor.cpp b/src/Storages/MergeTree/MergeTreeDataSelectExecutor.cpp index 5b844d80211d..62dc39ebc6d0 100644 --- a/src/Storages/MergeTree/MergeTreeDataSelectExecutor.cpp +++ b/src/Storages/MergeTree/MergeTreeDataSelectExecutor.cpp @@ -816,6 +816,7 @@ void MergeTreeDataSelectExecutor::filterPartsByPartition( if (metadata_snapshot->hasPartitionKey()) { + chassert(minmax_idx_condition && partition_pruner); const auto & partition_key = metadata_snapshot->getPartitionKey(); minmax_columns_types = data.getMinMaxColumnsTypes(partition_key); @@ -1231,7 +1232,6 @@ MergeTreeDataSelectAnalysisResultPtr MergeTreeDataSelectExecutor::estimateNumMar MergeTreeData::DataPartsVector parts, const PrewhereInfoPtr & prewhere_info, const Names & column_names_to_return, - const StorageMetadataPtr & metadata_snapshot_base, const StorageMetadataPtr & metadata_snapshot, const SelectQueryInfo & query_info, const ActionDAGNodes & added_filter_nodes, @@ -1260,7 +1260,6 @@ MergeTreeDataSelectAnalysisResultPtr MergeTreeDataSelectExecutor::estimateNumMar /*alter_conversions=*/ {}, prewhere_info, added_filter_nodes, - metadata_snapshot_base, metadata_snapshot, query_info, context, diff --git a/src/Storages/MergeTree/MergeTreeDataSelectExecutor.h b/src/Storages/MergeTree/MergeTreeDataSelectExecutor.h index 05d11a858b33..53f4ceba06ac 100644 --- a/src/Storages/MergeTree/MergeTreeDataSelectExecutor.h +++ b/src/Storages/MergeTree/MergeTreeDataSelectExecutor.h @@ -59,7 +59,6 @@ class MergeTreeDataSelectExecutor MergeTreeData::DataPartsVector parts, const PrewhereInfoPtr & prewhere_info, const Names & column_names, - const StorageMetadataPtr & metadata_snapshot_base, const StorageMetadataPtr & metadata_snapshot, const SelectQueryInfo & query_info, const ActionDAGNodes & added_filter_nodes, diff --git a/tests/queries/0_stateless/01710_projection_fix_crash.reference b/tests/queries/0_stateless/01710_projection_fix_crash.reference new file mode 100644 index 000000000000..e69de29bb2d1 diff --git a/tests/queries/0_stateless/01710_projection_fix_crash.sql b/tests/queries/0_stateless/01710_projection_fix_crash.sql new file mode 100644 index 000000000000..703a773ebcab --- /dev/null +++ b/tests/queries/0_stateless/01710_projection_fix_crash.sql @@ -0,0 +1,15 @@ +set force_index_by_date = 1; + +create table xxxxx (col1 String, col2 String, _time DateTime, projection p (select * order by col2)) engine=MergeTree partition by col1 order by tuple(); + +create table yyyyyyy (col1 String, col2 String, _time DateTime, projection p (select * order by col2)) engine=MergeTree partition by col1 order by tuple(); + +insert into xxxxx (col1, col2, _time) values ('xxx', 'zzzz', now()+1); +insert into yyyyyyy (col1, col2, _time) values ('xxx', 'zzzz', now()); + +SELECT count() +FROM xxxxx +WHERE (col1 = 'xxx') AND (_time = ( + SELECT max(_time) + FROM yyyyyyy + WHERE (col1 = 'xxx') AND (col2 = 'zzzz') AND (_time > (now() - toIntervalDay(3))))) diff --git a/tests/queries/0_stateless/02965_projection_with_partition_pruning.reference b/tests/queries/0_stateless/02965_projection_with_partition_pruning.reference new file mode 100644 index 000000000000..5816b4eb49bb --- /dev/null +++ b/tests/queries/0_stateless/02965_projection_with_partition_pruning.reference @@ -0,0 +1 @@ +3 4 diff --git a/tests/queries/0_stateless/02965_projection_with_partition_pruning.sql b/tests/queries/0_stateless/02965_projection_with_partition_pruning.sql new file mode 100644 index 000000000000..92f7cc0671c9 --- /dev/null +++ b/tests/queries/0_stateless/02965_projection_with_partition_pruning.sql @@ -0,0 +1,9 @@ +drop table if exists a; + +create table a (i int, j int, projection p (select * order by j)) engine MergeTree partition by i order by tuple() settings index_granularity = 1; + +insert into a values (1, 2), (0, 5), (3, 4); + +select * from a where i > 0 and j = 4 settings force_index_by_date = 1; + +drop table a; From 0ed34087bdc831d908bf34fa9ebae33dfa9813d5 Mon Sep 17 00:00:00 2001 From: Vasily Nemkov Date: Thu, 17 Oct 2024 18:53:06 +0200 Subject: [PATCH 117/123] Bumped version to 23.8.16.42.altinitystable --- cmake/autogenerated_versions.txt | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/cmake/autogenerated_versions.txt b/cmake/autogenerated_versions.txt index 879920553ed2..3da9f1d8a216 100644 --- a/cmake/autogenerated_versions.txt +++ b/cmake/autogenerated_versions.txt @@ -8,9 +8,9 @@ SET(VERSION_MINOR 8) SET(VERSION_PATCH 16) SET(VERSION_GITHASH 060ff8e813a4a16a540063127f8c91e2108d9adf) -SET(VERSION_TWEAK 41) +SET(VERSION_TWEAK 42) SET(VERSION_FLAVOUR altinitystable) -SET(VERSION_DESCRIBE v23.8.16.41.altinitystable) -SET(VERSION_STRING 23.8.16.41.altinitystable) +SET(VERSION_DESCRIBE v23.8.16.42.altinitystable) +SET(VERSION_STRING 23.8.16.42.altinitystable) # end of autochange From 2733026b27d6fed4cd610e273fd50238f8b03807 Mon Sep 17 00:00:00 2001 From: MyroTk Date: Thu, 17 Oct 2024 15:23:22 -0700 Subject: [PATCH 118/123] Swap to snapshot runners and set pr number to 1 for scheduled runs --- .github/actions/common_setup/action.yml | 10 +++ .github/workflows/release_branches.yml | 88 ++++++++++++------------- tests/ci/docker_images_check.py | 6 +- tests/ci/pr_info.py | 4 +- 4 files changed, 58 insertions(+), 50 deletions(-) diff --git a/.github/actions/common_setup/action.yml b/.github/actions/common_setup/action.yml index b02413adc44c..ab446c3ae7ae 100644 --- a/.github/actions/common_setup/action.yml +++ b/.github/actions/common_setup/action.yml @@ -26,6 +26,16 @@ runs: echo "The GITHUB_JOB_OVERRIDDEN ENV is unset, and must be set for the nested jobs" exit 1 fi + - name: Setup zram + shell: bash + run: | + sudo modprobe zram + MemTotal=$(grep -Po "(?<=MemTotal:)\s+\d+" /proc/meminfo) # KiB + Percent=200 + ZRAM_SIZE=$(($MemTotal / 1024 / 1024 * $Percent / 100)) # Convert to GiB + .github/retry.sh 30 2 sudo zramctl --size ${ZRAM_SIZE}GiB --algorithm zstd /dev/zram0 + sudo mkswap /dev/zram0 && sudo swapon -p 100 /dev/zram0 + sudo sysctl vm.swappiness=200 - name: Setup $TEMP_PATH shell: bash run: | diff --git a/.github/workflows/release_branches.yml b/.github/workflows/release_branches.yml index a8113d891bd8..3d6478785505 100644 --- a/.github/workflows/release_branches.yml +++ b/.github/workflows/release_branches.yml @@ -30,7 +30,7 @@ on: # yamllint disable-line rule:truthy jobs: DockerHubPushAarch64: - runs-on: [self-hosted, altinity-on-demand, altinity-type-cax41, altinity-in-hel1, altinity-image-arm-system-ubuntu-22.04] + runs-on: [self-hosted, altinity-on-demand, altinity-type-cax41, altinity-in-hel1, altinity-image-arm-snapshot-22.04-arm, altinity-startup-snapshot, altinity-setup-none] steps: - name: Check out repository code uses: Altinity/checkout@19599efdf36c4f3f30eb55d5bb388896faea69f6 @@ -52,7 +52,7 @@ jobs: path: ${{ runner.temp }}/docker_images_check/changed_images_aarch64.json DockerHubPushAmd64: - runs-on: [self-hosted, altinity-on-demand, altinity-type-cpx51, altinity-in-ash, altinity-image-x86-system-ubuntu-22.04] + runs-on: [self-hosted, altinity-on-demand, altinity-type-cpx51, altinity-in-ash, altinity-image-x86-snapshot-22.04-amd, altinity-startup-snapshot, altinity-setup-none] steps: - name: Check out repository code uses: Altinity/checkout@19599efdf36c4f3f30eb55d5bb388896faea69f6 @@ -75,7 +75,7 @@ jobs: DockerHubPush: needs: [DockerHubPushAmd64, DockerHubPushAarch64] - runs-on: [self-hosted, altinity-on-demand, altinity-type-cpx51, altinity-in-ash, altinity-image-x86-system-ubuntu-22.04] + runs-on: [self-hosted, altinity-on-demand, altinity-type-cpx51, altinity-in-ash, altinity-image-x86-snapshot-22.04-amd, altinity-startup-snapshot, altinity-setup-none] steps: - name: Check out repository code uses: Altinity/checkout@19599efdf36c4f3f30eb55d5bb388896faea69f6 @@ -119,7 +119,7 @@ jobs: secrets: inherit with: test_name: Compatibility check X86 - runner_type: altinity-on-demand, altinity-type-cpx51, altinity-in-ash, altinity-image-x86-system-ubuntu-22.04 + runner_type: altinity-on-demand, altinity-type-cpx51, altinity-in-ash, altinity-image-x86-snapshot-22.04-amd, altinity-startup-snapshot, altinity-setup-none timeout_minutes: 180 run_command: | cd "$REPO_COPY/tests/ci" @@ -131,7 +131,7 @@ jobs: secrets: inherit with: test_name: Compatibility check Aarch64 - runner_type: altinity-on-demand, altinity-type-cax41, altinity-image-arm-system-ubuntu-22.04 + runner_type: altinity-on-demand, altinity-type-cax41, altinity-image-arm-snapshot-22.04-arm, altinity-startup-snapshot, altinity-setup-none timeout_minutes: 180 run_command: | cd "$REPO_COPY/tests/ci" @@ -148,7 +148,7 @@ jobs: build_name: package_release checkout_depth: 0 timeout_minutes: 180 - runner_type: altinity-setup-builder, altinity-type-ccx53, altinity-on-demand, altinity-in-ash, altinity-image-x86-system-ubuntu-22.04 + runner_type: altinity-setup-builder, altinity-type-ccx53, altinity-on-demand, altinity-in-ash, altinity-image-x86-snapshot-22.04-amd, altinity-startup-snapshot, altinity-setup-none additional_envs: | CLICKHOUSE_STABLE_VERSION_SUFFIX=altinitystable @@ -160,7 +160,7 @@ jobs: build_name: package_aarch64 checkout_depth: 0 timeout_minutes: 180 - runner_type: altinity-setup-builder, altinity-type-ccx53, altinity-on-demand, altinity-in-ash, altinity-image-x86-system-ubuntu-22.04 + runner_type: altinity-setup-builder, altinity-type-ccx53, altinity-on-demand, altinity-in-ash, altinity-image-x86-snapshot-22.04-amd, altinity-startup-snapshot, altinity-setup-none additional_envs: | CLICKHOUSE_STABLE_VERSION_SUFFIX=altinitystable @@ -172,7 +172,7 @@ jobs: build_name: package_asan checkout_depth: 0 timeout_minutes: 180 - runner_type: altinity-on-demand, altinity-setup-builder, altinity-type-ccx53, altinity-in-ash, altinity-image-x86-system-ubuntu-22.04 + runner_type: altinity-on-demand, altinity-setup-builder, altinity-type-ccx53, altinity-in-ash, altinity-image-x86-snapshot-22.04-amd, altinity-startup-snapshot, altinity-setup-none additional_envs: | CLICKHOUSE_STABLE_VERSION_SUFFIX=altinitystable @@ -184,7 +184,7 @@ jobs: build_name: package_ubsan checkout_depth: 0 timeout_minutes: 180 - runner_type: altinity-on-demand, altinity-setup-builder, altinity-type-ccx53, altinity-in-ash, altinity-image-x86-system-ubuntu-22.04 + runner_type: altinity-on-demand, altinity-setup-builder, altinity-type-ccx53, altinity-in-ash, altinity-image-x86-snapshot-22.04-amd, altinity-startup-snapshot, altinity-setup-none additional_envs: | CLICKHOUSE_STABLE_VERSION_SUFFIX=altinitystable @@ -196,7 +196,7 @@ jobs: build_name: package_tsan checkout_depth: 0 timeout_minutes: 180 - runner_type: altinity-on-demand, altinity-setup-builder, altinity-type-ccx53, altinity-in-ash, altinity-image-x86-system-ubuntu-22.04 + runner_type: altinity-on-demand, altinity-setup-builder, altinity-type-ccx53, altinity-in-ash, altinity-image-x86-snapshot-22.04-amd, altinity-startup-snapshot, altinity-setup-none additional_envs: | CLICKHOUSE_STABLE_VERSION_SUFFIX=altinitystable @@ -208,7 +208,7 @@ jobs: build_name: package_msan checkout_depth: 0 timeout_minutes: 180 - runner_type: altinity-on-demand, altinity-setup-builder, altinity-type-ccx53, altinity-in-ash, altinity-image-x86-system-ubuntu-22.04 + runner_type: altinity-on-demand, altinity-setup-builder, altinity-type-ccx53, altinity-in-ash, altinity-image-x86-snapshot-22.04-amd, altinity-startup-snapshot, altinity-setup-none additional_envs: | CLICKHOUSE_STABLE_VERSION_SUFFIX=altinitystable @@ -220,7 +220,7 @@ jobs: build_name: package_debug checkout_depth: 0 timeout_minutes: 180 - runner_type: altinity-on-demand, altinity-setup-builder, altinity-type-ccx53, altinity-in-ash, altinity-image-x86-system-ubuntu-22.04 + runner_type: altinity-on-demand, altinity-setup-builder, altinity-type-ccx53, altinity-in-ash, altinity-image-x86-snapshot-22.04-amd, altinity-startup-snapshot, altinity-setup-none additional_envs: | CLICKHOUSE_STABLE_VERSION_SUFFIX=altinitystable @@ -231,7 +231,7 @@ jobs: needs: - BuilderDebRelease - BuilderDebAarch64 - runs-on: [self-hosted, altinity-on-demand, altinity-type-cpx41, altinity-in-ash, altinity-image-x86-system-ubuntu-22.04] + runs-on: [self-hosted, altinity-on-demand, altinity-type-cpx41, altinity-in-ash, altinity-image-x86-snapshot-22.04-amd, altinity-startup-snapshot, altinity-setup-none] timeout-minutes: 180 steps: - name: Check out repository code @@ -267,7 +267,7 @@ jobs: secrets: inherit with: test_name: ClickHouse build check - runner_type: altinity-on-demand, altinity-setup-reporter, altinity-type-cax11, altinity-in-hel1, altinity-image-arm-system-ubuntu-22.04 + runner_type: altinity-on-demand, altinity-type-cax11, altinity-in-hel1, altinity-image-arm-snapshot-22.04-arm, altinity-startup-snapshot, altinity-setup-none timeout_minutes: 180 additional_envs: | NEEDS_DATA< Date: Thu, 17 Oct 2024 23:05:51 +0000 Subject: [PATCH 119/123] Updated package maintainer metadata --- packages/clickhouse-client.yaml | 2 +- packages/clickhouse-common-static-dbg.yaml | 2 +- packages/clickhouse-common-static.yaml | 2 +- packages/clickhouse-keeper-dbg.yaml | 2 +- packages/clickhouse-keeper.yaml | 2 +- packages/clickhouse-server.yaml | 2 +- 6 files changed, 6 insertions(+), 6 deletions(-) diff --git a/packages/clickhouse-client.yaml b/packages/clickhouse-client.yaml index 059562835d8c..cc87aaf338f8 100644 --- a/packages/clickhouse-client.yaml +++ b/packages/clickhouse-client.yaml @@ -16,7 +16,7 @@ homepage: "https://altinity.com/" license: "Apache" section: "database" priority: "optional" -maintainer: "ClickHouse Dev Team " +maintainer: "Altinity Dev Team https://github.com/Altinity/ClickHouse/" deb: fields: Source: clickhouse diff --git a/packages/clickhouse-common-static-dbg.yaml b/packages/clickhouse-common-static-dbg.yaml index 63b95b034944..0821b0291805 100644 --- a/packages/clickhouse-common-static-dbg.yaml +++ b/packages/clickhouse-common-static-dbg.yaml @@ -16,7 +16,7 @@ homepage: "https://altinity.com/" license: "Apache" section: "database" priority: "optional" -maintainer: "ClickHouse Dev Team " +maintainer: "Altinity Dev Team https://github.com/Altinity/ClickHouse" deb: fields: Source: clickhouse diff --git a/packages/clickhouse-common-static.yaml b/packages/clickhouse-common-static.yaml index 96dd2d890a19..0140db3994b3 100644 --- a/packages/clickhouse-common-static.yaml +++ b/packages/clickhouse-common-static.yaml @@ -16,7 +16,7 @@ homepage: "https://altinity.com/" license: "Apache" section: "database" priority: "optional" -maintainer: "ClickHouse Dev Team " +maintainer: "Altinity Dev Team https://github.com/Altinity/ClickHouse" deb: fields: Source: clickhouse diff --git a/packages/clickhouse-keeper-dbg.yaml b/packages/clickhouse-keeper-dbg.yaml index c1c8a178ba74..07aaafae83ef 100644 --- a/packages/clickhouse-keeper-dbg.yaml +++ b/packages/clickhouse-keeper-dbg.yaml @@ -16,7 +16,7 @@ homepage: "https://altinity.com/" license: "Apache" section: "database" priority: "optional" -maintainer: "ClickHouse Dev Team " +maintainer: "Altinity Dev Team https://github.com/Altinity/ClickHouse" deb: fields: Source: clickhouse diff --git a/packages/clickhouse-keeper.yaml b/packages/clickhouse-keeper.yaml index f9780cd4ad9c..4fea5798ff9a 100644 --- a/packages/clickhouse-keeper.yaml +++ b/packages/clickhouse-keeper.yaml @@ -16,7 +16,7 @@ homepage: "https://altinity.com/" license: "Apache" section: "database" priority: "optional" -maintainer: "ClickHouse Dev Team " +maintainer: "Altinity Dev Team https://github.com/Altinity/ClickHouse" deb: fields: Source: clickhouse diff --git a/packages/clickhouse-server.yaml b/packages/clickhouse-server.yaml index 9a004c3eb1c6..3f396fe35137 100644 --- a/packages/clickhouse-server.yaml +++ b/packages/clickhouse-server.yaml @@ -16,7 +16,7 @@ homepage: "https://altinity.com/" license: "Apache" section: "database" priority: "optional" -maintainer: "ClickHouse Dev Team " +maintainer: "Altinity Dev Team https://github.com/Altinity/ClickHouse" deb: fields: Source: clickhouse From 875ab567a2ec89ca69806c135e41d8e084e381d4 Mon Sep 17 00:00:00 2001 From: Vasily Nemkov Date: Fri, 18 Oct 2024 01:43:12 +0200 Subject: [PATCH 120/123] Update 01710_projection_fix_crash.reference Fixed .reference with accordance with the reference (24.3) version output --- tests/queries/0_stateless/01710_projection_fix_crash.reference | 1 + 1 file changed, 1 insertion(+) diff --git a/tests/queries/0_stateless/01710_projection_fix_crash.reference b/tests/queries/0_stateless/01710_projection_fix_crash.reference index e69de29bb2d1..18748286e5b8 100644 --- a/tests/queries/0_stateless/01710_projection_fix_crash.reference +++ b/tests/queries/0_stateless/01710_projection_fix_crash.reference @@ -0,0 +1 @@ +0 From d77df9e2122c1839ec759fa08f3cfd61478d5a5f Mon Sep 17 00:00:00 2001 From: MyroTk <44327070+MyroTk@users.noreply.github.com> Date: Fri, 18 Oct 2024 09:32:34 -0700 Subject: [PATCH 121/123] Update sqllogic Dockerfile to 24.3 version --- docker/test/sqllogic/Dockerfile | 31 +++++++++++++++++-------------- 1 file changed, 17 insertions(+), 14 deletions(-) diff --git a/docker/test/sqllogic/Dockerfile b/docker/test/sqllogic/Dockerfile index 508fd25d6f42..34af8825eef6 100644 --- a/docker/test/sqllogic/Dockerfile +++ b/docker/test/sqllogic/Dockerfile @@ -15,28 +15,31 @@ RUN apt-get update --yes \ unixodbc-dev \ odbcinst \ sudo \ - && apt-get clean + && apt-get clean \ + && rm -rf /var/lib/apt/lists/* /var/cache/debconf /tmp/* RUN pip3 install \ numpy \ pyodbc \ - deepdiff + deepdiff \ + sqlglot -ARG odbc_repo="https://github.com/ClickHouse/clickhouse-odbc.git" +ARG odbc_driver_url="https://github.com/ClickHouse/clickhouse-odbc/releases/download/v1.1.6.20200320/clickhouse-odbc-1.1.6-Linux.tar.gz" + +RUN mkdir -p /tmp/clickhouse-odbc-tmp \ + && cd /tmp/clickhouse-odbc-tmp \ + && curl -L ${odbc_driver_url} | tar --strip-components=1 -xz clickhouse-odbc-1.1.6-Linux \ + && mkdir /usr/local/lib64 -p \ + && cp /tmp/clickhouse-odbc-tmp/lib64/*.so /usr/local/lib64/ \ + && odbcinst -i -d -f /tmp/clickhouse-odbc-tmp/share/doc/clickhouse-odbc/config/odbcinst.ini.sample \ + && odbcinst -i -s -l -f /tmp/clickhouse-odbc-tmp/share/doc/clickhouse-odbc/config/odbc.ini.sample \ + && sed -i 's"=libclickhouseodbc"=/usr/local/lib64/libclickhouseodbc"' /etc/odbcinst.ini \ + && rm -rf /tmp/clickhouse-odbc-tmp -RUN git clone --recursive ${odbc_repo} \ - && mkdir -p /clickhouse-odbc/build \ - && cmake -S /clickhouse-odbc -B /clickhouse-odbc/build \ - && ls /clickhouse-odbc/build/driver \ - && make -j 10 -C /clickhouse-odbc/build \ - && ls /clickhouse-odbc/build/driver \ - && mkdir -p /usr/local/lib64/ && cp /clickhouse-odbc/build/driver/lib*.so /usr/local/lib64/ \ - && odbcinst -i -d -f /clickhouse-odbc/packaging/odbcinst.ini.sample \ - && odbcinst -i -s -l -f /clickhouse-odbc/packaging/odbc.ini.sample ENV TZ=Europe/Amsterdam -ENV MAX_RUN_TIME=900 -RUN ln -snf /usr/share/zoneinfo/$TZ /etc/localtime && echo $TZ > /etc/timezone +ENV MAX_RUN_TIME=9000 +RUN ln -snf /usr/share/zoneinfo/$TZ /etc/localtime && echo $TZ > /etc/timezon ARG sqllogic_test_repo="https://github.com/gregrahn/sqllogictest.git" From 0122a6c90278df9f5fbd3b4b0ec7b51494f3a19b Mon Sep 17 00:00:00 2001 From: MyroTk <44327070+MyroTk@users.noreply.github.com> Date: Fri, 18 Oct 2024 10:21:13 -0700 Subject: [PATCH 122/123] Relocate builder runners to avoid hetzner outage --- .github/workflows/release_branches.yml | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/.github/workflows/release_branches.yml b/.github/workflows/release_branches.yml index 3d6478785505..bcc42b85f015 100644 --- a/.github/workflows/release_branches.yml +++ b/.github/workflows/release_branches.yml @@ -148,7 +148,7 @@ jobs: build_name: package_release checkout_depth: 0 timeout_minutes: 180 - runner_type: altinity-setup-builder, altinity-type-ccx53, altinity-on-demand, altinity-in-ash, altinity-image-x86-snapshot-22.04-amd, altinity-startup-snapshot, altinity-setup-none + runner_type: altinity-on-demand, altinity-type-ccx53, altinity-in-hil, altinity-image-x86-snapshot-22.04-amd, altinity-startup-snapshot, altinity-setup-none additional_envs: | CLICKHOUSE_STABLE_VERSION_SUFFIX=altinitystable @@ -160,7 +160,7 @@ jobs: build_name: package_aarch64 checkout_depth: 0 timeout_minutes: 180 - runner_type: altinity-setup-builder, altinity-type-ccx53, altinity-on-demand, altinity-in-ash, altinity-image-x86-snapshot-22.04-amd, altinity-startup-snapshot, altinity-setup-none + runner_type: altinity-on-demand, altinity-type-ccx53, altinity-in-hil, altinity-image-x86-snapshot-22.04-amd, altinity-startup-snapshot, altinity-setup-none additional_envs: | CLICKHOUSE_STABLE_VERSION_SUFFIX=altinitystable @@ -172,7 +172,7 @@ jobs: build_name: package_asan checkout_depth: 0 timeout_minutes: 180 - runner_type: altinity-on-demand, altinity-setup-builder, altinity-type-ccx53, altinity-in-ash, altinity-image-x86-snapshot-22.04-amd, altinity-startup-snapshot, altinity-setup-none + runner_type: altinity-on-demand, altinity-type-ccx53, altinity-in-hil, altinity-image-x86-snapshot-22.04-amd, altinity-startup-snapshot, altinity-setup-none additional_envs: | CLICKHOUSE_STABLE_VERSION_SUFFIX=altinitystable @@ -184,7 +184,7 @@ jobs: build_name: package_ubsan checkout_depth: 0 timeout_minutes: 180 - runner_type: altinity-on-demand, altinity-setup-builder, altinity-type-ccx53, altinity-in-ash, altinity-image-x86-snapshot-22.04-amd, altinity-startup-snapshot, altinity-setup-none + runner_type: altinity-on-demand, altinity-type-ccx53, altinity-in-hil, altinity-image-x86-snapshot-22.04-amd, altinity-startup-snapshot, altinity-setup-none additional_envs: | CLICKHOUSE_STABLE_VERSION_SUFFIX=altinitystable @@ -196,7 +196,7 @@ jobs: build_name: package_tsan checkout_depth: 0 timeout_minutes: 180 - runner_type: altinity-on-demand, altinity-setup-builder, altinity-type-ccx53, altinity-in-ash, altinity-image-x86-snapshot-22.04-amd, altinity-startup-snapshot, altinity-setup-none + runner_type: altinity-on-demand, altinity-type-ccx53, altinity-in-hil, altinity-image-x86-snapshot-22.04-amd, altinity-startup-snapshot, altinity-setup-none additional_envs: | CLICKHOUSE_STABLE_VERSION_SUFFIX=altinitystable @@ -208,7 +208,7 @@ jobs: build_name: package_msan checkout_depth: 0 timeout_minutes: 180 - runner_type: altinity-on-demand, altinity-setup-builder, altinity-type-ccx53, altinity-in-ash, altinity-image-x86-snapshot-22.04-amd, altinity-startup-snapshot, altinity-setup-none + runner_type: altinity-on-demand, altinity-type-ccx53, altinity-in-hil, altinity-image-x86-snapshot-22.04-amd, altinity-startup-snapshot, altinity-setup-none additional_envs: | CLICKHOUSE_STABLE_VERSION_SUFFIX=altinitystable @@ -220,7 +220,7 @@ jobs: build_name: package_debug checkout_depth: 0 timeout_minutes: 180 - runner_type: altinity-on-demand, altinity-setup-builder, altinity-type-ccx53, altinity-in-ash, altinity-image-x86-snapshot-22.04-amd, altinity-startup-snapshot, altinity-setup-none + runner_type: altinity-on-demand, altinity-type-ccx53, altinity-in-hil, altinity-image-x86-snapshot-22.04-amd, altinity-startup-snapshot, altinity-setup-none additional_envs: | CLICKHOUSE_STABLE_VERSION_SUFFIX=altinitystable From 95383fa740614c94aec608d2acf6188e90bfb7f9 Mon Sep 17 00:00:00 2001 From: MyroTk <44327070+MyroTk@users.noreply.github.com> Date: Fri, 18 Oct 2024 17:02:06 -0700 Subject: [PATCH 123/123] Tiered Storage regression test fix --- .github/workflows/regression.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/regression.yml b/.github/workflows/regression.yml index 66d89ddede43..66682ea320e1 100644 --- a/.github/workflows/regression.yml +++ b/.github/workflows/regression.yml @@ -525,6 +525,7 @@ jobs: uses: actions/checkout@v4 with: repository: Altinity/clickhouse-regression + ref: ${{ inputs.commit }} - name: Set envs run: | cat >> "$GITHUB_ENV" << 'EOF'