Merge pull request #504 from Altinity/customizations/23.8.16

23.8.16.42 Pre-release

.github/actions/common_setup/action.yml

@@ -26,6 +26,16 @@ runs:
             echo "The GITHUB_JOB_OVERRIDDEN ENV is unset, and must be set for the nested jobs"
             exit 1
           fi
+    - name: Setup zram
+      shell: bash
+      run: |
+          sudo modprobe zram
+          MemTotal=$(grep -Po "(?<=MemTotal:)\s+\d+" /proc/meminfo) # KiB
+          Percent=200
+          ZRAM_SIZE=$(($MemTotal / 1024 / 1024 * $Percent / 100)) # Convert to GiB
+          .github/retry.sh 30 2 sudo zramctl --size ${ZRAM_SIZE}GiB --algorithm zstd /dev/zram0
+          sudo mkswap /dev/zram0 && sudo swapon -p 100 /dev/zram0
+          sudo sysctl vm.swappiness=200
     - name: Setup $TEMP_PATH
       shell: bash
       run: |

.github/workflows/regression.yml

@@ -93,6 +93,9 @@ env:
   AWS_DEFAULT_REGION: ${{ secrets.AWS_REPORT_REGION }}
   DOCKER_USERNAME: ${{ secrets.DOCKER_USERNAME }}
   DOCKER_PASSWORD: ${{ secrets.DOCKER_PASSWORD }}
+  CHECKS_DATABASE_HOST: ${{ secrets.CHECKS_DATABASE_HOST }}
+  CHECKS_DATABASE_USER: ${{ secrets.CHECKS_DATABASE_USER }}
+  CHECKS_DATABASE_PASSWORD:  ${{ secrets.CHECKS_DATABASE_PASSWORD }}
   args: --test-to-end
     --no-colors
     --local
@@ -522,6 +525,7 @@ jobs:
         uses: actions/checkout@v4
         with:
           repository: Altinity/clickhouse-regression
+          ref: ${{ inputs.commit }}
       - name: Set envs
         run: |
           cat >> "$GITHUB_ENV" << 'EOF'

cmake/autogenerated_versions.txt

@@ -8,9 +8,9 @@ SET(VERSION_MINOR 8)
 SET(VERSION_PATCH 16)
 
 SET(VERSION_GITHASH 060ff8e813a4a16a540063127f8c91e2108d9adf)
-SET(VERSION_TWEAK 41)
+SET(VERSION_TWEAK 42)
 SET(VERSION_FLAVOUR altinitystable)
 
-SET(VERSION_DESCRIBE v23.8.16.41.altinitystable)
-SET(VERSION_STRING 23.8.16.41.altinitystable)
+SET(VERSION_DESCRIBE v23.8.16.42.altinitystable)
+SET(VERSION_STRING 23.8.16.42.altinitystable)
 # end of autochange

docker/test/integration/runner/Dockerfile

@@ -9,7 +9,7 @@ RUN sed -i "s|http://archive.ubuntu.com|$apt_archive|g" /etc/apt/sources.list
 RUN apt-get update \
     && env DEBIAN_FRONTEND=noninteractive apt-get install --yes \
     adduser='3.11*' \
-    ca-certificates='2023*' \
+    ca-certificates \
     bash='5.1-*' \
     btrfs-progs='5.16.*' \
     e2fsprogs='1.46.*' \
@@ -96,7 +96,7 @@ RUN python3 -m pip install --no-cache-dir \
     pytest-repeat~=0.9.3 \
     pytest-timeout~=2.2.0 \
     pytest-xdist~=3.5.0 \
-    pytz~=2023.3.post1 \
+    pytz \
     pyyaml~=5.3.1 \
     redis~=5.0.1 \
     requests==2.31.0 \

docker/test/sqllogic/Dockerfile

@@ -15,28 +15,31 @@ RUN apt-get update --yes \
             unixodbc-dev \
             odbcinst \
             sudo \
-    && apt-get clean
+    && apt-get clean \
+    && rm -rf /var/lib/apt/lists/* /var/cache/debconf /tmp/*
 
 RUN pip3 install \
     numpy \
     pyodbc \
-    deepdiff
+    deepdiff \
+    sqlglot
 
-ARG odbc_repo="https://github.com/ClickHouse/clickhouse-odbc.git"
+ARG odbc_driver_url="https://github.com/ClickHouse/clickhouse-odbc/releases/download/v1.1.6.20200320/clickhouse-odbc-1.1.6-Linux.tar.gz"
+
+RUN mkdir -p /tmp/clickhouse-odbc-tmp \
+  && cd /tmp/clickhouse-odbc-tmp \
+  && curl -L ${odbc_driver_url} | tar --strip-components=1 -xz clickhouse-odbc-1.1.6-Linux \
+  && mkdir /usr/local/lib64 -p \
+  && cp /tmp/clickhouse-odbc-tmp/lib64/*.so /usr/local/lib64/ \
+  && odbcinst -i -d -f /tmp/clickhouse-odbc-tmp/share/doc/clickhouse-odbc/config/odbcinst.ini.sample \
+  && odbcinst -i -s -l -f /tmp/clickhouse-odbc-tmp/share/doc/clickhouse-odbc/config/odbc.ini.sample \
+  && sed -i 's"=libclickhouseodbc"=/usr/local/lib64/libclickhouseodbc"' /etc/odbcinst.ini \
+  && rm -rf /tmp/clickhouse-odbc-tmp
 
-RUN git clone --recursive ${odbc_repo} \
-    && mkdir -p /clickhouse-odbc/build \
-    && cmake -S /clickhouse-odbc -B /clickhouse-odbc/build \
-    && ls /clickhouse-odbc/build/driver \
-    && make -j 10 -C /clickhouse-odbc/build \
-    && ls /clickhouse-odbc/build/driver \
-    && mkdir -p /usr/local/lib64/ && cp /clickhouse-odbc/build/driver/lib*.so /usr/local/lib64/ \
-    && odbcinst -i -d -f /clickhouse-odbc/packaging/odbcinst.ini.sample \
-    && odbcinst -i -s -l -f /clickhouse-odbc/packaging/odbc.ini.sample
 
 ENV TZ=Europe/Amsterdam
-ENV MAX_RUN_TIME=900
-RUN ln -snf /usr/share/zoneinfo/$TZ /etc/localtime && echo $TZ > /etc/timezone
+ENV MAX_RUN_TIME=9000
+RUN ln -snf /usr/share/zoneinfo/$TZ /etc/localtime && echo $TZ > /etc/timezon
 
 ARG sqllogic_test_repo="https://github.com/gregrahn/sqllogictest.git"
 

docker/test/util/Dockerfile

@@ -12,7 +12,7 @@ RUN apt-get update \
     && apt-get install \
         apt-transport-https='2.4.*' \
         apt-utils='2.4.*' \
-        ca-certificates='20230311ubuntu0.22.04.*' \
+        ca-certificates \
         curl='7.81.*' \
         dnsutils='1:9.18.*' \
         gnupg='2.2.*' \

packages/clickhouse-client.yaml

@@ -16,7 +16,7 @@ homepage: "https://altinity.com/"
 license: "Apache"
 section: "database"
 priority: "optional"
-maintainer: "ClickHouse Dev Team <packages+linux@clickhouse.com>"
+maintainer: "Altinity Dev Team https://github.com/Altinity/ClickHouse/"
 deb:
   fields:
     Source: clickhouse

packages/clickhouse-common-static-dbg.yaml

@@ -16,7 +16,7 @@ homepage: "https://altinity.com/"
 license: "Apache"
 section: "database"
 priority: "optional"
-maintainer: "ClickHouse Dev Team <packages+linux@clickhouse.com>"
+maintainer: "Altinity Dev Team https://github.com/Altinity/ClickHouse"
 deb:
   fields:
     Source: clickhouse

packages/clickhouse-common-static.yaml

@@ -16,7 +16,7 @@ homepage: "https://altinity.com/"
 license: "Apache"
 section: "database"
 priority: "optional"
-maintainer: "ClickHouse Dev Team <packages+linux@clickhouse.com>"
+maintainer: "Altinity Dev Team https://github.com/Altinity/ClickHouse"
 deb:
   fields:
     Source: clickhouse

packages/clickhouse-keeper-dbg.yaml

@@ -16,7 +16,7 @@ homepage: "https://altinity.com/"
 license: "Apache"
 section: "database"
 priority: "optional"
-maintainer: "ClickHouse Dev Team <packages+linux@clickhouse.com>"
+maintainer: "Altinity Dev Team https://github.com/Altinity/ClickHouse"
 deb:
   fields:
     Source: clickhouse

packages/clickhouse-keeper.yaml

@@ -16,7 +16,7 @@ homepage: "https://altinity.com/"
 license: "Apache"
 section: "database"
 priority: "optional"
-maintainer: "ClickHouse Dev Team <packages+linux@clickhouse.com>"
+maintainer: "Altinity Dev Team https://github.com/Altinity/ClickHouse"
 deb:
   fields:
     Source: clickhouse

packages/clickhouse-server.yaml

@@ -16,7 +16,7 @@ homepage: "https://altinity.com/"
 license: "Apache"
 section: "database"
 priority: "optional"
-maintainer: "ClickHouse Dev Team <packages+linux@clickhouse.com>"
+maintainer: "Altinity Dev Team https://github.com/Altinity/ClickHouse"
 deb:
   fields:
     Source: clickhouse

src/Core/Settings.h

@@ -594,6 +594,7 @@ class IColumn;
     M(Bool, optimize_use_projections, true, "Automatically choose projections to perform SELECT query", 0) ALIAS(allow_experimental_projection_optimization) \
     M(Bool, optimize_use_implicit_projections, true, "Automatically choose implicit projections to perform SELECT query", 0) \
     M(Bool, force_optimize_projection, false, "If projection optimization is enabled, SELECT queries need to use projection", 0) \
+    M(String, preferred_optimize_projection_name, "", "If it is set to a non-empty string, ClickHouse tries to apply specified projection", 0) \
     M(Bool, async_socket_for_remote, true, "Asynchronously read from socket executing remote query", 0) \
     M(Bool, async_query_sending_for_remote, true, "Asynchronously create connections and send query to shards in remote query", 0) \
     M(Bool, insert_null_as_default, true, "Insert DEFAULT values instead of NULL in INSERT SELECT (UNION ALL)", 0) \

src/Processors/QueryPlan/Optimizations/optimizeUseAggregateProjection.cpp

@@ -411,7 +411,6 @@ struct MinMaxProjectionCandidate
 {
     AggregateProjectionCandidate candidate;
     Block block;
-    MergeTreeData::DataPartsVector normal_parts;
 };
 
 struct AggregateProjectionCandidates
@@ -476,7 +475,6 @@ AggregateProjectionCandidates getAggregateProjectionCandidates(
         {
             // LOG_TRACE(&Poco::Logger::get("optimizeUseProjections"), "Projection analyzed DAG {}", proj_dag->dumpDAG());
             AggregateProjectionCandidate candidate{.info = std::move(info), .dag = std::move(proj_dag)};
-            MergeTreeData::DataPartsVector minmax_projection_normal_parts;
 
             // LOG_TRACE(&Poco::Logger::get("optimizeUseProjections"), "Projection sample block {}", sample_block.dumpStructure());
             auto block = reading.getMergeTreeData().getMinMaxCountProjectionBlock(
@@ -485,21 +483,20 @@ AggregateProjectionCandidates getAggregateProjectionCandidates(
                 dag.filter_node != nullptr,
                 query_info,
                 parts,
-                minmax_projection_normal_parts,
+                nullptr,
                 max_added_blocks.get(),
                 context);
 
             // LOG_TRACE(&Poco::Logger::get("optimizeUseProjections"), "Projection sample block 2 {}", block.dumpStructure());
 
-            // minmax_count_projection cannot be used used when there is no data to process, because
+            // minmax_count_projection cannot be used when there is no data to process, because
             // it will produce incorrect result during constant aggregation.
             // See https://github.com/ClickHouse/ClickHouse/issues/36728
             if (block)
             {
                 MinMaxProjectionCandidate minmax;
                 minmax.candidate = std::move(candidate);
                 minmax.block = std::move(block);
-                minmax.normal_parts = std::move(minmax_projection_normal_parts);
                 minmax.candidate.projection = projection;
                 candidates.minmax_projection.emplace(std::move(minmax));
             }
@@ -508,6 +505,18 @@ AggregateProjectionCandidates getAggregateProjectionCandidates(
 
     if (!candidates.minmax_projection)
     {
+        auto it = std::find_if(agg_projections.begin(), agg_projections.end(), [&](const auto * projection)
+        {
+            return projection->name == context->getSettings().preferred_optimize_projection_name.value;
+        });
+
+        if (it != agg_projections.end())
+        {
+            const ProjectionDescription * preferred_projection = *it;
+            agg_projections.clear();
+            agg_projections.push_back(preferred_projection);
+        }
+
         candidates.real.reserve(agg_projections.size());
         for (const auto * projection : agg_projections)
         {
@@ -569,49 +578,74 @@ bool optimizeUseAggregateProjections(QueryPlan::Node & node, QueryPlan::Nodes &
 
     auto candidates = getAggregateProjectionCandidates(node, *aggregating, *reading, max_added_blocks, allow_implicit_projections);
 
-    AggregateProjectionCandidate * best_candidate = nullptr;
-    if (candidates.minmax_projection)
-        best_candidate = &candidates.minmax_projection->candidate;
-    else if (candidates.real.empty())
-        return false;
-
     const auto & parts = reading->getParts();
+    const auto & alter_conversions = reading->getAlterConvertionsForParts();
     const auto & query_info = reading->getQueryInfo();
     const auto metadata = reading->getStorageMetadata();
     ContextPtr context = reading->getContext();
     MergeTreeDataSelectExecutor reader(reading->getMergeTreeData());
+    AggregateProjectionCandidate * best_candidate = nullptr;
 
-    auto ordinary_reading_select_result = reading->selectRangesToRead(parts, /* alter_conversions = */ {});
-    size_t ordinary_reading_marks = ordinary_reading_select_result->marks();
-
-    /// Selecting best candidate.
-    for (auto & candidate : candidates.real)
+    if (candidates.minmax_projection)
     {
-        auto required_column_names = candidate.dag->getRequiredColumnsNames();
-        ActionDAGNodes added_filter_nodes;
-        if (candidates.has_filter)
-            added_filter_nodes.nodes.push_back(candidate.dag->getOutputs().front());
+        best_candidate = &candidates.minmax_projection->candidate;
+    }
+    else if (!candidates.real.empty())
+    {
+        auto ordinary_reading_select_result = reading->selectRangesToRead(parts, alter_conversions);
+        size_t ordinary_reading_marks = ordinary_reading_select_result->marks();
 
-        bool analyzed = analyzeProjectionCandidate(
-            candidate, *reading, reader, required_column_names, parts,
-            metadata, query_info, context, max_added_blocks, added_filter_nodes);
+        /// Nothing to read. Ignore projections.
+        if (ordinary_reading_marks == 0)
+        {
+            reading->setAnalyzedResult(std::move(ordinary_reading_select_result));
+            return false;
+        }
 
-        if (!analyzed)
-            continue;
+        const auto & parts_with_ranges = ordinary_reading_select_result->partsWithRanges();
 
-        if (candidate.sum_marks > ordinary_reading_marks)
-            continue;
+        /// Selecting best candidate.
+        for (auto & candidate : candidates.real)
+        {
+            auto required_column_names = candidate.dag->getRequiredColumnsNames();
+            ActionDAGNodes added_filter_nodes;
+            if (candidates.has_filter)
+                added_filter_nodes.nodes.push_back(candidate.dag->getOutputs().front());
+
+            bool analyzed = analyzeProjectionCandidate(
+                candidate,
+                *reading,
+                reader,
+                required_column_names,
+                parts_with_ranges,
+                query_info,
+                context,
+                max_added_blocks,
+                added_filter_nodes);
 
-        if (best_candidate == nullptr || best_candidate->sum_marks > candidate.sum_marks)
-            best_candidate = &candidate;
-    }
+            if (!analyzed)
+                continue;
+
+            if (candidate.sum_marks > ordinary_reading_marks)
+                continue;
 
-    if (!best_candidate)
+            if (best_candidate == nullptr || best_candidate->sum_marks > candidate.sum_marks)
+                best_candidate = &candidate;
+        }
+
+        if (!best_candidate)
+        {
+            reading->setAnalyzedResult(std::move(ordinary_reading_select_result));
+            return false;
+        }
+    }
+    else
     {
-        reading->setAnalyzedResult(std::move(ordinary_reading_select_result));
         return false;
     }
 
+    chassert(best_candidate != nullptr);
+
     QueryPlanStepPtr projection_reading;
     bool has_ordinary_parts;
 
@@ -632,9 +666,7 @@ bool optimizeUseAggregateProjections(QueryPlan::Node & node, QueryPlan::Nodes &
                       .storage_id = reading->getMergeTreeData().getStorageID(),
                       .projection_name = candidates.minmax_projection->candidate.projection->name,
                   });
-        has_ordinary_parts = !candidates.minmax_projection->normal_parts.empty();
-        if (has_ordinary_parts)
-            reading->resetParts(std::move(candidates.minmax_projection->normal_parts));
+        has_ordinary_parts = false;
     }
     else
     {